Import the project as ReKoPub

Brings in the e-ink EPUB converter, renamed to ReKoPub (rekopub in
code), with module path git.monoexp.com/bajdev/rekopub.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitattributes

@@ -0,0 +1,4 @@
+# Batch files must keep Windows line endings: cmd.exe misreads some
+# lines in LF-only files.
+*.cmd text eol=crlf
+*.bat text eol=crlf

.github/workflows/ci.yml

@@ -0,0 +1,102 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  schedule:
+    # Weekly, so newly published vulnerabilities in Go or a dependency
+    # show up even when nothing has changed here.
+    - cron: "17 6 * * 1"
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      # The oldest Go that go.mod allows, so it keeps building there.
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: gofmt
+        run: |
+          unformatted=$(gofmt -l .)
+          if [ -n "$unformatted" ]; then
+            echo "These files need gofmt:"
+            echo "$unformatted"
+            exit 1
+          fi
+      - name: go vet
+        run: go vet ./...
+      - name: staticcheck
+        run: go run honnef.co/go/tools/cmd/staticcheck@2026.2.2 ./...
+
+  release-build:
+    # Builds every release binary without publishing, so a broken release
+    # config shows up on the PR rather than when a version is tagged.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v7
+        with:
+          go-version: stable
+          check-latest: true
+      - uses: goreleaser/goreleaser-action@v7
+        with:
+          version: "~> v2"
+          args: release --snapshot --clean
+
+  vulncheck:
+    # Reports known vulnerabilities in the Go standard library and the
+    # dependencies that the code actually calls.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
+        with:
+          go-version: stable
+          check-latest: true
+      - name: govulncheck
+        run: go run golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./...
+
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
+        with:
+          go-version: stable
+          check-latest: true
+      - uses: actions/setup-java@v6
+        with:
+          distribution: temurin
+          java-version: "21"
+      - name: Download epubcheck
+        shell: bash
+        run: |
+          curl -sSL -o "$RUNNER_TEMP/epubcheck.zip" \
+            https://github.com/w3c/epubcheck/releases/download/v${EPUBCHECK_VERSION}/epubcheck-${EPUBCHECK_VERSION}.zip
+          unzip -q "$RUNNER_TEMP/epubcheck.zip" -d "$RUNNER_TEMP"
+          echo "EPUBCHECK_JAR=$RUNNER_TEMP/epubcheck-${EPUBCHECK_VERSION}/epubcheck.jar" >> "$GITHUB_ENV"
+        env:
+          EPUBCHECK_VERSION: "5.2.1"
+      - name: Test
+        run: go test -race ./...
+      - name: Build
+        run: go build ./cmd/rekopub
+      - name: Build with build.cmd
+        if: runner.os == 'Windows'
+        shell: cmd
+        run: |
+          call build.cmd || exit /b 1
+          rekopub.exe -h || exit /b 1
+          rekopub.exe -VERSION || exit /b 1

.github/workflows/release.yml

@@ -0,0 +1,33 @@
+name: Release
+
+# Pushing a version tag builds the release binaries and creates a draft
+# GitHub release with them. Review the draft, then publish it.
+#   git tag v1.0.0 && git push origin v1.0.0
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: write # to create the release
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0 # goreleaser needs the tags and history for the changelog
+      # Release binaries include the Go standard library, so build them
+      # with the newest Go release to get its security fixes.
+      - uses: actions/setup-go@v7
+        with:
+          go-version: stable
+          check-latest: true
+      - name: govulncheck
+        run: go run golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./...
+      - uses: goreleaser/goreleaser-action@v7
+        with:
+          version: "~> v2"
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,11 @@
+# Built binaries
+/rekopub
+/rekopub.exe
+/dist/
+
+# Test output
+*.test
+*.out
+
+# Books made by test-examples.cmd
+/test-examples/

.goreleaser.yaml

@@ -0,0 +1,41 @@
+# Release builds: run by .github/workflows/release.yml when a version tag
+# (v1.0.0, v1.0.1, ...) is pushed. Check locally with:
+#   goreleaser release --snapshot --clean
+version: 2
+
+project_name: rekopub
+
+builds:
+  - id: rekopub
+    main: ./cmd/rekopub
+    binary: rekopub
+    env:
+      - CGO_ENABLED=0 # pure Go: one self-contained file per platform
+    goos: [windows, linux, darwin]
+    goarch: [amd64, arm64]
+    flags: [-trimpath]
+    ldflags:
+      - -s -w -X main.version={{ .Version }}
+
+archives:
+  - id: rekopub
+    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
+    formats: [tar.gz]
+    format_overrides:
+      - goos: windows
+        formats: [zip]
+    files:
+      - README.md
+      - LICENSE
+
+checksum:
+  name_template: checksums.txt
+
+changelog:
+  use: github
+  sort: asc
+
+release:
+  # Created as a draft so the binaries and notes can be checked before the
+  # release is published.
+  draft: true

LICENSE

@@ -1,8 +1,21 @@
 MIT License
-Copyright (c) <year> <copyright holders>
 
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+Copyright (c) 2026 bajdev
 
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

PLAN.md

@@ -0,0 +1,544 @@
+# ReKoPub: Implementation Plan
+
+## 1. Goal
+
+ReKoPub is a command-line program written in Go. You give it a web page URL and a path on your computer. It downloads the page, extracts the readable article content (dropping navigation, ads, comments and similar clutter), and writes a valid EPUB 3 file to that path.
+
+```bash
+rekopub -URL https://example.com/some-article -SAVE ~/Books/some-article.epub
+rekopub -URL https://example.com/some-article -SAVE ~/Books/      # a folder (created if missing): filename comes from the article title
+rekopub -URL https://example.com/some-article -SAVE ~/Books/a.epub -FORCE   # replace an existing file
+rekopub -URL https://example.com/some-article -SAVE /media/$USER/KOBOeReader/Articles/ -KEPUB   # Kobo-optimised
+```
+
+### Target devices
+
+The primary targets are two grayscale e-ink readers. Every output decision (image format, image size, CSS, cover) is tuned for them first:
+
+| | reMarkable 2 | Kobo Clara HD |
+|---|---|---|
+| Screen | 10.3", 1872×1404 px, 226 PPI | 6", 1448×1072 px, 300 PPI |
+| Colour | Grayscale only | Grayscale only (16 levels, E Ink Carta) |
+| Aspect ratio | 3:4 | ≈3:4 |
+| EPUB rendering | Reflows EPUB with its own renderer; limited CSS support | Plain `.epub` uses the older Adobe RMSDK engine; `.kepub.epub` uses Kobo's faster WebKit engine |
+| Getting the book onto it | You upload the saved file yourself (desktop or mobile app, or the USB web interface at `http://10.11.99.1`). ReKoPub only makes the file | USB mass storage (FAT32): `-SAVE` straight onto the device, or copy the file |
+
+Both screens share a 3:4 aspect ratio, so one image size and one cover size fit both. The default plain EPUB must work on both devices. `-KEPUB` produces a Kobo-optimised `.kepub.epub` instead (§5.7). Use it for the Clara HD, and plain EPUB for the reMarkable.
+
+### Scope for v1
+- One URL in, one EPUB out.
+- Extracts the main content plus metadata: title, author, date, site name, language.
+- Images are downloaded and embedded, so the book works offline. They are converted to grayscale and sized for the target screens.
+- Never overwrites an existing file unless `-FORCE` is given. Missing output folders are created.
+- `-KEPUB` writes a Kobo KEPUB (`.kepub.epub`) instead of a plain EPUB.
+- The output passes `epubcheck` (the standard EPUB validator) with zero errors.
+- Ships as a single static binary with no runtime dependencies.
+
+### Out of scope for v1 (see §9)
+- Several URLs in one book (added in M7, §8). Crawling a site and following "next page" links stay out of scope.
+- Pages that only show content after JavaScript runs.
+- Pages behind a login or paywall.
+- Uploading to a device. ReKoPub stays focused on making the file; you copy it over yourself.
+- A web UI or hosted service.
+
+---
+
+## 2. Command-line interface
+
+The program uses Go's standard-library `flag` package. Flag names are case-sensitive: exactly `-URL`, `-SAVE`, `-FORCE` and `-KEPUB`. (`flag` also accepts `--URL` and the other double-dash forms.)
+
+```
+Usage: rekopub -URL <url> -SAVE <path> [-FORCE] [-KEPUB]
+
+  -URL string
+        Web page to convert (required). Must be http:// or https://.
+  -SAVE string
+        Where to write the EPUB (required). Missing folders are created.
+          - Ends in "/"            → a folder: <folder>/<slug-of-article-title>.epub
+          - An existing folder     → same as above
+          - Ends in .epub          → written to exactly that file
+          - Anything else          → treated as a file path; ".epub" is appended
+        With -KEPUB, the extension is ".kepub.epub" instead (see below).
+  -FORCE
+        Overwrite the output file if it already exists. Without it, an
+        existing file is never touched and the program exits with code 5.
+  -KEPUB
+        Write a Kobo KEPUB (.kepub.epub) instead of a plain EPUB. Kobo
+        devices open it with their faster WebKit renderer. Use it for the
+        Kobo Clara HD; leave it off for the reMarkable.
+```
+
+How the flags behave:
+- **Validation**:
+  - Missing `-URL` or `-SAVE` prints the usage text and exits with code `2`.
+  - A URL that doesn't parse, or isn't http/https, is rejected before anything is fetched.
+  - Any positional arguments left over are an error, which catches typos like `-url`.
+- **The `-SAVE` path**:
+  - A leading `~` is expanded, in case the shell didn't do it (for example, when the path is quoted).
+  - A path that doesn't exist yet is a file unless it ends in `/`. So `-SAVE ~/Books/new/` creates the folder `new`, while `-SAVE ~/Books/new` writes `~/Books/new.epub`.
+  - **Folders are created automatically** (`os.MkdirAll`, mode `0755`), including any missing parent folders. They are created only at save time, so a run that fails earlier (for example, the download fails) doesn't leave empty folders behind.
+  - If part of the path exists as a regular file (for example, `~/Books` is a file), the program fails with exit code `4` and a clear message.
+- **Filenames with `-KEPUB`**: Kobo picks its renderer from the file extension. A KEPUB only gets the faster renderer if the file ends in `.kepub.epub`, so the extension is set automatically:
+  - Folder targets get `<slug>.kepub.epub`.
+  - `-SAVE x.kepub.epub` is used as is.
+  - `-SAVE x.epub` becomes `x.kepub.epub`, with a note on stderr. Otherwise the Kobo would open the file with the old renderer and the conversion would be wasted.
+  - Any other path gets `.kepub.epub` appended.
+  - A `-SAVE` path ending in `.kepub.epub` **without** `-KEPUB` is an error (exit code `2`, "add -KEPUB or use a .epub filename"). A plain EPUB with that name would be opened by the Kobo's KEPUB renderer and display badly.
+  - The overwrite check below always uses the final name, after the extension is adjusted.
+- **Overwrite protection (`-FORCE`)**:
+  - **Without `-FORCE`**:
+    - An existing file at the output path is never modified. The program prints `error: <path> already exists (use -FORCE to overwrite)` and exits with code `5`.
+    - The check runs as early as possible, to avoid wasted downloads. When `-SAVE` names a file, the check runs before the page is fetched. When `-SAVE` names a folder, the filename depends on the article title, so the check runs right after extraction and before any images are downloaded.
+    - The final write is also race-free, in case another program creates the file mid-run. The name is first reserved with `O_CREATE|O_EXCL`, which fails if the file exists. The book is then written to a temp file in the same folder and renamed over the reservation. If anything fails, both the reservation and the temp file are removed.
+  - **With `-FORCE`**:
+    - The existing file is replaced atomically (written to a temp file, then renamed into place). Until the new file is fully written, the old one stays intact.
+    - `-FORCE` never deletes a folder. If the output path is an existing folder, the "existing folder" rule from `-SAVE` applies instead.
+  - Either way, a failed run never leaves a half-written `.epub` behind.
+- **Exit codes**:
+
+  | Code | Meaning |
+  |---|---|
+  | `0` | Success. The absolute path of the written file is printed to stdout. |
+  | `1` | Fetch failed (network error, non-200 status, page isn't HTML). |
+  | `2` | Bad arguments. |
+  | `3` | No readable content could be extracted. |
+  | `4` | Building or writing the EPUB failed (including folder creation). |
+  | `5` | The output file already exists and `-FORCE` was not given. |
+
+- **Output**: progress and warnings go to stderr (for example, `warning: image https://… failed: 404`), so stdout stays clean for scripting.
+- **Saving straight to a Kobo**: `-SAVE /media/<user>/KOBOeReader/Articles/` works, because the Kobo mounts as a FAT32 USB drive. Filenames are already restricted to ASCII letters, digits and `-`, which FAT32 accepts. The folder is created on the device if it's missing.
+
+---
+
+## 3. Technology choices
+
+The project targets Go 1.26 or newer. The module path is `git.monoexp.com/bajdev/rekopub`.
+
+| Concern | Package | Why |
+|---|---|---|
+| CLI | `flag` (stdlib) | Matches the `-URL` / `-SAVE` / `-FORCE` style with no extra dependency |
+| HTTP | `net/http` (stdlib) | Timeouts, redirects and proxy support are built in |
+| Charset decoding | `golang.org/x/net/html/charset` | Picks the encoding from the HTTP header, then the `<meta>` tag, then detection |
+| HTML parsing / DOM work | `golang.org/x/net/html` | Standard HTML5 parser; the cleanup passes walk its node tree directly (goquery turned out to be unnecessary) |
+| Content extraction | `github.com/markusmobius/go-trafilatura` | Go port of trafilatura. Returns metadata (title, author, date, site name, language, image), and already falls back to readability and DOM-distiller internally |
+| Sanitizing | `github.com/microcosm-cc/bluemonday` | Mature allow-list HTML sanitizer |
+| Images | `image/jpeg`, `image/png`, `image/gif`, `image/color/palette` (stdlib), `golang.org/x/image/webp`, `golang.org/x/image/draw` | Decoding (WebP included), high-quality resizing, grayscale conversion and 16-level quantizing |
+| SVG images | `github.com/srwiley/oksvg` + `github.com/srwiley/rasterx` | Pure-Go SVG rasterizer. SVGs are turned into grayscale PNGs, because neither target renders SVG reliably |
+| AVIF images (optional) | `github.com/gen2brain/avif` | Pure Go (WASM-based), so the binary still needs no cgo |
+| EPUB packaging | `github.com/go-shiori/go-epub` | Maintained fork of `bmaupin/go-epub`. Writes EPUB 3 with a nav document, an NCX, cover, CSS and images |
+| Cover text | `golang.org/x/image/font/opentype` + the Go fonts (`golang.org/x/image/font/gofont`, BSD licence) | Draws the title on generated covers. The fonts ship inside `x/image`, so there is no font file to add |
+| KEPUB output (`-KEPUB`, §5.7) | `github.com/pgaskin/kepubify/v4/kepub` | The standard Go library for converting EPUB to Kobo's KEPUB format |
+| Book identifier | `github.com/google/uuid` | `uuid.NewSHA1` gives a stable ID for each URL |
+| Testing | `testing` + `net/http/httptest` (stdlib), `epubcheck` in CI | A real local HTTP server instead of mocks |
+| Lint | `go vet`, `staticcheck` or `golangci-lint` | |
+| Release | `goreleaser` v2 | Static binaries for linux, macOS and windows on amd64 and arm64, with checksums; a version tag creates a draft GitHub release |
+
+The build uses no cgo (`CGO_ENABLED=0`), so `go install git.monoexp.com/bajdev/rekopub/cmd/rekopub@latest` works everywhere.
+
+---
+
+## 4. Architecture
+
+The program is a straight pipeline of small packages under `internal/`. Each package can be tested on its own:
+
+```
+ -URL
+   │
+   ▼
+┌─────────┐   ┌──────────┐   ┌─────────┐   ┌─────────┐   ┌─────────┐   ┌─────────┐
+│  fetch  │──▶│ extract  │──▶│  clean  │──▶│ assets  │──▶│  build  │──▶│  save   │──▶ -SAVE path
+│net/http │   │trafilatura│  │bluemonday│  │download,│   │ go-epub │   │temp file│
+│+charset │   │+metadata │   │→ XHTML  │   │convert  │   │         │   │+ rename │
+└─────────┘   └──────────┘   └─────────┘   └─────────┘   └─────────┘   └─────────┘
+                                                                  │  -KEPUB  ▲
+                                                                  └─▶ kepub ─┘
+                                                                     (kepubify)
+```
+
+### Project layout
+
+```
+rekopub/
+├── go.mod
+├── go.sum
+├── README.md
+├── PLAN.md
+├── cmd/rekopub/
+│   └── main.go              # flag parsing, -SAVE resolution, exit codes
+├── internal/
+│   ├── pipeline/pipeline.go # Run(ctx, url) → *Book; wires the stages together
+│   ├── fetch/fetch.go       # HTTP client, size limits, charset decoding
+│   ├── extract/extract.go   # go-trafilatura + metadata fallbacks
+│   ├── clean/clean.go       # URL resolution, lazy images, sanitizing, XHTML output
+│   ├── device/profile.go    # e-ink target constants (image box, gray levels, cover size)
+│   ├── assets/assets.go     # concurrent image download, grayscale conversion, resizing
+│   ├── build/
+│   │   ├── build.go         # go-epub packaging
+│   │   ├── cover.go         # generated cover when the article has no lead image
+│   │   ├── chapter.tmpl     # title / byline / source header
+│   │   └── style.css        # embedded with //go:embed
+│   ├── kepub/kepub.go       # -KEPUB conversion via kepubify (§5.7)
+│   └── save/save.go         # path resolution, folder creation, -FORCE handling, atomic write
+└── testdata/
+    ├── pages/               # saved HTML: blog, news, Substack, docs, lazy images, Shift-JIS, RTL
+    └── golden/              # expected extraction output (title, author, text)
+```
+
+### Core types (`internal/pipeline`)
+
+```go
+type Page struct {
+    URL         *url.URL // final URL after redirects
+    HTML        string   // decoded to UTF-8
+    ContentType string
+}
+
+type Image struct {
+    SourceURL string
+    Filename  string // "images/3fa2c1d9e0ab.jpg" (content hash)
+    MediaType string // image/jpeg, image/png, ...
+    Data      []byte
+}
+
+type Article struct {
+    URL          string
+    Title        string
+    Author       string
+    Published    time.Time
+    SiteName     string
+    Language     string // BCP-47, default "en"
+    LeadImageURL string
+    ContentXHTML string // sanitized, well-formed XHTML fragment
+    Images       []Image
+}
+```
+
+### Device profile (`internal/device`)
+
+The e-ink tuning lives in one place, so a different device can be supported later by adding a second profile:
+
+```go
+// EInk targets reMarkable 2 (1404×1872) and Kobo Clara HD (1072×1448).
+// Both are 3:4 grayscale, so images are sized to the larger screen and
+// the Kobo scales them down.
+var EInk = Profile{
+    MaxImageWidth:  1404,
+    MaxImageHeight: 1872,
+    CoverWidth:     1404,
+    CoverHeight:    1872,
+    Grayscale:      true,
+    GrayLevels:     16,   // both panels display 16 shades
+    JPEGQuality:    80,
+}
+```
+
+The `main` function stays thin:
+
+```go
+func main() {
+    urlFlag  := flag.String("URL", "", "web page to convert (required)")
+    saveFlag := flag.String("SAVE", "", "file or folder to write the EPUB to (required)")
+    force    := flag.Bool("FORCE", false, "overwrite the output file if it already exists")
+    kepubOut := flag.Bool("KEPUB", false, "write a Kobo KEPUB (.kepub.epub) instead of a plain EPUB")
+    flag.Parse()
+    // validate → resolve -SAVE (extension depends on -KEPUB) → early exists-check (unless -FORCE)
+    // → ctx with signal.NotifyContext(SIGINT) → pipeline.Run → build EPUB
+    // → if -KEPUB: kepub.Convert → save.Write(path, bytes, force) → print path
+}
+```
+
+---
+
+## 5. Stage details
+
+### 5.1 fetch
+- Uses one `http.Client` with:
+  - a 60 s overall timeout, plus dial and TLS-handshake timeouts;
+  - redirects followed up to 10 times;
+  - the proxy taken from the environment (`http.ProxyFromEnvironment`).
+- Sends request headers:
+  - a realistic `User-Agent`, because some sites return 403 to Go's default UA;
+  - `Accept: text/html,application/xhtml+xml`.
+- Rejects any status other than 2xx, and any content type other than `text/html` or `application/xhtml+xml`.
+- Caps the body at 20 MB (`io.LimitReader`), with a clear error if the page is larger.
+- Converts the body to UTF-8 with `charset.NewReader(body, contentType)`.
+- Retries twice, with backoff, on 429 and 5xx responses and on temporary network errors.
+- Every request uses the `context.Context`, so Ctrl-C cancels cleanly.
+
+### 5.2 extract
+- Calls `trafilatura.Extract(reader, trafilatura.Options{OriginalURL: u, IncludeImages: true, IncludeLinks: true, EnableFallback: true})`.
+  - The result's `ContentNode` (an `*html.Node`) is the article body.
+  - The result's `Metadata` holds the title, author, date, site name, language and image.
+- When trafilatura leaves a metadata field empty, it is filled from the page's own declarations (read before extraction, which modifies the page):
+  - `og:title`, `og:site_name`, `article:author`, `article:published_time`;
+  - JSON-LD `Article` / `NewsArticle` / `BlogPosting`;
+  - `<title>` and `<html lang>`.
+- `LeadImageURL` comes from `og:image`, then `twitter:image`.
+- If the extracted text is under about 200 characters, extraction fails with exit code `3`. The error message suggests the page may need JavaScript, which is not supported in v1.
+
+### 5.3 clean
+EPUB content files must be **well-formed XHTML**, and e-readers are strict about it. This stage works on the `*html.Node` tree:
+1. **Make URLs absolute**: resolve every `href` and `src` against the final page URL with `url.ResolveReference`.
+2. **Fix lazy-loaded images**:
+   - Move `data-src`, `data-original` or `data-lazy-src` into `src` when one is present.
+   - Pick the smallest candidate from `srcset` or `<picture><source>` that is at least 1404 px wide (the reMarkable's screen width). If none is that wide, take the largest.
+   - Remove 1×1 tracking pixels, and images without a usable `src`.
+3. **Replace embeds**: turn YouTube, Vimeo and Twitter/X iframes into a paragraph link (`▶ Video: <link>`), so the reader knows something was there.
+4. **Sanitize** with a `bluemonday` policy built from an allow-list:
+   - Tags: `p h1–h6 blockquote pre code em strong b i u s sub sup a img figure figcaption ul ol li dl dt dd table thead tbody tfoot tr th td hr br span div aside abbr cite q small`.
+   - Attributes: `href` on `a`; `src`, `alt` and `title` on `img`; `colspan` and `rowspan` on cells; `lang` and `dir` globally.
+   - Everything else is removed: `script`, `style`, `iframe`, `form`, `video`, `on*` handlers, `class`, `id` and inline styles.
+5. **Normalize headings**: the article title becomes the chapter's only `<h1>`. Body headings shift down so none sits above `<h2>`.
+6. **Serialize to XHTML**: render with `html.Render`, which closes void elements (`<br/>`), quotes attributes and escapes `& < >`. Then check the result with `encoding/xml`'s decoder. If it isn't well-formed, fail loudly in tests rather than producing an EPUB that e-readers refuse to open. HTML named entities such as `&nbsp;` are never emitted, because the content is written as raw UTF-8.
+
+### 5.4 assets
+- Collects every `<img src>` in the cleaned content, plus the lead image.
+- Downloads them concurrently with an `errgroup` and a limit of 8 at a time. Each request sends `Referer: <article URL>`, because some CDNs require it, and each image is capped at 10 MB.
+- Names files by content hash (`images/<sha1[:12]>.<ext>`), so duplicate images are stored once. Each `src` is rewritten to its local filename.
+- **Every image is processed for e-ink**, using the values in `device.EInk`:
+  1. **Decode** JPEG, PNG, GIF and WebP (and AVIF, if that optional decoder is included). SVG is rasterized with `oksvg` at the target width. GIFs keep only their first frame, since e-ink can't animate. Images in any other format are dropped, and their alt text is kept as a placeholder.
+  2. **Flatten transparency onto white.** Some readers draw transparent areas as black, which ruins logos and diagrams.
+  3. **Convert to grayscale** using the luminance formula (BT.601). Both devices are grayscale anyway, so the image looks the same, and the file is about a third of the size. Doing it ahead of time also keeps each device's own conversion from altering the result.
+  4. **Resize** to fit inside 1404×1872, keeping the aspect ratio, with `draw.CatmullRom`. Images are never upscaled.
+  5. **Encode**:
+     - Photos (many distinct tones) become 8-bit grayscale JPEG at quality 80.
+     - Line art, screenshots, charts and diagrams (few distinct tones) become PNG, quantized to a 16-level gray palette. That matches the panels exactly and keeps text in screenshots crisp, with no JPEG artifacts.
+     - The choice is made per image by counting distinct gray levels after conversion.
+  - WebP and SVG are never written into the book. Kobo's RMSDK renderer can't display WebP, and SVG support on both devices is unreliable.
+- A failed download or decode is a warning on stderr, never a fatal error. The `<img>` becomes `<span class="missing-image">[Image: alt text]</span>` (or is removed if it has no alt text); a `<span>` is valid everywhere an image is.
+- Files stay small: a typical article with 10 images should come out well under 5 MB. That matters for the reMarkable, which converts each EPUB on import, and for page-turn speed on the Clara HD.
+
+### 5.5 build
+Uses `go-epub`:
+- `epub.NewEpub(title)`, then sets:
+  - author, language, description, and a UUID identifier (`uuid.NewSHA1` of the URL, so rebuilding the same URL gives the same ID);
+  - `dc:source` set to the URL, and the publisher set to the site name.
+- `AddCSS` with the embedded `style.css`, written for e-ink and for the limited CSS support of RMSDK and the reMarkable:
+  - **No colour and no gray text.** Body text, bylines, captions and links are all pure black. Links are shown by an underline, not colour, because gray or coloured text looks faint on e-ink.
+  - **No background colours or shading**, including on `pre`, `blockquote` and table headers. Light gray fills turn into dithered noise on e-ink. Blockquotes get a solid black left border instead. Table headers are bold with 1px solid black borders.
+  - **Let the reader control layout.** No fixed font sizes, no `font-family` beyond generic `serif`/`monospace`, and no page margins or `max-width` in px. Both devices apply the user's own font, size and margin settings.
+  - **Images**: `img { max-width: 100%; height: auto; }` and figures set to `page-break-inside: avoid`, so a figure isn't split across two pages.
+  - **Code**: `pre { white-space: pre-wrap; word-wrap: break-word; font-size: 0.85em; }`. E-ink can't scroll sideways, so long lines must wrap.
+  - Only simple, widely supported CSS is used: no flexbox, grid, `calc()` or CSS variables, which RMSDK ignores or mis-renders.
+- Adds every image with `AddImage`, from a local temp file or data URL, so go-epub never downloads anything itself.
+- Adds one chapter with `AddSection(body, title, "chapter.xhtml", cssPath)`. The body is `chapter.tmpl` filled in:
+  ```html
+  <h1>{{.Title}}</h1>
+  <p class="byline">{{.Author}} · {{.SiteName}} · {{.Published}}</p>
+  <p class="source"><a href="{{.URL}}">{{.URL}}</a></p>
+  {{.ContentXHTML}}
+  ```
+- Builds the **table of contents** from the article's `<h2>` and `<h3>` headings:
+  - inject `id` anchors into those headings;
+  - write the EPUB 3 `nav.xhtml` and the NCX ourselves, with the `<h2>` sections nested under the article and `<h3>` subsections under those. go-epub can only list whole files (its `AddSubSection` creates new files), and its NCX lacks `dtb:uid`, which Kobo's RMSDK renderer reads. The finished zip is rewritten to replace both files and to add `dc:publisher`, `dc:source` and `dc:date` to the OPF, which go-epub has no setters for.
+- The **cover** is always 1404×1872 grayscale JPEG. That is the reMarkable's exact screen size and the same 3:4 shape as the Clara HD, so it fills the screen on both devices and in their library views.
+  - With a lead image: the image is cropped to fill the top three-fifths, with the title, site name and date set in large black text on a white band below. Library thumbnails are tiny on e-ink, so the title must be readable at thumbnail size.
+  - Without one: a plain typographic cover with the title, site name and date in black on white, with a thin black rule.
+  - Text is drawn with `x/image/font/opentype` and the embedded font. Long titles are wrapped and shrunk to fit.
+  - It is set with `SetCover`. go-epub also writes the cover page and the `cover-image` property that Kobo and reMarkable use for thumbnails.
+
+### 5.6 save
+- `save.Resolve(saveFlag, title)` turns `-SAVE` into the final file path, using the rules in §2.
+  - The filename slug comes from the title: lowercase ASCII, with runs of other characters collapsed to `-`, at most 80 characters. If the slug comes out empty, the filename falls back to `article.epub`.
+  - Because the slug never contains path separators, nothing from the page can steer the file outside the chosen folder.
+- `save.CheckExists(path, force)` is the early check from §2. It is called by `main` before fetching when `-SAVE` names a file, and by the pipeline right after extraction when it names a folder.
+- `save.Write(path, book, force)`:
+  1. `os.MkdirAll(filepath.Dir(path), 0o755)`.
+  2. Without `force`: reserve the name with `os.OpenFile(path, O_CREATE|O_EXCL|O_WRONLY, 0o644)`. If that returns `fs.ErrExist`, stop with exit code 5.
+  3. Write the book with `e.WriteTo()` into `os.CreateTemp(dir, ".rekopub-*")`, then `Sync` and `Close` it.
+  4. `os.Rename(temp, path)`. This replaces either the reservation or, with `-FORCE`, the old file.
+  5. On any error, remove the temp file and the reservation, but never a file that existed before the run.
+- Rename-into-place works on FAT32 too, so saving straight onto a Kobo is safe.
+
+### 5.7 KEPUB for Kobo (`-KEPUB`)
+The Clara HD renders plain `.epub` files with Adobe's older RMSDK engine. Files ending in `.kepub.epub` use Kobo's own WebKit-based engine instead, which gives:
+- faster page turns and better CSS support;
+- reading statistics (time left in the chapter or book), and more reliable highlights and bookmarks.
+
+How the conversion works:
+1. `build` always produces a normal EPUB first, into an in-memory buffer. This is the same book the reMarkable gets.
+2. `internal/kepub` opens that buffer with `zip.NewReader`. It then calls kepubify's `kepub.NewConverterWithOptions(...).Convert(ctx, w, zipReader)`, writing straight into the temp file that `save` provides.
+   - kepubify wraps every sentence in `<span class="koboSpan">`. Kobo needs these spans for reading statistics and highlights.
+   - It adds the `kobostylehacks` wrapper and makes sure the `cover-image` property is set.
+3. `save` renames the result into place, applying the same `-FORCE` rules as any other output.
+
+Converter options:
+- `ConverterOptionHyphenate(true)`: turns on hyphenation, which helps justified text on the 6" screen.
+- `ConverterOptionSmartypants()`: left **off**. It rewrites quotes and dashes, which would change the article's text.
+- `ConverterOptionAddCSS`: not used. The e-ink stylesheet from §5.5 already suits Kobo.
+
+Validation:
+- `epubcheck` runs on the plain EPUB before conversion.
+- In practice kepubify's output also passes epubcheck with no errors or warnings, so the KEPUB tests run epubcheck too, alongside structural tests (sentence spans, unchanged text, cover and contents kept) and checks on a real device.
+
+---
+
+## 6. Edge cases and risks
+
+| Issue | Handling in v1 |
+|---|---|
+| Pages that need JavaScript (SPAs) | Detect near-empty extraction, exit with code `3` and a clear message. JavaScript rendering with `chromedp` is a future option (§9) |
+| Paywalls and cookie or consent walls | Overlays are stripped by sanitizing. Content the server never sends is out of scope |
+| Bot protection (Cloudflare and similar) | Use a realistic UA. Otherwise report the 403 clearly. Never try to get around it |
+| Malformed HTML | `x/net/html` follows the HTML5 parsing algorithm, so it always produces a tree |
+| Non-UTF-8 pages | Decoded with `x/net/html/charset`. Output is always UTF-8 |
+| Huge pages or images | 20 MB page cap, 10 MB per image, downscaling to 1404×1872 |
+| Charts that rely on colour | Grayscale makes some colours look identical, so a legend can become unreadable. v1 keeps the image as is and the alt text or caption. Raising contrast is a possible later improvement |
+| Wide tables on a 6" screen | CSS lets tables wrap their cells. A table wider than the screen stays readable on the reMarkable but may be cramped on the Clara HD. Turning wide tables into images is out of scope |
+| Long code lines | `pre-wrap`, because e-ink readers can't scroll sideways |
+| go-trafilatura quirks | Found while building M2, and worked around in `internal/clean`:<br>• It resolves relative links as if the page URL were a folder, so links are made absolute before extraction.<br>• It joins `mailto:`/`tel:` links onto the page path; these are repaired afterwards.<br>• It turns `<pre>` code blocks into `<code>`; block-level `<code>` is turned back into `<pre><code>`. |
+| Table cell spans | go-trafilatura drops `colspan`/`rowspan`, so a table that relies on them can come out with uneven rows. Known limitation in v1 |
+| Image URLs without a file extension | go-trafilatura keeps an image only when `filepath.Ext` of its whole URL is an image type, so it drops URLs with a query string (`photo.jpg?utm_source=…`, as on every Wikipedia image), extensionless CDN URLs and `data:` URLs. `Prepare` replaces each image URL with a stand-in (`https://rekopub.invalid/img/<encoded URL>.png`) that passes the check, and `Content` restores the original |
+| Images in zoom links | go-trafilatura drops an image wrapped in a link (such as a "click to zoom" lightbox link) when it sits in a `<div>` or `<p>`. WordPress themes, including Ars Technica's, do this for every image. `Prepare` unwraps links that contain only an image |
+| Figure captions | go-trafilatura drops `<figcaption>`. `Prepare` turns each caption into an italic paragraph, which survives extraction, and `Content` rebuilds the `<figure>`. Captions laid out with `<div>`s are reduced to their text first |
+| Invented dates | go-trafilatura's default date search scans body text and can invent a date from a stray number (an image `width="2000"` became 1 January 2000). It runs in `Fast` mode, which only trusts metadata, URLs and date markup |
+| AVIF and JPEG XL images | Not decoded in v1. `<picture>` sources in those formats are skipped in favour of a JPEG, PNG, GIF or WebP alternative; a lone AVIF image is replaced by its alt text |
+| Math (MathJax/KaTeX) | Keep MathML if the page has it (EPUB 3 supports MathML). Otherwise keep the TeX source as text |
+| Right-to-left languages | Carry `lang` and `dir` through, and set the spine's `page-progression-direction` |
+| Copyright | For personal offline reading. Every book records the source URL in `dc:source` and in the chapter header |
+
+**Security**
+- Page scripts never run.
+- Everything that goes into the EPUB is sanitized, because some e-readers run JavaScript inside books.
+- Download sizes and times are capped.
+- `-SAVE` is used exactly as given. Nothing taken from the page (such as the title) can push the output outside the chosen directory, because the slug never contains path separators.
+- Only `http` and `https` URLs are fetched, for the page, its redirects, images and cover. Local files are never read: `file:` URLs and local paths in a page are refused (tests in `cmd/rekopub`, `assets` and `fetch` cover each route).
+- When `-URL` is on the internet, connections to loopback, private, link-local, carrier-grade NAT and reserved addresses are refused. The check runs on the address actually dialled, after DNS lookup, so redirects and DNS rebinding can't get around it. If `-URL` is itself local, local addresses are allowed. Configured proxies are exempt.
+- Release binaries are built with the newest Go release, and govulncheck runs on every PR, weekly and before each release.
+
+---
+
+## 7. Testing strategy
+
+- **Unit tests per package**, written as table-driven tests in Go:
+  - `fetch`: `httptest.Server` covering redirects, 404, 429 retry, wrong content type, oversized body, Shift-JIS and ISO-8859-1 pages.
+  - `extract`: golden tests over `testdata/pages/*.html`, compared with `testdata/golden/*.json`. A `-update` flag regenerates the golden files.
+  - `clean`: lazy-image and `srcset` selection, sanitizing, heading changes, and an XHTML well-formedness check on every fixture.
+  - `assets`: every output image is grayscale and fits 1404×1872. A photo becomes JPEG, line art becomes a 16-level PNG. Transparency is flattened to white. WebP and SVG input never appear in the output. The first frame is taken from animated GIFs. Images are never upscaled. Also covers deduplication and failed downloads.
+  - `kepub`: convert a fixture EPUB, then check that the result is a valid zip with `mimetype` first, the content contains `koboSpan` spans, the OPF still marks the cover with `cover-image`, and the text is unchanged once the spans are removed.
+  - `save`, all in `t.TempDir()`:
+    - `-SAVE` resolution: `.epub` file, existing folder, trailing `/`, missing extension, `~`.
+    - Folder creation: nested missing folders, and part of the path being an existing file (exit code 4).
+    - **Without `-FORCE`**: an existing file means exit code 5, with the file's bytes and modification time unchanged. A file created by someone else between the early check and the write is also left untouched.
+    - **With `-FORCE`**: the file is replaced, and the old file survives if the build fails part-way.
+    - No temp or reservation files are left behind after any failure.
+- **End-to-end** (`cmd/rekopub`): an `httptest.Server` serves a fixture page and its images. The test runs the CLI with `-URL` and `-SAVE`, then checks:
+  - the exit code;
+  - the zip layout: `mimetype` is the first entry and stored uncompressed, `META-INF/container.xml` exists, every manifest item is present;
+  - `epubcheck` reports zero errors.
+- **CLI flag tests**: missing flags, bad URL, extra arguments, `-FORCE`, `-KEPUB`, and the exit code for each. `-KEPUB` cases: a folder target gets `.kepub.epub`, `x.epub` is rewritten to `x.kepub.epub` with a note, a `.kepub.epub` path without `-KEPUB` exits with code 2, and the overwrite check uses the adjusted name. An existing output file must fail before any HTTP request is made: the test server counts its requests.
+- **Device checks (manual, at each milestone from M3 onward)**: sideload a set of reference books onto a real reMarkable 2 and Kobo Clara HD. The set covers a long text article, an image-heavy post, code and tables, and a non-Latin script. Check:
+  - the cover and its thumbnail in the library;
+  - the table of contents;
+  - image sharpness;
+  - page-turn speed;
+  - that the user's font and size settings still apply;
+  - on the Clara HD, the `-KEPUB` version too: it should open in the KEPUB renderer and show time-left statistics.
+
+  Record the results in `docs/device-testing.md`.
+- **CI (GitHub Actions)**:
+  - `go vet`, `staticcheck` and `go test -race ./...`;
+  - epubcheck (Java) for the end-to-end test;
+  - a `goreleaser` dry run.
+
+---
+
+## 8. Milestones
+
+| # | Milestone | Deliverable |
+|---|---|---|
+| M0 | Scaffold | `go.mod`, layout, `main.go` with `-URL`/`-SAVE`/`-FORCE`/`-KEPUB` parsing and validation, the `save` package (path rules, folder creation, `-FORCE`, atomic writes), CI workflow. `-KEPUB` conversion itself arrives in M5 |
+| M1 | Basic working version | fetch, trafilatura and go-epub produce a text-only EPUB |
+| M2 | Clean XHTML and e-ink CSS | Sanitizer, URL resolution, heading changes, chapter template, e-ink stylesheet. epubcheck passes in CI. First check on real devices |
+| M3 | Images for e-ink | Lazy-image fixes, concurrent download, grayscale, resizing to 1404×1872, JPEG or 16-level PNG choice, SVG rasterizing, deduplication |
+| M4 | Metadata and cover | Metadata fallbacks, 1404×1872 grayscale cover (lead image or typographic), heading-based table of contents |
+| M5 | KEPUB | `-KEPUB` flag via kepubify, extension handling, tested on the Clara HD |
+| M6 | Polish and v1 release | Error messages and exit codes, README with per-device instructions (reMarkable: plain EPUB plus manual upload; Kobo: `-KEPUB`), goreleaser binaries |
+| M7 | *(post-v1)* Several URLs in one book | Repeatable `-URL`, a `-URLS` list file and an optional `-TITLE`; one chapter per article (design below) |
+
+Each milestone is a separate PR. After M1 the core job already works from end to end. M0–M6 make up v1.
+
+**Status:** M0–M6 are done. M6 added:
+- friendlier messages for common network failures (unknown host, refused connection, bad certificate, timeout) and hints on 403/404/429 responses;
+- exit code 130 for Ctrl-C;
+- a `-VERSION` flag, stamped by release builds and `build.cmd`;
+- the GoReleaser config, a tag-triggered release workflow that creates a draft release, and a test build of every release binary on each PR;
+- CI actions moved off the retiring Node 20 runtime;
+- the README rewritten for v1, with downloads, troubleshooting and releasing sections.
+
+### M7 design: several URLs in one book
+
+**Input.** Both ways can be used, and combined, in one command:
+- `-URL` may be repeated: `-URL https://a.example/one -URL https://b.example/two`. A custom `flag.Value` collects them.
+- `-URLS <file>` reads a text file with one URL per line. Surrounding spaces are trimmed, and blank lines and lines starting with `#` are ignored. `-URLS -` reads the list from standard input. A file avoids shell quoting problems (an unquoted `&` breaks a Windows `cmd` command) and keeps a reading list reusable.
+- Articles appear in the order given on the command line. A `-URLS` file's lines go where the flag appears.
+- Every URL is checked before anything is fetched, with the same rules as today. A bad entry is a usage error (exit `2`) naming it, with its line number for a list file. An empty list is an error too.
+- A URL given twice is included once. URLs that differ only in their `#fragment` count as the same.
+- At most 50 URLs per book. That's plenty for a reading list, and it catches mistakes such as passing the wrong file. Duplicates are removed before counting. Going over the limit is a usage error (exit `2`), reported before anything is fetched, with a message that says what to do:
+
+  ```
+  error: too many URLs: 63 given, but a book can hold at most 50. Split the list into several books.
+  ```
+
+**Title.** `-TITLE` is optional.
+- With one URL, the book uses the article's title as today; `-TITLE` overrides it.
+- With several URLs, the book is titled `-TITLE`, or "Reading list, 19 October 2026" (today's date) when it's missing.
+- For folder targets the filename is the slug of that title. With several URLs the title is known before fetching, so an existing file is caught before anything is downloaded.
+- One URL without `-TITLE` behaves exactly as v1.
+
+**Fetching and failures.**
+- Articles are fetched one at a time, in order, so sites aren't flooded and progress reads naturally. Each article's images download concurrently as today. Progress goes to stderr: `[2/5] https://…`.
+- A URL that can't be fetched, or has no readable content, is skipped with a warning, and the book is made from the rest. A summary follows: `made a book from 4 of 5 articles`. If every URL fails, the exit code is that of the first failure (`1` or `3`) and no file is written.
+- The local-network guard is decided per URL: an article may use local addresses only if its own URL is local.
+
+**Book structure.**
+- One chapter file per article (`chapter-01.xhtml`, …), each starting with its own header: title, byline and source link, as in v1. Each chapter keeps its article's language in `xml:lang`.
+- The table of contents lists the articles, with each article's section headings nested under it.
+- Images are stored once across the whole book, since files are already named by content hash.
+- The cover is typographic: the book title, "5 articles", and the first few article titles.
+- Metadata: `dc:title` is the book title. `dc:creator` lists the distinct authors, or "Various authors" when there are more than three. There's one `dc:source` per article. `dc:language` is the most common article language, and `dc:date` is the day the book was made. The identifier is derived from the list of URLs, so rebuilding the same list gives the same book identity.
+- `-KEPUB` works the same as for one article.
+
+**Code changes.**
+- `pipeline` splits into an article stage (fetch, extract, clean, images) and a book stage that takes the list of articles.
+- `build` generalises from one chapter to several, and the generated nav and NCX gain a level.
+- `cover` gets a reading-list layout.
+- `main` parses the new flags; the single-URL path stays as it is.
+
+**Tests.**
+- Flag and list-file parsing: comments, blank lines, stdin, duplicates, line numbers in errors.
+- The 50-URL limit: exactly 50 is accepted; 51 gives the error above with exit `2` and nothing fetched; 51 entries that are 50 after removing a duplicate are accepted.
+- End-to-end with three local pages, one of which returns 404.
+- Nested table of contents and per-chapter language.
+- epubcheck on multi-article EPUB and KEPUB output.
+
+---
+
+## 9. Future extensions
+
+- **JavaScript rendering**: a `-RENDER` flag using `chromedp` to drive a local Chrome/Chromium for SPA pages.
+- **More flags**:
+  - an `-AUTHOR` override (`-TITLE` arrives in M7);
+  - `-COVER` for a custom cover image;
+  - `-NOIMAGES`;
+  - `-TIMEOUT`.
+- **Follow pagination**: follow `rel="next"` links for multi-page articles.
+- **More device profiles**: a `-DEVICE` flag for colour e-ink (e.g. Kobo Libra Colour) or tablets, so colour images are kept. The `device.Profile` type already allows for this.
+- **Better contrast for e-ink**: optionally raise image contrast and gamma, since images often look washed out on e-ink.
+- **RSS/Atom digest**: turn the last N feed items into one EPUB.
+- **Web service**: an `http.Handler` around `pipeline.Run`. This would need SSRF protection: refuse private and loopback IP addresses, and re-check after each redirect.
+
+---
+
+## 10. Decisions made
+
+| Question | Decision |
+|---|---|
+| Language | Go |
+| Interface | CLI with `-URL` and `-SAVE` |
+| Existing output file | Never overwritten unless `-FORCE` is given (exit code 5) |
+| Missing output folders | Created automatically |
+| Target devices | reMarkable 2 and Kobo Clara HD (grayscale e-ink) |
+| Kobo-optimised output | `-KEPUB` flag in v1 (M5) |
+| Getting books onto the reMarkable | You upload the saved file by hand. Direct upload was dropped, so ReKoPub stays focused on making the file |
+| Several URLs in one book (M7) | Repeatable `-URL` plus a `-URLS` list file; optional `-TITLE`; at most 50 URLs, with a clear error above that; failed URLs are skipped with a warning |

README.md

@@ -1,3 +1,328 @@
 # ReKoPub
 
-Turn any web article into a clean EPUB for your e-ink reader. ReKoPub is a small command-line tool written in Go. You give it a URL. It downloads the page, keeps the article itself (dropping navigation, ads, pop-ups and comments) and writes an EPUB tuned for grayscale e-ink screens. It is built for the reMarkable 2 and the Kobo Clara HD.
+Turn any web article into a clean EPUB for your e-ink reader.
+
+`rekopub` is a small command-line tool written in Go. You give it a URL. It downloads the page, keeps the article itself (dropping navigation, ads, pop-ups and comments) and writes an EPUB tuned for grayscale e-ink screens. It is built for the **reMarkable 2** and the **Kobo Clara HD**.
+
+```bash
+rekopub -URL https://example.com/some-article -SAVE ~/Books/
+```
+
+---
+
+## Features
+
+- **Readable content only.** ReKoPub finds the main article text and removes menus, ads, cookie banners, share buttons and comments.
+- **Metadata included.** Title, author, publish date, site name and language are filled in automatically.
+- **Works offline.** Images are downloaded and embedded in the book.
+- **Tuned for e-ink.**
+  - Images are converted to grayscale and sized for the screen (no larger than 1404×1872).
+  - Diagrams and screenshots are kept crisp as 16-shade PNGs.
+  - The stylesheet uses no colour and no gray text, and leaves fonts and margins to your reader's settings.
+- **A cover every time.** Each book gets a 3:4 grayscale cover whose title is readable at library-thumbnail size.
+- **Kobo-optimised output.** `-KEPUB` writes a `.kepub.epub` file, which Kobo opens with its faster renderer and shows reading stats for.
+- **Safe saving.**
+  - An existing file is never overwritten unless you pass `-FORCE`.
+  - Missing folders are created.
+  - A failed run never leaves a half-written file behind.
+- **Single binary.** No runtime dependencies. It runs on Linux, macOS and Windows.
+
+---
+
+## Installation
+
+### Download (recommended)
+
+Download the archive for your computer from the [Releases](https://git.monoexp.com/bajdev/ReKoPub/releases) page:
+
+| Computer | File |
+|---|---|
+| Windows (most PCs) | `rekopub_<version>_windows_amd64.zip` |
+| Windows on Arm (e.g. Snapdragon laptops) | `rekopub_<version>_windows_arm64.zip` |
+| Mac with Apple silicon (M1 and later) | `rekopub_<version>_darwin_arm64.tar.gz` |
+| Mac with Intel | `rekopub_<version>_darwin_amd64.tar.gz` |
+| Linux | `rekopub_<version>_linux_amd64.tar.gz` (or `_arm64`) |
+
+Unpack it and put `rekopub` (`rekopub.exe` on Windows) somewhere convenient. There's nothing to install and nothing else to download. It's a command-line program, so run it from Command Prompt, PowerShell or Terminal rather than by double-clicking. `checksums.txt` on the release page lets you check the download.
+
+The binaries aren't code-signed, so the first run may show a warning:
+
+- **Windows:** SmartScreen may say "Windows protected your PC". Choose **More info → Run anyway**.
+- **macOS:** Gatekeeper may refuse to open it. Run `xattr -d com.apple.quarantine rekopub` once, in the folder where you unpacked it.
+
+### With Go (1.26 or newer)
+
+```bash
+go install git.monoexp.com/bajdev/rekopub/cmd/rekopub@latest
+```
+
+The binary is installed into `$(go env GOPATH)/bin`. Make sure that folder is on your `PATH`.
+
+### From source
+
+```bash
+git clone https://git.monoexp.com/bajdev/ReKoPub.git
+cd ReKoPub
+go build -o rekopub ./cmd/rekopub
+```
+
+### From source on Windows
+
+Install [Go](https://go.dev/dl/) 1.26 or newer, then run `build.cmd` from the repository folder:
+
+```bat
+git clone https://git.monoexp.com/bajdev/ReKoPub.git
+cd ReKoPub
+build.cmd
+```
+
+In PowerShell, run `.\build.cmd` instead. The script builds `rekopub.exe` in the repository folder, whichever folder you start it from. To update, run `git pull`, then `build.cmd` again.
+
+To try it on real articles, run `test-examples.cmd` (`.\test-examples.cmd` in PowerShell). It rebuilds `rekopub.exe`, converts a handful of sample articles into `test-examples\`, replacing books from earlier runs, and prints how many succeeded. Run `test-examples.cmd kepub` to make Kobo KEPUBs (`.kepub.epub`) instead. Edit the list of URLs in the script to test other pages.
+
+---
+
+## Usage
+
+```
+rekopub -URL <url> -SAVE <path> [-FORCE] [-KEPUB]
+rekopub -VERSION
+```
+
+| Flag | Required | Description |
+|---|---|---|
+| `-URL` | yes | The web page to convert. Must start with `http://` or `https://`. |
+| `-SAVE` | yes | Where to write the book: a file or a folder (see below). Missing folders are created. |
+| `-FORCE` | no | Overwrite the output file if it already exists. |
+| `-KEPUB` | no | Write a Kobo KEPUB (`.kepub.epub`) instead of a plain EPUB. |
+| `-VERSION` | no | Print the version and exit. Include it when reporting a problem. |
+
+Flag names are case-sensitive: `-URL` works, `-url` does not.
+
+### How `-SAVE` is interpreted
+
+| `-SAVE` value | Result |
+|---|---|
+| Ends in `/`, e.g. `~/Books/` | A folder (created if missing). The filename comes from the article title: `~/Books/<article-title>.epub` |
+| An existing folder, e.g. `~/Books` | Same as above |
+| Ends in `.epub`, e.g. `~/Books/a.epub` | Written to exactly that file |
+| Anything else, e.g. `~/Books/a` | Treated as a file name, and `.epub` is appended: `~/Books/a.epub` |
+
+Watch the trailing slash when the folder doesn't exist yet. `-SAVE ~/Books/new/` creates a folder called `new`, while `-SAVE ~/Books/new` writes a file called `new.epub`.
+
+### Overwriting
+
+ReKoPub never replaces an existing file unless you ask it to:
+
+```bash
+$ rekopub -URL https://example.com/a -SAVE ~/Books/a.epub
+error: /home/me/Books/a.epub already exists (use -FORCE to overwrite)
+
+$ rekopub -URL https://example.com/a -SAVE ~/Books/a.epub -FORCE
+/home/me/Books/a.epub
+```
+
+When the output path is known up front, this check happens before anything is downloaded. With `-FORCE`, the old file stays intact until the new one has been fully written.
+
+### KEPUB output (`-KEPUB`)
+
+Kobo only uses its faster renderer for files ending in `.kepub.epub`, so `-KEPUB` adjusts the extension for you:
+
+| `-SAVE` value | Saved as |
+|---|---|
+| `~/Books/` (folder) | `~/Books/<article-title>.kepub.epub` |
+| `~/Books/a.kepub.epub` | `~/Books/a.kepub.epub` |
+| `~/Books/a.epub` | `~/Books/a.kepub.epub` (with a note on stderr) |
+| `~/Books/a` | `~/Books/a.kepub.epub` |
+
+Passing a `.kepub.epub` path **without** `-KEPUB` is an error, because a plain EPUB with that name would display badly on a Kobo.
+
+### Output
+
+- On success, the absolute path of the saved file is printed to **stdout**, so you can use it in scripts.
+- Progress messages and warnings, such as an image that failed to download, go to **stderr**.
+
+---
+
+## Device guides
+
+### Kobo Clara HD
+
+Use `-KEPUB` for the best experience: faster page turns, time-left statistics and more reliable highlights.
+
+1. Connect the Kobo by USB and tap **Connect** on the device.
+2. Save straight to it:
+   ```bash
+   # Linux
+   rekopub -URL https://example.com/article -SAVE /media/$USER/KOBOeReader/Articles/ -KEPUB
+   # macOS
+   rekopub -URL https://example.com/article -SAVE /Volumes/KOBOeReader/Articles/ -KEPUB
+   # Windows (the drive letter may differ)
+   rekopub.exe -URL https://example.com/article -SAVE E:\Articles\ -KEPUB
+   ```
+3. Eject the Kobo. The article appears in your library.
+
+A plain EPUB (without `-KEPUB`) also works on the Kobo, but it opens in Kobo's older renderer.
+
+### reMarkable 2
+
+Use plain EPUB, without `-KEPUB`.
+
+1. Save the book locally:
+   ```bash
+   rekopub -URL https://example.com/article -SAVE ~/reMarkable/
+   ```
+2. Upload it with any of these:
+   - **reMarkable desktop or mobile app:** drag the file in.
+   - **USB web interface:** connect by USB, turn on *Settings → Storage → USB web interface*, open <http://10.11.99.1> and upload the file.
+
+---
+
+## Exit codes
+
+| Code | Meaning |
+|---|---|
+| `0` | Success |
+| `1` | The page couldn't be fetched (network error, non-200 response, not an HTML page) |
+| `2` | Invalid arguments |
+| `3` | No readable article content was found on the page |
+| `4` | The book couldn't be built or written (including folder creation) |
+| `5` | The output file already exists and `-FORCE` was not given |
+| `130` | Stopped with Ctrl-C |
+
+Example in a script:
+
+```bash
+while read -r url; do
+  rekopub -URL "$url" -SAVE ~/Books/ || echo "failed: $url" >&2
+done < urls.txt
+```
+
+---
+
+## Troubleshooting
+
+| Message | What it means |
+|---|---|
+| `can't find the server …` | The address is mistyped, or the computer is offline. |
+| `… refused the connection` / `… took too long to respond` | The site is down or very slow. Try again later. |
+| `… security certificate isn't valid` | The connection can't be trusted, so ReKoPub won't use it. Check the address; on a work or school network, a proxy may be intercepting secure connections. |
+| `… returned 403 Forbidden; the site may need a login or may block automated downloads` | The site refused the request. Some sites block anything that isn't a full web browser. |
+| `… returned 404 Not Found; check the address` | The page doesn't exist at that address. |
+| `no readable article content found` | The page has no article text ReKoPub can find. Usually the page builds its content with JavaScript, or it's a paywall or cookie wall. |
+| `warning: image … skipped: …` | One image couldn't be downloaded or read. The book is still made, with the image's description in its place. |
+| `warning: cover image … skipped, using a plain cover` | The page's preview image couldn't be used, so the book gets a typographic cover instead. |
+| `… is a local or private address …` | The page tried to send ReKoPub to your own computer or network, through a redirect or an image. That's refused for pages on the internet (see [Privacy and security](#privacy-and-security)). To convert a page on your own network, give its local address as `-URL`. |
+
+To report a problem with a particular page, include the output of `rekopub -VERSION`, the full command, everything it printed, and a saved copy of the page:
+
+```bash
+curl -L -A "Mozilla/5.0" -o page.html "https://example.com/the-article"
+```
+
+(On Windows, use `curl.exe`, which comes with Windows 10 and 11.)
+
+---
+
+## Privacy and security
+
+- **Only web pages are read.** ReKoPub downloads the page given as `-URL`, its images and its preview image, over `http://` or `https://` only. It never reads files on your computer: `file:` links and local paths in a page are ignored, and the only file it writes is the book.
+- **Pages on the internet can't reach your network.** A web page chooses where its images come from, and can redirect. So when `-URL` is on the internet, ReKoPub refuses to connect to your own computer (`localhost`), your home or office network (such as `192.168.x.x` or `10.x.x.x`) and similar private addresses. If `-URL` itself is a local address, such as a wiki on your network, its local images are allowed. With a proxy set in `HTTP_PROXY` or `HTTPS_PROXY`, the proxy makes the connections, so ReKoPub can't check them.
+- **What sites see.** Requests look like a desktop browser and include "ReKoPub" in the user agent. Image requests send the article's address as the referrer, as browsers do. ReKoPub sends no cookies, never logs in and runs no JavaScript.
+- **Your files are safe.** Existing files are only replaced with `-FORCE`. Filenames made from article titles contain only letters, digits and dashes, so a title can't place the book outside the folder you chose.
+
+To report a security problem, see [`SECURITY.md`](SECURITY.md).
+
+---
+
+## Limitations
+
+- **Pages that need JavaScript.** Single-page apps that only render content in the browser aren't supported in v1. ReKoPub detects these and exits with code `3`.
+- **Paywalls and logins.** Only content the server sends without logging in can be converted.
+- **One URL per book.** Combining several articles into one book is a possible future feature.
+- **Colour-coded charts.** These can be harder to read after grayscale conversion. The image's caption and alt text are kept.
+- **AVIF and JPEG XL images** aren't supported. When a page offers the image in another format too, that one is used; otherwise the image's description is shown instead.
+- **Tables that merge cells** (`colspan`/`rowspan`) may come out with uneven rows, because the extraction library drops those attributes.
+
+ReKoPub is meant for saving articles for your own offline reading. Every book records its source URL on the first page and in its metadata.
+
+---
+
+## Development
+
+```bash
+go test ./...                 # unit and end-to-end tests
+go test -race ./...           # with the race detector
+go vet ./... && staticcheck ./...
+```
+
+The tests also check the generated books with [epubcheck](https://github.com/w3c/epubcheck), which needs Java. Set `EPUBCHECK_JAR` to the path of `epubcheck.jar` to turn those checks on; without it they're skipped. CI always runs them, on Linux, macOS and Windows.
+
+On Windows, `build.cmd` builds `rekopub.exe` and `test-examples.cmd` converts a set of real articles into `test-examples\` for checking on the devices (see [From source on Windows](#from-source-on-windows)).
+
+### Project layout
+
+```
+cmd/rekopub/        CLI entry point (flags, exit codes, -VERSION)
+internal/pipeline/  Runs the stages below in order
+internal/fetch/     HTTP download, local-network guard, character-set decoding, friendly network errors
+internal/extract/   Article and metadata extraction (go-trafilatura, page metadata)
+internal/clean/     Pre-extraction fixes, sanitising, XHTML conversion
+internal/assets/    Image download and e-ink processing
+internal/cover/     Cover drawing
+internal/device/    e-ink device profile (image and cover sizes, gray levels)
+internal/build/     EPUB packaging (go-epub), stylesheet, contents and metadata
+internal/kepub/     KEPUB conversion (kepubify)
+internal/save/      Output path rules, folder creation, -FORCE, atomic writes
+internal/epubtest/  Test helpers for checking generated books
+testdata/           Saved HTML pages and images used by the tests
+```
+
+### Releasing
+
+Releases are built by [GoReleaser](https://goreleaser.com) from `.goreleaser.yaml`: self-contained binaries for Windows, macOS and Linux on amd64 and arm64, plus checksums. Every pull request runs a test build of all of them.
+
+To make a release, tag the commit on `main` and push the tag:
+
+```bash
+git tag v1.0.0
+git push origin v1.0.0
+```
+
+The Release workflow then builds everything with the newest Go release (so the binaries include its security fixes), runs [govulncheck](https://go.dev/doc/security/vuln/), and creates a **draft** release on GitHub. Check the files and the notes, then press **Publish release**. CI also runs govulncheck weekly, so a newly published vulnerability in Go or a dependency shows up even when nothing has changed here; when one does, make a new release.
+
+See [`PLAN.md`](PLAN.md) for the full design.
+
+---
+
+## Roadmap
+
+| Milestone | Scope | Status |
+|---|---|---|
+| M0 | Project setup, flag handling, saving rules (`-SAVE`, `-FORCE`, folders) | Done |
+| M1 | Basic working version: text-only EPUB | Done |
+| M2 | Clean XHTML and the e-ink stylesheet | Done |
+| M3 | Images processed for e-ink | Done |
+| M4 | Metadata, cover and table of contents | Done |
+| M5 | `-KEPUB` for Kobo | Done |
+| M6 | Polish and **v1 release** | Done |
+| M7 | Several URLs in one book: repeat `-URL`, or list them in a file with `-URLS`; optional `-TITLE` | Planned |
+
+Possible later features: JavaScript rendering, author and cover overrides, and profiles for colour e-ink devices.
+
+---
+
+## Acknowledgements
+
+ReKoPub builds on these open-source projects:
+
+- [go-trafilatura](https://github.com/markusmobius/go-trafilatura): content extraction
+- [go-epub](https://github.com/go-shiori/go-epub): EPUB packaging
+- [kepubify](https://github.com/pgaskin/kepubify): KEPUB conversion
+- [bluemonday](https://github.com/microcosm-cc/bluemonday): HTML sanitising
+- [oksvg](https://github.com/srwiley/oksvg): SVG rendering
+- The Go [x/image](https://pkg.go.dev/golang.org/x/image) and [x/net](https://pkg.go.dev/golang.org/x/net) packages: WebP decoding, image scaling, the Go fonts used on covers, and HTML parsing
+
+## License
+
+ReKoPub is released under the [MIT License](LICENSE).

SECURITY.md

@@ -0,0 +1,22 @@
+# Security policy
+
+## Supported versions
+
+Security fixes go into the latest release only. Please update to it before reporting a problem.
+
+## Reporting a vulnerability
+
+Please report security problems privately, not in a public issue: on GitHub, open the repository's **Security** tab and choose **Report a vulnerability**. Include the output of `rekopub -VERSION`, the command you ran, and what happened.
+
+You should get a reply within a week. Once a fix is released, the report is published as a security advisory, with credit to you unless you prefer otherwise.
+
+## Scope
+
+ReKoPub downloads web pages chosen by its user, which are untrusted input. Problems in scope include:
+
+- a page making ReKoPub read or write files other than the book it was asked to save;
+- a page making ReKoPub connect to the user's own computer or network when `-URL` is on the internet;
+- a page producing a book that runs scripts or loads remote content in a reader;
+- crashes or runaway memory use caused by a page or image.
+
+What ReKoPub does to stay safe is described under [Privacy and security](README.md#privacy-and-security) in the README.

build.cmd

@@ -0,0 +1,42 @@
+@echo off
+rem Builds rekopub.exe in the repository root.
+rem
+rem Run it from Command Prompt or PowerShell:
+rem     build.cmd          (Command Prompt)
+rem     .\build.cmd        (PowerShell)
+rem
+rem Needs Go 1.26 or newer: https://go.dev/dl/
+setlocal
+
+rem Work from the folder this script is in, wherever it was started from.
+cd /d "%~dp0"
+
+where go >nul 2>nul
+if errorlevel 1 (
+    echo error: Go was not found. Install Go 1.26 or newer from https://go.dev/dl/
+    echo and open a new terminal so the installer's PATH change takes effect.
+    exit /b 1
+)
+
+rem A pure-Go Windows build for this machine's processor. -trimpath and
+rem -s -w leave out local paths and debug symbols, making the file smaller.
+set CGO_ENABLED=0
+set GOOS=windows
+
+rem Stamp the version from git (e.g. v1.0.0, or v1.0.0-3-gabc1234 after
+rem it) so "rekopub.exe -VERSION" says what was built.
+set VERSION=dev
+for /f "delims=" %%v in ('git describe --tags --always --dirty 2^>nul') do set VERSION=%%v
+
+echo Building rekopub.exe %VERSION%...
+go build -trimpath -ldflags="-s -w -X main.version=%VERSION%" -o rekopub.exe .\cmd\rekopub
+if errorlevel 1 (
+    echo error: the build failed; see the messages above.
+    exit /b 1
+)
+
+echo Built %CD%\rekopub.exe
+echo.
+echo Example:
+echo     rekopub.exe -URL https://example.com/article -SAVE %%USERPROFILE%%\Books\
+endlocal

cmd/rekopub/main.go

@@ -0,0 +1,182 @@
+// Command rekopub converts a web article into an EPUB for e-ink readers.
+//
+//	rekopub -URL <url> -SAVE <path> [-FORCE] [-KEPUB]
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"os/signal"
+
+	"git.monoexp.com/bajdev/rekopub/internal/pipeline"
+	"git.monoexp.com/bajdev/rekopub/internal/save"
+)
+
+// Exit codes, as documented in the README.
+const (
+	exitOK        = 0
+	exitFetch     = 1 // page couldn't be fetched
+	exitUsage     = 2 // invalid arguments
+	exitNoContent = 3 // no readable content found
+	exitBuild     = 4 // book couldn't be built or written
+	exitExists    = 5 // output exists and -FORCE wasn't given
+
+	// exitInterrupted is the conventional code for a program stopped
+	// with Ctrl-C (128 + SIGINT).
+	exitInterrupted = 130
+)
+
+const usage = `Usage: rekopub -URL <url> -SAVE <path> [-FORCE] [-KEPUB]
+
+Convert a web article into an EPUB for e-ink readers.
+
+  -URL string
+        Web page to convert (required). Must be http:// or https://.
+  -SAVE string
+        Where to write the book (required). Missing folders are created.
+          - Ends in "/"          → a folder: <folder>/<article-title>.epub
+          - An existing folder   → same as above
+          - Ends in .epub        → written to exactly that file
+          - Anything else        → ".epub" is appended
+        With -KEPUB the extension is ".kepub.epub" instead.
+  -FORCE
+        Overwrite the output file if it already exists.
+  -KEPUB
+        Write a Kobo KEPUB (.kepub.epub) instead of a plain EPUB.
+  -VERSION
+        Print the version and exit.
+`
+
+// convertFunc turns a URL into a book. It is pipeline.Run in production and
+// a fake in tests.
+type convertFunc func(ctx context.Context, u *url.URL, opts pipeline.Options) (*pipeline.Book, error)
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	code := run(ctx, os.Args[1:], os.Stdout, os.Stderr, pipeline.Run)
+	stop()
+	os.Exit(code)
+}
+
+func run(ctx context.Context, args []string, stdout, stderr io.Writer, convert convertFunc) int {
+	fs := flag.NewFlagSet("rekopub", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprint(stderr, usage) }
+	urlArg := fs.String("URL", "", "web page to convert (required)")
+	saveArg := fs.String("SAVE", "", "file or folder to write the book to (required)")
+	force := fs.Bool("FORCE", false, "overwrite the output file if it already exists")
+	kepub := fs.Bool("KEPUB", false, "write a Kobo KEPUB (.kepub.epub) instead of a plain EPUB")
+	showVersion := fs.Bool("VERSION", false, "print the version and exit")
+
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return exitOK
+		}
+		return exitUsage // flag has already printed the error and usage
+	}
+
+	fail := func(code int, format string, a ...any) int {
+		fmt.Fprintf(stderr, "error: "+format+"\n", a...)
+		return code
+	}
+	usageError := func(format string, a ...any) int {
+		fail(exitUsage, format, a...)
+		fmt.Fprint(stderr, "\n"+usage)
+		return exitUsage
+	}
+
+	if *showVersion {
+		fmt.Fprintln(stdout, "rekopub", versionString())
+		return exitOK
+	}
+	if fs.NArg() > 0 {
+		return usageError("unexpected argument %q", fs.Arg(0))
+	}
+	if *urlArg == "" {
+		return usageError("-URL is required")
+	}
+	if *saveArg == "" {
+		return usageError("-SAVE is required")
+	}
+	pageURL, err := parseURL(*urlArg)
+	if err != nil {
+		return usageError("-URL %q: %v", *urlArg, err)
+	}
+	target, err := save.Resolve(*saveArg, *kepub)
+	if err != nil {
+		return usageError("-SAVE: %v", err)
+	}
+	if target.Note != "" {
+		fmt.Fprintf(stderr, "note: %s\n", target.Note)
+	}
+
+	// Fail before downloading anything when the filename is already known.
+	if target.Known() && !*force {
+		if err := save.CheckExists(target.Path("")); err != nil {
+			return saveFailure(fail, err)
+		}
+	}
+
+	opts := pipeline.Options{
+		Warn:  func(msg string) { fmt.Fprintf(stderr, "warning: %s\n", msg) },
+		KEPUB: *kepub,
+	}
+	if !target.Known() && !*force {
+		// For folder targets the filename comes from the title; check it
+		// before downloading images.
+		opts.BeforeImages = func(title string) error { return save.CheckExists(target.Path(title)) }
+	}
+	book, err := convert(ctx, pageURL, opts)
+	switch {
+	case errors.Is(err, context.Canceled):
+		return fail(exitInterrupted, "interrupted")
+	case errors.Is(err, save.ErrExists):
+		return saveFailure(fail, err)
+	case errors.Is(err, pipeline.ErrFetch):
+		return fail(exitFetch, "%v", err)
+	case errors.Is(err, pipeline.ErrNoContent):
+		return fail(exitNoContent, "%v", err)
+	case err != nil:
+		return fail(exitBuild, "%v", err)
+	}
+
+	path := target.Path(book.Title)
+	if !*force {
+		if err := save.CheckExists(path); err != nil {
+			return saveFailure(fail, err)
+		}
+	}
+	if err := save.Write(path, book.Content, *force); err != nil {
+		return saveFailure(fail, err)
+	}
+	fmt.Fprintln(stdout, path)
+	return exitOK
+}
+
+// saveFailure reports an error from the save package with its exit code.
+func saveFailure(fail func(int, string, ...any) int, err error) int {
+	if errors.Is(err, save.ErrExists) {
+		return fail(exitExists, "%v (use -FORCE to overwrite)", err)
+	}
+	return fail(exitBuild, "saving: %v", err)
+}
+
+// parseURL accepts absolute http and https URLs only.
+func parseURL(raw string) (*url.URL, error) {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return nil, errors.New("not a valid URL")
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil, errors.New("must start with http:// or https://")
+	}
+	if u.Host == "" {
+		return nil, errors.New("missing host name")
+	}
+	return u, nil
+}

cmd/rekopub/main_test.go

@@ -0,0 +1,798 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"image"
+	"image/jpeg"
+	_ "image/png"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"git.monoexp.com/bajdev/rekopub/internal/epubtest"
+	"git.monoexp.com/bajdev/rekopub/internal/pipeline"
+)
+
+// fakeConverter returns a fixed book and counts how often it was called, so
+// tests can check that nothing is fetched when arguments are rejected.
+type fakeConverter struct {
+	title string
+	err   error
+	calls int
+	kepub bool // the KEPUB option of the last call
+}
+
+func (f *fakeConverter) convert(ctx context.Context, u *url.URL, opts pipeline.Options) (*pipeline.Book, error) {
+	f.calls++
+	f.kepub = opts.KEPUB
+	if f.err != nil {
+		return nil, f.err
+	}
+	if opts.BeforeImages != nil {
+		if err := opts.BeforeImages(f.title); err != nil {
+			return nil, err
+		}
+	}
+	return &pipeline.Book{Title: f.title, Content: strings.NewReader("epub:" + f.title)}, nil
+}
+
+type result struct {
+	code           int
+	stdout, stderr string
+}
+
+func runCLI(t *testing.T, conv *fakeConverter, args ...string) result {
+	t.Helper()
+	var stdout, stderr bytes.Buffer
+	code := run(context.Background(), args, &stdout, &stderr, conv.convert)
+	return result{code, stdout.String(), stderr.String()}
+}
+
+const testURL = "https://example.com/article"
+
+func TestUsageErrors(t *testing.T) {
+	dir := t.TempDir()
+	out := filepath.Join(dir, "a.epub")
+	tests := []struct {
+		name string
+		args []string
+		want string // expected in stderr
+	}{
+		{"no arguments", nil, "-URL is required"},
+		{"missing -SAVE", []string{"-URL", testURL}, "-SAVE is required"},
+		{"missing -URL", []string{"-SAVE", out}, "-URL is required"},
+		{"lowercase flag", []string{"-url", testURL, "-SAVE", out}, "flag provided but not defined: -url"},
+		{"extra argument", []string{"-URL", testURL, "-SAVE", out, "extra"}, `unexpected argument "extra"`},
+		{"ftp URL", []string{"-URL", "ftp://example.com/a", "-SAVE", out}, "must start with http:// or https://"},
+		{"no scheme", []string{"-URL", "example.com/a", "-SAVE", out}, "must start with http:// or https://"},
+		{"file URL", []string{"-URL", "file:///etc/passwd", "-SAVE", out}, "must start with http:// or https://"},
+		{"local path", []string{"-URL", "/etc/passwd", "-SAVE", out}, "must start with http:// or https://"},
+		{"Windows path", []string{"-URL", `C:\Users\me\notes.html`, "-SAVE", out}, "-URL"},
+		{"UNC path", []string{"-URL", `\\server\share\page.html`, "-SAVE", out}, "must start with http:// or https://"},
+		{"no host", []string{"-URL", "https://", "-SAVE", out}, "missing host name"},
+		{"invalid URL", []string{"-URL", "https://exa mple.com/%zz", "-SAVE", out}, "not a valid URL"},
+		{"kepub name without -KEPUB", []string{"-URL", testURL, "-SAVE", filepath.Join(dir, "a.kepub.epub")}, "needs -KEPUB"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			conv := &fakeConverter{title: "T"}
+			r := runCLI(t, conv, tt.args...)
+			if r.code != exitUsage {
+				t.Errorf("exit code = %d, want %d", r.code, exitUsage)
+			}
+			if !strings.Contains(r.stderr, tt.want) {
+				t.Errorf("stderr = %q, want it to contain %q", r.stderr, tt.want)
+			}
+			if !strings.Contains(r.stderr, "Usage: rekopub") {
+				t.Errorf("stderr does not show usage: %q", r.stderr)
+			}
+			if conv.calls != 0 {
+				t.Errorf("converter called %d times, want 0", conv.calls)
+			}
+			if r.stdout != "" {
+				t.Errorf("stdout = %q, want empty", r.stdout)
+			}
+		})
+	}
+}
+
+func TestVersion(t *testing.T) {
+	conv := &fakeConverter{}
+	r := runCLI(t, conv, "-VERSION")
+	if r.code != exitOK || !strings.HasPrefix(r.stdout, "rekopub ") || len(strings.TrimSpace(r.stdout)) <= len("rekopub") {
+		t.Errorf("exit code = %d, stdout = %q", r.code, r.stdout)
+	}
+	if conv.calls != 0 {
+		t.Errorf("-VERSION ran a conversion")
+	}
+
+	defer func(v string) { version = v }(version)
+	version = "v1.2.3"
+	if r := runCLI(t, conv, "-VERSION"); r.stdout != "rekopub v1.2.3\n" {
+		t.Errorf("stdout = %q, want the version set at build time", r.stdout)
+	}
+}
+
+func TestInterrupted(t *testing.T) {
+	dir := t.TempDir()
+	conv := &fakeConverter{err: fmt.Errorf("%w: %w", pipeline.ErrFetch, context.Canceled)}
+	r := runCLI(t, conv, "-URL", testURL, "-SAVE", dir)
+	if r.code != exitInterrupted || strings.TrimSpace(r.stderr) != "error: interrupted" {
+		t.Errorf("exit code = %d, stderr = %q; want %d and \"error: interrupted\"", r.code, r.stderr, exitInterrupted)
+	}
+	assertEmptyDir(t, dir)
+}
+
+func TestHelp(t *testing.T) {
+	r := runCLI(t, &fakeConverter{}, "-h")
+	if r.code != exitOK {
+		t.Errorf("exit code = %d, want %d", r.code, exitOK)
+	}
+	if !strings.Contains(r.stderr, "Usage: rekopub") {
+		t.Errorf("stderr = %q, want usage", r.stderr)
+	}
+}
+
+func TestSaveToFile(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "a.epub")
+	r := runCLI(t, &fakeConverter{title: "Title"}, "-URL", testURL, "-SAVE", path)
+	if r.code != exitOK {
+		t.Fatalf("exit code = %d, stderr = %q", r.code, r.stderr)
+	}
+	if strings.TrimSpace(r.stdout) != path {
+		t.Errorf("stdout = %q, want %q", r.stdout, path)
+	}
+	assertFile(t, path, "epub:Title")
+}
+
+func TestSaveToFolderCreatesIt(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "Books", "Articles")
+	r := runCLI(t, &fakeConverter{title: "My Great Article"}, "-URL", testURL, "-SAVE", dir+"/")
+	if r.code != exitOK {
+		t.Fatalf("exit code = %d, stderr = %q", r.code, r.stderr)
+	}
+	want := filepath.Join(dir, "my-great-article.epub")
+	if strings.TrimSpace(r.stdout) != want {
+		t.Errorf("stdout = %q, want %q", r.stdout, want)
+	}
+	assertFile(t, want, "epub:My Great Article")
+}
+
+func TestExistingFileStopsBeforeFetching(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "a.epub")
+	writeFile(t, path, "old")
+	conv := &fakeConverter{title: "Title"}
+
+	r := runCLI(t, conv, "-URL", testURL, "-SAVE", path)
+	if r.code != exitExists {
+		t.Errorf("exit code = %d, want %d", r.code, exitExists)
+	}
+	if !strings.Contains(r.stderr, "already exists (use -FORCE to overwrite)") {
+		t.Errorf("stderr = %q", r.stderr)
+	}
+	if conv.calls != 0 {
+		t.Errorf("converter called %d times, want 0", conv.calls)
+	}
+	assertFile(t, path, "old")
+}
+
+func TestExistingFileInFolder(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "title.epub")
+	writeFile(t, path, "old")
+
+	r := runCLI(t, &fakeConverter{title: "Title"}, "-URL", testURL, "-SAVE", dir)
+	if r.code != exitExists {
+		t.Errorf("exit code = %d, want %d; stderr = %q", r.code, exitExists, r.stderr)
+	}
+	assertFile(t, path, "old")
+}
+
+func TestForceOverwrites(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "a.epub")
+	writeFile(t, path, "old")
+
+	r := runCLI(t, &fakeConverter{title: "Title"}, "-URL", testURL, "-SAVE", path, "-FORCE")
+	if r.code != exitOK {
+		t.Fatalf("exit code = %d, stderr = %q", r.code, r.stderr)
+	}
+	assertFile(t, path, "epub:Title")
+
+	// -FORCE also applies to folder targets.
+	writeFile(t, filepath.Join(dir, "title.epub"), "old")
+	r = runCLI(t, &fakeConverter{title: "Title"}, "-URL", testURL, "-SAVE", dir, "-FORCE")
+	if r.code != exitOK {
+		t.Fatalf("folder: exit code = %d, stderr = %q", r.code, r.stderr)
+	}
+	assertFile(t, filepath.Join(dir, "title.epub"), "epub:Title")
+}
+
+func TestConversionFailureWritesNothing(t *testing.T) {
+	dir := t.TempDir()
+	conv := &fakeConverter{err: errors.New("boom")}
+
+	r := runCLI(t, conv, "-URL", testURL, "-SAVE", filepath.Join(dir, "a.epub"))
+	if r.code != exitBuild {
+		t.Errorf("exit code = %d, want %d", r.code, exitBuild)
+	}
+	if !strings.Contains(r.stderr, "error: boom") {
+		t.Errorf("stderr = %q", r.stderr)
+	}
+	assertEmptyDir(t, dir)
+}
+
+func TestSavePathThroughFile(t *testing.T) {
+	dir := t.TempDir()
+	writeFile(t, filepath.Join(dir, "Books"), "not a folder")
+
+	r := runCLI(t, &fakeConverter{title: "Title"}, "-URL", testURL, "-SAVE", filepath.Join(dir, "Books", "a.epub"))
+	if r.code != exitBuild {
+		t.Errorf("exit code = %d, want %d; stderr = %q", r.code, exitBuild, r.stderr)
+	}
+}
+
+func TestKEPUBExtensionNote(t *testing.T) {
+	dir := t.TempDir()
+	writeFile(t, filepath.Join(dir, "a.kepub.epub"), "old")
+
+	// The overwrite check uses the adjusted name, before anything is fetched.
+	conv := &fakeConverter{title: "Title"}
+	r := runCLI(t, conv, "-URL", testURL, "-SAVE", filepath.Join(dir, "a.epub"), "-KEPUB")
+	if !strings.Contains(r.stderr, "note: saving as a.kepub.epub") {
+		t.Errorf("stderr = %q, want a note about the new name", r.stderr)
+	}
+	if r.code != exitExists || conv.calls != 0 {
+		t.Errorf("exit code = %d, converter calls = %d; want %d and 0", r.code, conv.calls, exitExists)
+	}
+}
+
+func TestKEPUBOption(t *testing.T) {
+	dir := t.TempDir()
+	for _, kepub := range []bool{false, true} {
+		conv := &fakeConverter{title: "Title"}
+		args := []string{"-URL", testURL, "-SAVE", dir, "-FORCE"}
+		want := filepath.Join(dir, "title.epub")
+		if kepub {
+			args = append(args, "-KEPUB")
+			want = filepath.Join(dir, "title.kepub.epub")
+		}
+		r := runCLI(t, conv, args...)
+		if r.code != exitOK {
+			t.Fatalf("kepub=%v: exit code = %d, stderr = %q", kepub, r.code, r.stderr)
+		}
+		if conv.kepub != kepub {
+			t.Errorf("kepub=%v: pipeline got KEPUB=%v", kepub, conv.kepub)
+		}
+		if strings.TrimSpace(r.stdout) != want {
+			t.Errorf("kepub=%v: saved to %q, want %q", kepub, r.stdout, want)
+		}
+	}
+}
+
+// assertCover checks the book has a 1404x1872 grayscale cover, listed
+// first in the spine. With photo, the cover's upper half must show the
+// photo (mid-grays) rather than the plain cover's black band and white
+// space.
+func assertCover(t *testing.T, book *epubtest.Book, photo bool) {
+	t.Helper()
+	opf := book.File(t, ".opf")
+	if !strings.Contains(opf, `href="images/cover.jpg" media-type="image/jpeg" properties="cover-image"`) {
+		t.Fatalf("OPF has no cover image:\n%s", opf)
+	}
+	if strings.Index(opf, `<itemref idref="cover.xhtml">`) > strings.Index(opf, `<itemref idref="chapter.xhtml">`) {
+		t.Errorf("cover isn't first in the spine:\n%s", opf)
+	}
+	img, err := jpeg.Decode(strings.NewReader(book.Files["EPUB/images/cover.jpg"]))
+	if err != nil {
+		t.Fatal(err)
+	}
+	gray, ok := img.(*image.Gray)
+	if !ok || gray.Bounds().Dx() != 1404 || gray.Bounds().Dy() != 1872 {
+		t.Fatalf("cover is %T %v, want 1404x1872 grayscale", img, img.Bounds())
+	}
+	mid := 0
+	for y := 300; y < 900; y += 10 {
+		for x := 0; x < 1404; x += 10 {
+			if v := gray.GrayAt(x, y).Y; v > 40 && v < 215 {
+				mid++
+			}
+		}
+	}
+	if hasPhoto := mid > 2000; hasPhoto != photo {
+		t.Errorf("cover photo = %v (%d mid-gray samples), want %v", hasPhoto, mid, photo)
+	}
+}
+
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func assertFile(t *testing.T, path, want string) {
+	t.Helper()
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != want {
+		t.Errorf("%s contains %q, want %q", path, got, want)
+	}
+}
+
+func assertEmptyDir(t *testing.T, dir string) {
+	t.Helper()
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 0 {
+		t.Errorf("%s is not empty: %v", dir, entries)
+	}
+}
+
+// TestEndToEnd runs the real pipeline against a local web server.
+func TestEndToEnd(t *testing.T) {
+	blog, err := os.ReadFile("../../testdata/pages/blog.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+	docs, err := os.ReadFile("../../testdata/pages/docs.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+	essay, err := os.ReadFile("../../testdata/pages/photo-essay.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+	lightbox, err := os.ReadFile("../../testdata/pages/lightbox.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+	wiki, err := os.ReadFile("../../testdata/pages/wiki.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+	images := map[string]string{ // served path → file in testdata/images
+		"/img/press.jpg":   "press.jpg",
+		"/img/diagram.png": "diagram.png",
+		"/img/type.webp":   "gradient.webp",
+		// lightbox.html; srcset should select the 1536w file.
+		"/wp-content/uploads/2026/10/upgrade-dialog-1536x864.png": "diagram.png",
+		"/wp-content/uploads/2026/10/settings-640x480.jpg":        "press.jpg",
+		"/wp-content/uploads/2026/10/hero.jpg":                    "press.jpg", // og:image, for the cover
+		// wiki.html; served whatever the query string, as Wikipedia does.
+		"/upload/a/a6/Walking_Robots_logo.png":                          "diagram.png",
+		"/thumb/9/99/Painted_miniature.jpg/500px-Painted_miniature.jpg": "press.jpg",
+		"/media/record-sheet":                                           "gradient.webp",
+	}
+	var imageRequests atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.Path, "/img/") || strings.HasPrefix(r.URL.Path, "/wp-content/") ||
+			strings.HasPrefix(r.URL.Path, "/upload/") || strings.HasPrefix(r.URL.Path, "/thumb/") || strings.HasPrefix(r.URL.Path, "/media/") {
+			imageRequests.Add(1)
+			name, ok := images[r.URL.Path]
+			if !ok {
+				http.NotFound(w, r)
+				return
+			}
+			http.ServeFile(w, r, filepath.Join("../../testdata/images", name))
+			return
+		}
+		switch r.URL.Path {
+		case "/wiki/Walking_Robots":
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			w.Write(wiki)
+		case "/news/2026/10/kde-turns-30/":
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			w.Write(lightbox)
+		case "/essays/print-shop":
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			w.Write(essay)
+		case "/posts/eink":
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			w.Write(blog)
+		case "/docs/config":
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			w.Write(docs)
+		case "/app":
+			w.Header().Set("Content-Type", "text/html")
+			w.Write([]byte(`<html><body><div id="root"></div><script src="/app.js"></script></body></html>`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	realRun := func(args ...string) result {
+		var stdout, stderr bytes.Buffer
+		code := run(context.Background(), args, &stdout, &stderr, pipeline.Run)
+		return result{code, stdout.String(), stderr.String()}
+	}
+
+	t.Run("article", func(t *testing.T) {
+		dir := t.TempDir()
+		r := realRun("-URL", srv.URL+"/posts/eink", "-SAVE", dir)
+		if r.code != exitOK {
+			t.Fatalf("exit code = %d, stderr = %q", r.code, r.stderr)
+		}
+		path := filepath.Join(dir, "why-e-ink-is-easier-on-the-eyes.epub")
+		if strings.TrimSpace(r.stdout) != path {
+			t.Errorf("stdout = %q, want %q", r.stdout, path)
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		book := epubtest.Open(t, data)
+		book.AssertWellFormed(t)
+		chapter := book.File(t, "chapter.xhtml")
+		for _, want := range []string{
+			"<h1>Why E-Ink Is Easier on the Eyes</h1>",
+			"Ada Lovelace · The Reading Desk · 14 March 2025",
+			"Electronic paper does not shine light into your eyes.",
+			`▶ Video: <a href="https://www.youtube.com/watch?v=dQw4w9WgXcQ">How E Ink works</a>`,
+		} {
+			if !strings.Contains(chapter, want) {
+				t.Errorf("chapter is missing %q", want)
+			}
+		}
+		if strings.Contains(chapter, "Great article, thanks!") {
+			t.Errorf("chapter contains page comments")
+		}
+		assertCover(t, book, false)
+		epubtest.EPUBCheck(t, path)
+	})
+
+	t.Run("messy documentation page", func(t *testing.T) {
+		dir := t.TempDir()
+		r := realRun("-URL", srv.URL+"/docs/config", "-SAVE", dir)
+		if r.code != exitOK {
+			t.Fatalf("exit code = %d, stderr = %q", r.code, r.stderr)
+		}
+		path := filepath.Join(dir, "configuring-the-sync-daemon.epub")
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		book := epubtest.Open(t, data)
+		book.AssertWellFormed(t)
+		chapter := book.File(t, "chapter.xhtml")
+		for _, want := range []string{
+			`<div lang="en" xml:lang="en">`,
+			`<h2 id="sec-1">File location</h2>`,
+			`<h3 id="sec-3">Durations</h3>`,
+			"<pre><code>interval = ",
+			"<td><code>wifi_only</code></td>",
+			`<a href="mailto:docs@example.com">`,
+			`<a href="` + srv.URL + `/faq#restart">`,
+			"recipes — including", // &mdash; as a character, not an entity
+		} {
+			if !strings.Contains(chapter, want) {
+				t.Errorf("chapter is missing %q", want)
+			}
+		}
+		for _, unwanted := range []string{"onclick", "style=", "javascript:", "analytics", "<form", "<button", "Was this page helpful"} {
+			if strings.Contains(chapter, unwanted) {
+				t.Errorf("chapter contains %q", unwanted)
+			}
+		}
+		assertCover(t, book, false)
+		squash := func(s string) string { return strings.Join(strings.Fields(s), " ") }
+		nav := squash(book.File(t, "nav.xhtml"))
+		for _, want := range []string{
+			`<a href="xhtml/chapter.xhtml#sec-1">File location</a>`,
+			`<a href="xhtml/chapter.xhtml#sec-2">Options</a> <ol> <li><a href="xhtml/chapter.xhtml#sec-3">Durations</a></li> </ol>`,
+			`<a href="xhtml/chapter.xhtml#sec-4">Example</a>`,
+		} {
+			if !strings.Contains(nav, want) {
+				t.Errorf("nav is missing %q:\n%s", want, nav)
+			}
+		}
+		if strings.Contains(nav, "Edge cases") {
+			t.Errorf("nav lists an <h4>; only sections and subsections belong in it")
+		}
+		epubtest.EPUBCheck(t, path)
+	})
+
+	t.Run("photo essay", func(t *testing.T) {
+		dir := t.TempDir()
+		r := realRun("-URL", srv.URL+"/essays/print-shop", "-SAVE", dir)
+		if r.code != exitOK {
+			t.Fatalf("exit code = %d, stderr = %q", r.code, r.stderr)
+		}
+		if !strings.Contains(r.stderr, "warning: image "+srv.URL+"/img/missing.png skipped: server returned 404 Not Found") {
+			t.Errorf("stderr = %q, want a warning about the missing image", r.stderr)
+		}
+		path := filepath.Join(dir, "a-walk-through-the-print-shop.epub")
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		book := epubtest.Open(t, data)
+		book.AssertWellFormed(t)
+
+		var stored []string
+		for _, name := range book.Order {
+			if strings.HasPrefix(name, "EPUB/images/") && name != "EPUB/images/cover.jpg" {
+				stored = append(stored, name)
+			}
+		}
+		if len(stored) != 3 {
+			t.Fatalf("book has images %v, want 3 (photo, diagram, WebP picture)", stored)
+		}
+		for _, name := range stored {
+			img, format, err := image.Decode(strings.NewReader(book.Files[name]))
+			if err != nil {
+				t.Fatalf("%s: %v", name, err)
+			}
+			if format != "jpeg" && format != "png" {
+				t.Errorf("%s is %s, want jpeg or png", name, format)
+			}
+			if b := img.Bounds(); b.Dx() > 1404 || b.Dy() > 1872 {
+				t.Errorf("%s is %dx%d, larger than the screen", name, b.Dx(), b.Dy())
+			}
+			switch img.(type) {
+			case *image.Gray, *image.Paletted:
+			default:
+				t.Errorf("%s decodes as %T, want grayscale", name, img)
+			}
+		}
+
+		chapter := book.File(t, "chapter.xhtml")
+		for _, want := range []string{
+			`alt="The old printing press"/><figcaption>The 1890s press still runs every Friday.</figcaption></figure>`,
+			`alt="How ink reaches the paper"/></figure>`,
+			`alt="Drawers of wooden type"/></figure>`,
+			`<span class="missing-image">[Image: small press icon]</span>`,
+			`<p class="byline">Grace Hopper · Letterpress Weekly</p>`, // no invented date
+		} {
+			if !strings.Contains(chapter, want) {
+				t.Errorf("chapter is missing %q", want)
+			}
+		}
+		if strings.Contains(chapter, "tracker.example") || strings.Contains(chapter, `<img src="http`) {
+			t.Errorf("chapter references remote images:\n%s", chapter)
+		}
+		assertCover(t, book, true)
+		if opf := book.File(t, ".opf"); !strings.Contains(opf, "<dc:publisher>Letterpress Weekly</dc:publisher>") ||
+			!strings.Contains(opf, "<dc:source>"+srv.URL+"/essays/print-shop</dc:source>") {
+			t.Errorf("OPF is missing the publisher or source:\n%s", opf)
+		}
+		epubtest.EPUBCheck(t, path)
+	})
+
+	t.Run("images inside lightbox zoom links", func(t *testing.T) {
+		// Regression test: go-trafilatura dropped every image on pages that
+		// wrap images in zoom links, such as Ars Technica articles.
+		dir := t.TempDir()
+		r := realRun("-URL", srv.URL+"/news/2026/10/kde-turns-30/", "-SAVE", dir)
+		if r.code != exitOK {
+			t.Fatalf("exit code = %d, stderr = %q", r.code, r.stderr)
+		}
+		if strings.Contains(r.stderr, "warning") {
+			t.Errorf("unexpected warnings: %q", r.stderr)
+		}
+		path := filepath.Join(dir, "kde-turns-30-with-plasma-6-8.epub")
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		book := epubtest.Open(t, data)
+		book.AssertWellFormed(t)
+		var stored int
+		for _, name := range book.Order {
+			if strings.HasPrefix(name, "EPUB/images/") && name != "EPUB/images/cover.jpg" {
+				stored++
+			}
+		}
+		if stored != 2 {
+			t.Errorf("book has %d images, want 2", stored)
+		}
+		chapter := book.File(t, "chapter.xhtml")
+		for _, want := range []string{
+			`<figcaption>The new remote upgrade dialog. Credit: KDE</figcaption></figure>`,
+			`alt="System Settings in Plasma 6.8"/><figcaption>System Settings has been reorganised.</figcaption></figure>`,
+		} {
+			if !strings.Contains(chapter, want) {
+				t.Errorf("chapter is missing %q", want)
+			}
+		}
+		if n := strings.Count(chapter, "The new remote upgrade dialog."); n != 1 {
+			t.Errorf("caption appears %d times, want once", n)
+		}
+		assertCover(t, book, true) // the page's og:image, as on Ars Technica
+		epubtest.EPUBCheck(t, path)
+	})
+
+	t.Run("image URLs with query strings or no extension", func(t *testing.T) {
+		// Regression test: go-trafilatura dropped images whose URL doesn't end
+		// in a file extension, such as every image on Wikipedia.
+		dir := t.TempDir()
+		r := realRun("-URL", srv.URL+"/wiki/Walking_Robots", "-SAVE", dir)
+		if r.code != exitOK {
+			t.Fatalf("exit code = %d, stderr = %q", r.code, r.stderr)
+		}
+		if strings.Contains(r.stderr, "warning") {
+			t.Errorf("unexpected warnings: %q", r.stderr)
+		}
+		path := filepath.Join(dir, "walking-robots-game.epub")
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		book := epubtest.Open(t, data)
+		book.AssertWellFormed(t)
+		var stored int
+		for _, name := range book.Order {
+			if strings.HasPrefix(name, "EPUB/images/") && name != "EPUB/images/cover.jpg" {
+				stored++
+			}
+		}
+		if stored != 3 {
+			t.Errorf("book has %d images, want 3 (logo, miniature, record sheet)", stored)
+		}
+		chapter := book.File(t, "chapter.xhtml")
+		for _, want := range []string{
+			`<figcaption>The game&#39;s original logo</figcaption></figure>`,
+			`<figcaption>A painted miniature on a hex map</figcaption></figure>`,
+			`<figcaption>A machine&#39;s record sheet</figcaption></figure>`,
+		} {
+			if !strings.Contains(chapter, want) {
+				t.Errorf("chapter is missing %q", want)
+			}
+		}
+		if strings.Contains(chapter, "rekopub.invalid") {
+			t.Errorf("chapter still contains stand-in image URLs:\n%s", chapter)
+		}
+		epubtest.EPUBCheck(t, path)
+	})
+
+	t.Run("KEPUB for Kobo", func(t *testing.T) {
+		dir := t.TempDir()
+		r := realRun("-URL", srv.URL+"/essays/print-shop", "-SAVE", filepath.Join(dir, "essay.epub"), "-KEPUB")
+		if r.code != exitOK {
+			t.Fatalf("exit code = %d, stderr = %q", r.code, r.stderr)
+		}
+		if !strings.Contains(r.stderr, "note: saving as essay.kepub.epub") {
+			t.Errorf("stderr = %q, want a note about the .kepub.epub name", r.stderr)
+		}
+		path := filepath.Join(dir, "essay.kepub.epub")
+		if strings.TrimSpace(r.stdout) != path {
+			t.Errorf("stdout = %q, want %q", r.stdout, path)
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		book := epubtest.Open(t, data)
+		book.AssertWellFormed(t)
+		chapter := book.File(t, "chapter.xhtml")
+		for _, want := range []string{`class="koboSpan"`, "The print shop on Mill Street has been running since 1894.", "<figcaption>"} {
+			if !strings.Contains(chapter, want) {
+				t.Errorf("chapter is missing %q", want)
+			}
+		}
+		assertCover(t, book, true)
+		epubtest.EPUBCheck(t, path)
+	})
+
+	t.Run("existing file stops before images are downloaded", func(t *testing.T) {
+		dir := t.TempDir()
+		existing := filepath.Join(dir, "a-walk-through-the-print-shop.epub")
+		writeFile(t, existing, "old")
+		before := imageRequests.Load()
+		r := realRun("-URL", srv.URL+"/essays/print-shop", "-SAVE", dir)
+		if r.code != exitExists {
+			t.Errorf("exit code = %d, want %d; stderr = %q", r.code, exitExists, r.stderr)
+		}
+		if n := imageRequests.Load() - before; n != 0 {
+			t.Errorf("%d images downloaded, want 0", n)
+		}
+		assertFile(t, existing, "old")
+	})
+
+	t.Run("not found", func(t *testing.T) {
+		dir := t.TempDir()
+		r := realRun("-URL", srv.URL+"/missing", "-SAVE", dir)
+		if r.code != exitFetch || !strings.Contains(r.stderr, "returned 404 Not Found") {
+			t.Errorf("exit code = %d, stderr = %q", r.code, r.stderr)
+		}
+		assertEmptyDir(t, dir)
+	})
+
+	t.Run("no content", func(t *testing.T) {
+		dir := t.TempDir()
+		r := realRun("-URL", srv.URL+"/app", "-SAVE", dir)
+		if r.code != exitNoContent || !strings.Contains(r.stderr, "no readable article content") {
+			t.Errorf("exit code = %d, stderr = %q", r.code, r.stderr)
+		}
+		assertEmptyDir(t, dir)
+	})
+}
+
+// TestLocalFilesNeverRead checks that a page can't pull files from the
+// reader's computer into the book, whichever way it refers to them.
+func TestLocalFilesNeverRead(t *testing.T) {
+	const marker = "TOP-SECRET-MARKER-7f3a"
+	secrets := t.TempDir()
+	secretText := filepath.Join(secrets, "secret.txt")
+	if err := os.WriteFile(secretText, []byte(marker), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	png, err := os.ReadFile("../../testdata/images/diagram.png")
+	if err != nil {
+		t.Fatal(err)
+	}
+	secretPNG := filepath.Join(secrets, "secret.png")
+	if err := os.WriteFile(secretPNG, png, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	fileURL := func(path string) string {
+		p := filepath.ToSlash(path)
+		if !strings.HasPrefix(p, "/") {
+			p = "/" + p // a Windows path such as C:/Users/...
+		}
+		return "file://" + p
+	}
+
+	blog, err := os.ReadFile("../../testdata/pages/blog.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+	page := strings.Replace(string(blog), "</head>",
+		`<meta property="og:image" content="`+fileURL(secretPNG)+`"></head>`, 1)
+	page = strings.Replace(page, "</p>", "</p>"+
+		`<p><img src="`+fileURL(secretPNG)+`" alt="one">`+
+		`<img src="file://localhost`+filepath.ToSlash(secretPNG)+`" alt="two">`+
+		`<img src="`+filepath.ToSlash(secretPNG)+`" alt="three">`+
+		`<img src="`+secretPNG+`" alt="four">`+
+		`<img src="`+fileURL(secretText)+`" alt="five">`+
+		`<picture><source srcset="`+fileURL(secretPNG)+` 2x"><img src="`+fileURL(secretPNG)+`" alt="six"></picture>`+
+		`<a href="`+fileURL(secretText)+`">notes</a></p>`, 1)
+	var requests []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests = append(requests, r.URL.Path)
+		if r.URL.Path != "/posts/eink" {
+			http.NotFound(w, r) // relative paths resolve to this server
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		w.Write([]byte(page))
+	}))
+	defer srv.Close()
+
+	out := filepath.Join(t.TempDir(), "book.epub")
+	var stdout, stderr bytes.Buffer
+	code := run(context.Background(), []string{"-URL", srv.URL + "/posts/eink", "-SAVE", out}, &stdout, &stderr, pipeline.Run)
+	if code != exitOK {
+		t.Fatalf("exit %d: %s", code, stderr.String())
+	}
+	data, err := os.ReadFile(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	book := epubtest.Open(t, data)
+	for name, content := range book.Files {
+		if strings.Contains(content, marker) {
+			t.Errorf("%s contains the secret file's content", name)
+		}
+		if strings.HasSuffix(name, ".png") || (strings.HasSuffix(name, ".jpg") && !strings.HasSuffix(name, "cover.jpg")) {
+			t.Errorf("book contains image %s; the only image should be the typographic cover", name)
+		}
+	}
+	if chapter := book.File(t, "chapter.xhtml"); strings.Contains(chapter, "file:") {
+		t.Errorf("chapter still refers to a local file:\n%s", chapter)
+	}
+	t.Logf("server requests: %v\nwarnings:\n%s", requests, stderr.String())
+}

cmd/rekopub/version.go

@@ -0,0 +1,39 @@
+package main
+
+import "runtime/debug"
+
+// version is set at build time by release builds and build.cmd:
+//
+//	go build -ldflags "-X main.version=v1.0.0" ./cmd/rekopub
+var version string
+
+// versionString reports the version: the one set at build time, else the
+// module version for "go install ...@v1.0.0", else the git commit the
+// binary was built from.
+func versionString() string {
+	if version != "" {
+		return version
+	}
+	info, ok := debug.ReadBuildInfo()
+	if !ok {
+		return "unknown"
+	}
+	if v := info.Main.Version; v != "" && v != "(devel)" {
+		return v
+	}
+	var rev, modified string
+	for _, s := range info.Settings {
+		switch s.Key {
+		case "vcs.revision":
+			rev = s.Value
+		case "vcs.modified":
+			if s.Value == "true" {
+				modified = "-dirty"
+			}
+		}
+	}
+	if rev == "" {
+		return "dev"
+	}
+	return "dev-" + rev[:min(len(rev), 12)] + modified
+}

go.mod

@@ -0,0 +1,53 @@
+module git.monoexp.com/bajdev/rekopub
+
+go 1.26.0
+
+require (
+	github.com/go-shiori/go-epub v1.2.1
+	github.com/google/uuid v1.6.0
+	github.com/markusmobius/go-trafilatura v1.12.2
+	github.com/microcosm-cc/bluemonday v1.0.27
+	github.com/pgaskin/kepubify/v4 v4.0.4
+	github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c
+	github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef
+	github.com/vincent-petithory/dataurl v1.0.0
+	golang.org/x/image v0.47.0
+	golang.org/x/net v0.61.0
+	golang.org/x/sync v0.24.0
+	golang.org/x/text v0.43.0
+)
+
+require (
+	github.com/RadhiFadlillah/whatlanggo v0.0.0-20240916001553-aac1f0f737fc // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
+	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/beevik/etree v1.2.0 // indirect
+	github.com/elliotchance/pie/v2 v2.9.0 // indirect
+	github.com/forPelevin/gomoji v1.2.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
+	github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f // indirect
+	github.com/gofrs/uuid/v5 v5.0.0 // indirect
+	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
+	github.com/gorilla/css v1.0.1 // indirect
+	github.com/hablullah/go-hijri v1.0.2 // indirect
+	github.com/hablullah/go-juliandays v1.0.0 // indirect
+	github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect
+	github.com/kr/smartypants v0.1.0 // indirect
+	github.com/markusmobius/go-dateparser v1.2.3 // indirect
+	github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4 // indirect
+	github.com/markusmobius/go-htmldate v1.9.1 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/pgaskin/kepubify/_/go116-zip.go117 v0.0.0-20210611152744-2d89b3182523 // indirect
+	github.com/pgaskin/kepubify/_/html v0.0.0-20211223234002-6ee2cc632cdc // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/rs/zerolog v1.33.0 // indirect
+	github.com/tetratelabs/wazero v1.8.1 // indirect
+	github.com/wasilibs/go-re2 v1.7.0 // indirect
+	github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect
+	github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect
+	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
+	golang.org/x/sys v0.49.0 // indirect
+)

go.sum

@@ -0,0 +1,161 @@
+github.com/RadhiFadlillah/whatlanggo v0.0.0-20240916001553-aac1f0f737fc h1:6aA31zw7fnfJ/G1ebisIesCDl44slkIVFqk3YTSadd8=
+github.com/RadhiFadlillah/whatlanggo v0.0.0-20240916001553-aac1f0f737fc/go.mod h1:PgrPWaMBxL1lyq1k5DEMqC0Y67R3pG1vEsHzxFXeDxc=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b/go.mod h1:obBQGGIFbbv9KWg92Qu9UHeD94JXmHD1jovY/z6I3O8=
+github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
+github.com/beevik/etree v1.2.0 h1:l7WETslUG/T+xOPs47dtd6jov2Ii/8/OjCldk5fYfQw=
+github.com/beevik/etree v1.2.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elliotchance/pie/v2 v2.9.0 h1:BkEhh8b/avGCSpXpABSjNuytxlI/S2snkjT3vtVORjw=
+github.com/elliotchance/pie/v2 v2.9.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og=
+github.com/forPelevin/gomoji v1.2.0 h1:9k4WVSSkE1ARO/BWywxgEUBvR/jMnao6EZzrql5nxJ8=
+github.com/forPelevin/gomoji v1.2.0/go.mod h1:8+Z3KNGkdslmeGZBC3tCrwMrcPy5GRzAD+gL9NAwMXg=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
+github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
+github.com/go-shiori/go-epub v1.2.1 h1:+K/WxrvmfFQY69cpryiObrT6X7WhkwpqhHY65AHs2Rg=
+github.com/go-shiori/go-epub v1.2.1/go.mod h1:3rCTODnigEgy2j3ksndClrGT9h/dcz3js9q4yPX7hf8=
+github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f h1:cypj7SJh+47G9J3VCPdMzT3uWcXWAWDJA54ErTfOigI=
+github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f/go.mod h1:YWa00ashoPZMAOElrSn4E1cJErhDVU6PWAll4Hxzn+w=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/uuid/v5 v5.0.0 h1:p544++a97kEL+svbcFbCQVM9KFu0Yo25UoISXGNNH9M=
+github.com/gofrs/uuid/v5 v5.0.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
+github.com/hablullah/go-hijri v1.0.2 h1:drT/MZpSZJQXo7jftf5fthArShcaMtsal0Zf/dnmp6k=
+github.com/hablullah/go-hijri v1.0.2/go.mod h1:OS5qyYLDjORXzK4O1adFw9Q5WfhOcMdAKglDkcTxgWQ=
+github.com/hablullah/go-juliandays v1.0.0 h1:A8YM7wIj16SzlKT0SRJc9CD29iiaUzpBLzh5hr0/5p0=
+github.com/hablullah/go-juliandays v1.0.0/go.mod h1:0JOYq4oFOuDja+oospuc61YoX+uNEn7Z6uHYTbBzdGc=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE=
+github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958/go.mod h1:Wqfu7mjUHj9WDzSSPI5KfBclTTEnLveRUFr/ujWnTgE=
+github.com/kr/smartypants v0.1.0 h1:Sn8hn5XrY+uXrxSWUdcr621Gfpk11mOGGVs4XX06kEw=
+github.com/kr/smartypants v0.1.0/go.mod h1:EcTX9ge+SWNaGwbQvHwNICsMGavh98FLUqyOWFr+j9c=
+github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a h1:tdPcGgyiH0K+SbsJBBm2oPyEIOTAvLBwD9TuUwVtZho=
+github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/markusmobius/go-dateparser v1.2.3 h1:TvrsIvr5uk+3v6poDjaicnAFJ5IgtFHgLiuMY2Eb7Nw=
+github.com/markusmobius/go-dateparser v1.2.3/go.mod h1:cMwQRrBUQlK1UI5TIFHEcvpsMbkWrQLXuaPNMFzuYLk=
+github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4 h1:+7kfF1+dmSXV469sqjeNC+eKJF7xDuS5mvZA3DFVLLY=
+github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4/go.mod h1:E7PoeC3nd4GqtxP1A64v7JDBxpAbpTSnhlq9/DHmQ28=
+github.com/markusmobius/go-htmldate v1.9.1 h1:0kfVz0wdxGCBaotWNzdtIZKhy7+8ClBlzvANQ67Rlt8=
+github.com/markusmobius/go-htmldate v1.9.1/go.mod h1:fLls4rjQDxYR+Pxhf0YR6Ht8dEeHd4SxK/NPaVqhMa8=
+github.com/markusmobius/go-trafilatura v1.12.2 h1:JgEto0kDjwTuyXFl6TB+psrs1QGJqTdYJEbLhDy1vrw=
+github.com/markusmobius/go-trafilatura v1.12.2/go.mod h1:2WnYLuvGBgJAarHaAQnsvofihEojt2xDDrtVJU5UXZI=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
+github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
+github.com/pgaskin/kepubify/_/go116-zip.go117 v0.0.0-20210611152744-2d89b3182523 h1:pYGj3rKTy+TDs5Z707kT+ztjoIDCy76lc2UPkZocAFM=
+github.com/pgaskin/kepubify/_/go116-zip.go117 v0.0.0-20210611152744-2d89b3182523/go.mod h1:FNMbV/TSSnhqyzjq8jsS+VD0o/gwpuCH0dh8G1uQ/fw=
+github.com/pgaskin/kepubify/_/html v0.0.0-20211223234002-6ee2cc632cdc h1:mJk4TIXTO+JmxgHJ5iyil42PLQJWkyaKB/qNcjJU6h4=
+github.com/pgaskin/kepubify/_/html v0.0.0-20211223234002-6ee2cc632cdc/go.mod h1:fxzoIpMFAReNKunZ+ttVbf3hNVrJGtrSZMI4olZizbs=
+github.com/pgaskin/kepubify/v4 v4.0.4 h1:8ePyepo4eRNSmeDs5MdJLJtit+zxmK36wILmGcvpccU=
+github.com/pgaskin/kepubify/v4 v4.0.4/go.mod h1:wzUdFNYW2uZh2xfHDuzNRRUO4WqV+y99UBxVd3rBTus=
+github.com/pgaskin/koboutils/v2 v2.1.2-0.20220306004009-a07e72ebae42/go.mod h1:wTzkDIlsxmUyfwfspGcm0Ap+HOxSUYV0S8kMYrf+0gM=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
+github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
+github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE=
+github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q=
+github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ=
+github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550=
+github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
+github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI=
+github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U=
+github.com/wasilibs/go-re2 v1.7.0 h1:bYhl8gn+a9h01dxwotNycxkiFPTiSgwUrIz8KZJ90Lc=
+github.com/wasilibs/go-re2 v1.7.0/go.mod h1:sUsZMLflgl+LNivDE229omtmvjICmOseT9xOy199VDU=
+github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ=
+github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
+github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4=
+github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY=
+github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 h1:0sw0nJM544SpsihWx1bkXdYLQDlzRflMgFJQ4Yih9ts=
+github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4/go.mod h1:+ccdNT0xMY1dtc5XBxumbYfOUhmduiGudqaDgD2rVRE=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY=
+golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
+golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo=
+golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
+golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
+golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
+golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
+golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/assets/assets.go

@@ -0,0 +1,254 @@
+// Package assets downloads the images in an article and prepares them for
+// e-ink screens.
+package assets
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"image"
+	"io"
+	"mime"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/vincent-petithory/dataurl"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+	"golang.org/x/sync/errgroup"
+
+	"git.monoexp.com/bajdev/rekopub/internal/device"
+	"git.monoexp.com/bajdev/rekopub/internal/fetch"
+)
+
+// Image is a processed image to add to the book.
+type Image struct {
+	Filename  string // e.g. "3fa2c1d9e0ab.jpg", named by content hash
+	MediaType string // image/jpeg or image/png
+	Data      []byte
+}
+
+// Processor downloads and converts images.
+type Processor struct {
+	Client      *http.Client
+	UserAgent   string
+	MaxBytes    int64 // largest image accepted
+	Concurrency int   // downloads at once
+	Profile     device.Profile
+	// Warn is called for every image that couldn't be used. It may be nil.
+	Warn func(msg string)
+}
+
+// New returns a Processor using client and the e-ink profile, with the
+// limits from PLAN.md: 10 MB per image and 8 downloads at once.
+func New(client *http.Client, userAgent string) *Processor {
+	return &Processor{
+		Client:      client,
+		UserAgent:   userAgent,
+		MaxBytes:    10 << 20,
+		Concurrency: 8,
+		Profile:     device.EInk,
+	}
+}
+
+// Process downloads every image in body, converts it for e-ink and points
+// its <img> at the new file. The returned images are in document order,
+// one per distinct image. An image that can't be used is replaced by its
+// alt text, or removed if it has none; that is never an error. Only
+// cancellation of ctx stops processing.
+func (p *Processor) Process(ctx context.Context, body *html.Node, page *url.URL) ([]Image, error) {
+	var imgs []*html.Node
+	var sources []string // distinct src values, in document order
+	seen := map[string]bool{}
+	walk(body, func(n *html.Node) {
+		if n.DataAtom != atom.Img {
+			return
+		}
+		imgs = append(imgs, n)
+		if src := attr(n, "src"); src != "" && !seen[src] {
+			seen[src] = true
+			sources = append(sources, src)
+		}
+	})
+
+	results := make([]*encoded, len(sources))
+	errs := make([]error, len(sources))
+	g, gctx := errgroup.WithContext(ctx)
+	g.SetLimit(max(1, p.Concurrency))
+	for i, src := range sources {
+		g.Go(func() error {
+			results[i], errs[i] = p.load(gctx, src, page)
+			return nil
+		})
+	}
+	g.Wait()
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+
+	// Name files by content hash, so the same image found under different
+	// URLs is stored once.
+	var images []Image
+	filenames := map[string]string{} // src → filename
+	byHash := map[string]bool{}
+	for i, src := range sources {
+		if errs[i] != nil {
+			if !errors.Is(errs[i], errTooSmall) {
+				p.warn(fmt.Sprintf("image %s skipped: %v", short(src), errs[i]))
+			}
+			continue
+		}
+		sum := sha1.Sum(results[i].data)
+		name := hex.EncodeToString(sum[:])[:12] + results[i].ext
+		filenames[src] = name
+		if !byHash[name] {
+			byHash[name] = true
+			images = append(images, Image{Filename: name, MediaType: results[i].mediaType, Data: results[i].data})
+		}
+	}
+
+	for _, n := range imgs {
+		if name, ok := filenames[attr(n, "src")]; ok {
+			n.Attr = []html.Attribute{{Key: "src", Val: name}, {Key: "alt", Val: attr(n, "alt")}}
+		} else {
+			replaceWithAlt(n)
+		}
+	}
+	return images, nil
+}
+
+// Decode fetches src (an http(s) or data: URL) and decodes it without
+// converting it, for uses such as the cover. SVG is rasterized.
+func (p *Processor) Decode(ctx context.Context, src string, page *url.URL) (image.Image, error) {
+	data, svg, err := p.fetch(ctx, src, page)
+	if err != nil {
+		return nil, err
+	}
+	return decodeAny(data, svg, p.Profile)
+}
+
+// load fetches src (an http(s) or data: URL) and converts it.
+func (p *Processor) load(ctx context.Context, src string, page *url.URL) (*encoded, error) {
+	data, svg, err := p.fetch(ctx, src, page)
+	if err != nil {
+		return nil, err
+	}
+	return convert(data, svg, p.Profile)
+}
+
+// fetch returns the bytes of src and whether they are SVG.
+func (p *Processor) fetch(ctx context.Context, src string, page *url.URL) (data []byte, svg bool, err error) {
+	var mediaType string
+	if strings.HasPrefix(src, "data:") {
+		du, err := dataurl.DecodeString(src)
+		if err != nil {
+			return nil, false, fmt.Errorf("bad data URL: %w", err)
+		}
+		data, mediaType = du.Data, du.ContentType()
+	} else {
+		data, mediaType, err = p.download(ctx, src, page)
+		if err != nil {
+			return nil, false, err
+		}
+	}
+	return data, mediaType == "image/svg+xml" || looksLikeSVG(data), nil
+}
+
+// download fetches an image, sending the article as Referer because some
+// image hosts refuse requests without one.
+func (p *Processor) download(ctx context.Context, src string, page *url.URL) ([]byte, string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, src, nil)
+	if err != nil {
+		return nil, "", err
+	}
+	if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
+		return nil, "", fmt.Errorf("unsupported URL scheme %q", req.URL.Scheme)
+	}
+	req.Header.Set("User-Agent", p.UserAgent)
+	req.Header.Set("Accept", "image/jpeg,image/png,image/webp,image/gif,image/svg+xml;q=0.9,*/*;q=0.5")
+	req.Header.Set("Referer", page.String())
+
+	resp, err := p.Client.Do(req)
+	if err != nil {
+		return nil, "", fetch.Explain(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return nil, "", fmt.Errorf("server returned %d %s", resp.StatusCode, http.StatusText(resp.StatusCode))
+	}
+	data, err := io.ReadAll(io.LimitReader(resp.Body, p.MaxBytes+1))
+	if err != nil {
+		return nil, "", err
+	}
+	if int64(len(data)) > p.MaxBytes {
+		return nil, "", fmt.Errorf("larger than %d MB", p.MaxBytes>>20)
+	}
+	mediaType, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type"))
+	if mediaType != "" && !strings.HasPrefix(mediaType, "image/") && mediaType != "application/octet-stream" {
+		return nil, "", fmt.Errorf("not an image (content type %q)", mediaType)
+	}
+	return data, mediaType, nil
+}
+
+// looksLikeSVG reports whether data starts like an SVG document, for
+// servers that send SVG with a generic content type.
+func looksLikeSVG(data []byte) bool {
+	head := bytes.ToLower(data[:min(len(data), 1024)])
+	return bytes.Contains(head, []byte("<svg"))
+}
+
+// replaceWithAlt swaps an unusable image for "[Image: alt text]", or
+// removes it when there is no alt text. A <span> is valid everywhere an
+// <img> is.
+func replaceWithAlt(n *html.Node) {
+	parent := n.Parent
+	if parent == nil {
+		return
+	}
+	if alt := strings.TrimSpace(attr(n, "alt")); alt != "" {
+		span := &html.Node{Type: html.ElementNode, Data: "span", DataAtom: atom.Span,
+			Attr: []html.Attribute{{Key: "class", Val: "missing-image"}}}
+		span.AppendChild(&html.Node{Type: html.TextNode, Data: "[Image: " + alt + "]"})
+		parent.InsertBefore(span, n)
+	}
+	parent.RemoveChild(n)
+}
+
+func (p *Processor) warn(msg string) {
+	if p.Warn != nil {
+		p.Warn(msg)
+	}
+}
+
+// short trims long image URLs (and data URLs) for warnings.
+func short(src string) string {
+	if strings.HasPrefix(src, "data:") {
+		return "(embedded data URL)"
+	}
+	if len(src) > 100 {
+		return src[:97] + "..."
+	}
+	return src
+}
+
+func attr(n *html.Node, key string) string {
+	for _, a := range n.Attr {
+		if a.Key == key {
+			return a.Val
+		}
+	}
+	return ""
+}
+
+func walk(n *html.Node, f func(*html.Node)) {
+	if n.Type == html.ElementNode {
+		f(n)
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		walk(c, f)
+	}
+}

internal/assets/assets_test.go

@@ -0,0 +1,421 @@
+package assets
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/binary"
+	"hash/crc32"
+	"image"
+	"image/color"
+	"image/gif"
+	"image/jpeg"
+	"image/png"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+// photo returns a w×h colour image with smooth gradients and noise, which
+// uses many gray levels like a photograph.
+func photo(w, h int) image.Image {
+	r := rand.New(rand.NewSource(1))
+	img := image.NewRGBA(image.Rect(0, 0, w, h))
+	for y := 0; y < h; y++ {
+		for x := 0; x < w; x++ {
+			n := uint8(r.Intn(40))
+			img.Set(x, y, color.RGBA{uint8(x*200/w) + n, uint8(y*200/h) + n, 120 + n, 255})
+		}
+	}
+	return img
+}
+
+// lineArt returns a transparent image with a black frame and a red bar,
+// like a diagram exported with a transparent background.
+func lineArt(w, h int) image.Image {
+	img := image.NewNRGBA(image.Rect(0, 0, w, h))
+	for y := 0; y < h; y++ {
+		for x := 0; x < w; x++ {
+			switch {
+			case x < 4 || y < 4 || x >= w-4 || y >= h-4:
+				img.Set(x, y, color.NRGBA{0, 0, 0, 255})
+			case y > h/2-10 && y < h/2+10:
+				img.Set(x, y, color.NRGBA{220, 30, 30, 255})
+			}
+		}
+	}
+	return img
+}
+
+func encodePNG(img image.Image) []byte {
+	var b bytes.Buffer
+	if err := png.Encode(&b, img); err != nil {
+		panic(err)
+	}
+	return b.Bytes()
+}
+
+func encodeJPEG(img image.Image) []byte {
+	var b bytes.Buffer
+	if err := jpeg.Encode(&b, img, &jpeg.Options{Quality: 90}); err != nil {
+		panic(err)
+	}
+	return b.Bytes()
+}
+
+// animatedGIF has a white first frame and a black second frame.
+func animatedGIF() []byte {
+	pal := color.Palette{color.White, color.Black}
+	frame := func(c uint8) *image.Paletted {
+		f := image.NewPaletted(image.Rect(0, 0, 50, 50), pal)
+		for i := range f.Pix {
+			f.Pix[i] = c
+		}
+		f.Pix[0] = 1 - c // one odd pixel so the frame isn't entirely flat
+		return f
+	}
+	var b bytes.Buffer
+	if err := gif.EncodeAll(&b, &gif.GIF{Image: []*image.Paletted{frame(0), frame(1)}, Delay: []int{10, 10}}); err != nil {
+		panic(err)
+	}
+	return b.Bytes()
+}
+
+// pngBomb is a valid PNG header claiming 100000×100000 pixels.
+func pngBomb() []byte {
+	var b bytes.Buffer
+	b.Write([]byte("\x89PNG\r\n\x1a\n"))
+	ihdr := make([]byte, 13)
+	binary.BigEndian.PutUint32(ihdr[0:], 100000)
+	binary.BigEndian.PutUint32(ihdr[4:], 100000)
+	ihdr[8], ihdr[9] = 8, 2 // 8-bit RGB
+	binary.Write(&b, binary.BigEndian, uint32(len(ihdr)))
+	chunk := append([]byte("IHDR"), ihdr...)
+	b.Write(chunk)
+	binary.Write(&b, binary.BigEndian, crc32.ChecksumIEEE(chunk))
+	return b.Bytes()
+}
+
+const svgDoc = `<?xml version="1.0"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 200 100" width="200" height="100">
+  <rect x="10" y="10" width="180" height="80" fill="none" stroke="black" stroke-width="4"/>
+  <circle cx="100" cy="50" r="30" fill="#3366cc"/>
+</svg>`
+
+type testServer struct {
+	*httptest.Server
+	mu       sync.Mutex
+	requests map[string]int
+	referers map[string]string
+}
+
+type file struct {
+	contentType string
+	body        []byte
+}
+
+// files are the images the test server offers, generated once per run.
+var files = sync.OnceValue(func() map[string]file {
+	webp, err := os.ReadFile("../../testdata/images/gradient.webp")
+	if err != nil {
+		panic(err)
+	}
+	return map[string]file{
+		"/photo.jpg":     {"image/jpeg", encodeJPEG(photo(3000, 2000))},
+		"/small.jpg":     {"image/jpeg", encodeJPEG(photo(300, 200))},
+		"/tall.jpg":      {"image/jpeg", encodeJPEG(photo(1000, 4000))},
+		"/diagram.png":   {"image/png", encodePNG(lineArt(600, 400))},
+		"/copy.png":      {"image/png", encodePNG(lineArt(600, 400))},
+		"/anim.gif":      {"image/gif", animatedGIF()},
+		"/drawing.svg":   {"image/svg+xml", []byte(svgDoc)},
+		"/drawing-bin":   {"application/octet-stream", []byte(svgDoc)},
+		"/gradient.webp": {"image/webp", webp},
+		"/pixel.png":     {"image/png", encodePNG(image.NewGray(image.Rect(0, 0, 1, 1)))},
+		"/bomb.png":      {"image/png", pngBomb()},
+		"/page.html":     {"text/html", []byte("<html>not an image</html>")},
+		"/broken.jpg":    {"image/jpeg", []byte("not really a jpeg")},
+	}
+})
+
+func newServer(t *testing.T) *testServer {
+	t.Helper()
+	files := files()
+	s := &testServer{requests: map[string]int{}, referers: map[string]string{}}
+	s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		s.mu.Lock()
+		s.requests[r.URL.Path]++
+		s.referers[r.URL.Path] = r.Referer()
+		s.mu.Unlock()
+		f, ok := files[r.URL.Path]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", f.contentType)
+		w.Write(f.body)
+	}))
+	t.Cleanup(s.Close)
+	return s
+}
+
+type run struct {
+	images   []Image
+	body     string
+	warnings []string
+}
+
+// process runs Process on markup (with {S} replaced by the server URL).
+func process(t *testing.T, s *testServer, markup string) run {
+	t.Helper()
+	doc, err := html.Parse(strings.NewReader("<body>" + strings.ReplaceAll(markup, "{S}", s.URL) + "</body>"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	body := doc.FirstChild.LastChild
+	page, _ := url.Parse(s.URL + "/article")
+	var r run
+	p := New(s.Client(), "test-agent")
+	p.Warn = func(msg string) { r.warnings = append(r.warnings, msg) }
+	r.images, err = p.Process(context.Background(), body, page)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var b strings.Builder
+	for c := body.FirstChild; c != nil; c = c.NextSibling {
+		html.Render(&b, c)
+	}
+	r.body = b.String()
+	return r
+}
+
+func decodeImage(t *testing.T, img Image) image.Image {
+	t.Helper()
+	m, format, err := image.Decode(bytes.NewReader(img.Data))
+	if err != nil {
+		t.Fatalf("%s doesn't decode: %v", img.Filename, err)
+	}
+	if want := strings.TrimPrefix(img.MediaType, "image/"); format != want {
+		t.Errorf("%s: format %s, media type %s", img.Filename, format, img.MediaType)
+	}
+	return m
+}
+
+// assertGray fails unless every pixel of m is a shade of gray.
+func assertGray(t *testing.T, name string, m image.Image) {
+	t.Helper()
+	b := m.Bounds()
+	for y := b.Min.Y; y < b.Max.Y; y += 7 {
+		for x := b.Min.X; x < b.Max.X; x += 7 {
+			r, g, bl, _ := m.At(x, y).RGBA()
+			if r != g || g != bl {
+				t.Fatalf("%s: pixel (%d,%d) is not gray: %d,%d,%d", name, x, y, r, g, bl)
+			}
+		}
+	}
+}
+
+func TestPhotoBecomesGrayJPEGWithinScreen(t *testing.T) {
+	s := newServer(t)
+	r := process(t, s, `<img src="{S}/photo.jpg" alt="A photo">`)
+	if len(r.images) != 1 {
+		t.Fatalf("got %d images, want 1; warnings: %v", len(r.images), r.warnings)
+	}
+	img := r.images[0]
+	if img.MediaType != "image/jpeg" || !strings.HasSuffix(img.Filename, ".jpg") {
+		t.Errorf("photo stored as %s (%s), want JPEG", img.Filename, img.MediaType)
+	}
+	m := decodeImage(t, img)
+	if b := m.Bounds(); b.Dx() != 1404 || b.Dy() != 936 {
+		t.Errorf("photo is %dx%d, want 1404x936", b.Dx(), b.Dy())
+	}
+	if _, ok := m.(*image.Gray); !ok {
+		t.Errorf("photo decodes as %T, want a grayscale JPEG", m)
+	}
+	if want := `<img src="` + img.Filename + `" alt="A photo"/>`; r.body != want {
+		t.Errorf("body = %q, want %q", r.body, want)
+	}
+	if ref := s.referers["/photo.jpg"]; ref != s.URL+"/article" {
+		t.Errorf("Referer = %q, want the article URL", ref)
+	}
+}
+
+func TestTallImageFitsHeight(t *testing.T) {
+	r := process(t, newServer(t), `<img src="{S}/tall.jpg">`)
+	m := decodeImage(t, r.images[0])
+	if b := m.Bounds(); b.Dx() != 468 || b.Dy() != 1872 {
+		t.Errorf("tall image is %dx%d, want 468x1872", b.Dx(), b.Dy())
+	}
+}
+
+func TestSmallImageIsNotEnlarged(t *testing.T) {
+	r := process(t, newServer(t), `<img src="{S}/small.jpg">`)
+	m := decodeImage(t, r.images[0])
+	if b := m.Bounds(); b.Dx() != 300 || b.Dy() != 200 {
+		t.Errorf("small image is %dx%d, want 300x200", b.Dx(), b.Dy())
+	}
+}
+
+func TestLineArtBecomes16LevelPNGOnWhite(t *testing.T) {
+	r := process(t, newServer(t), `<img src="{S}/diagram.png" alt="Diagram">`)
+	if len(r.images) != 1 {
+		t.Fatalf("got %d images; warnings: %v", len(r.images), r.warnings)
+	}
+	img := r.images[0]
+	if img.MediaType != "image/png" {
+		t.Fatalf("diagram stored as %s, want PNG", img.MediaType)
+	}
+	m := decodeImage(t, img)
+	assertGray(t, img.Filename, m)
+	pal, ok := m.(*image.Paletted)
+	if !ok || len(pal.Palette) != 16 {
+		t.Fatalf("diagram decodes as %T, want a 16-colour paletted PNG", m)
+	}
+	// The transparent background must be white, not black.
+	if r, _, _, _ := m.At(300, 100).RGBA(); r>>8 != 255 {
+		t.Errorf("transparent area became %d, want white (255)", r>>8)
+	}
+	if r, _, _, _ := m.At(1, 1).RGBA(); r>>8 != 0 {
+		t.Errorf("black frame became %d, want black (0)", r>>8)
+	}
+}
+
+func TestFormatsAreConverted(t *testing.T) {
+	s := newServer(t)
+	for _, path := range []string{"/gradient.webp", "/anim.gif", "/drawing.svg", "/drawing-bin"} {
+		t.Run(path, func(t *testing.T) {
+			r := process(t, s, `<img src="{S}`+path+`">`)
+			if len(r.images) != 1 {
+				t.Fatalf("got %d images; warnings: %v", len(r.images), r.warnings)
+			}
+			img := r.images[0]
+			if img.MediaType != "image/jpeg" && img.MediaType != "image/png" {
+				t.Errorf("stored as %s, want JPEG or PNG", img.MediaType)
+			}
+			assertGray(t, path, decodeImage(t, img))
+		})
+	}
+}
+
+func TestAnimatedGIFUsesFirstFrame(t *testing.T) {
+	r := process(t, newServer(t), `<img src="{S}/anim.gif">`)
+	m := decodeImage(t, r.images[0])
+	if v, _, _, _ := m.At(25, 25).RGBA(); v>>8 != 255 {
+		t.Errorf("pixel = %d, want white from the first frame", v>>8)
+	}
+}
+
+func TestSVGIsRasterized(t *testing.T) {
+	r := process(t, newServer(t), `<img src="{S}/drawing.svg">`)
+	m := decodeImage(t, r.images[0])
+	// Drawn at twice its 200×100 size.
+	if b := m.Bounds(); b.Dx() != 400 || b.Dy() != 200 {
+		t.Errorf("SVG rendered at %dx%d, want 400x200", b.Dx(), b.Dy())
+	}
+	if v, _, _, _ := m.At(200, 100).RGBA(); v>>8 > 200 {
+		t.Errorf("circle centre is %d, want a dark gray", v>>8)
+	}
+}
+
+func TestDataURLImage(t *testing.T) {
+	data := base64.StdEncoding.EncodeToString(encodePNG(lineArt(100, 80)))
+	r := process(t, newServer(t), `<img src="data:image/png;base64,`+data+`" alt="inline">`)
+	if len(r.images) != 1 || r.images[0].MediaType != "image/png" {
+		t.Fatalf("images = %+v; warnings: %v", r.images, r.warnings)
+	}
+}
+
+func TestDuplicatesAreStoredOnce(t *testing.T) {
+	s := newServer(t)
+	r := process(t, s, `<img src="{S}/diagram.png"><p><img src="{S}/copy.png"></p><img src="{S}/diagram.png">`)
+	if len(r.images) != 1 {
+		t.Fatalf("got %d images, want 1 (identical content)", len(r.images))
+	}
+	if strings.Count(r.body, r.images[0].Filename) != 3 {
+		t.Errorf("body = %q, want all three images pointing at %s", r.body, r.images[0].Filename)
+	}
+	if n := s.requests["/diagram.png"]; n != 1 {
+		t.Errorf("diagram.png downloaded %d times, want 1", n)
+	}
+}
+
+func TestUnusableImages(t *testing.T) {
+	s := newServer(t)
+	tests := []struct {
+		name, markup, want, warning string
+	}{
+		{"not found, with alt", `<p>See <img src="{S}/missing.jpg" alt="the chart">.</p>`,
+			`<p>See <span class="missing-image">[Image: the chart]</span>.</p>`, "404 Not Found"},
+		{"not found, without alt", `<p>See <img src="{S}/missing.jpg">.</p>`, `<p>See .</p>`, "404"},
+		{"not an image", `<img src="{S}/page.html" alt="x">`, `<span class="missing-image">[Image: x]</span>`, "not an image"},
+		{"damaged", `<img src="{S}/broken.jpg" alt="x">`, `<span class="missing-image">[Image: x]</span>`, "unsupported or damaged"},
+		{"decompression bomb", `<img src="{S}/bomb.png" alt="x">`, `<span class="missing-image">[Image: x]</span>`, "too large"},
+		{"unsupported scheme", `<img src="ftp://example.com/a.png" alt="x">`, `<span class="missing-image">[Image: x]</span>`, "unsupported URL scheme"},
+		// Local files are never read, whatever form the path takes.
+		{"file URL", `<img src="file:///etc/hosts" alt="x">`, `<span class="missing-image">[Image: x]</span>`, "unsupported URL scheme"},
+		{"file URL with host", `<img src="file://localhost/etc/hosts" alt="x">`, `<span class="missing-image">[Image: x]</span>`, "unsupported URL scheme"},
+		{"absolute path", `<img src="/etc/hosts" alt="x">`, `<span class="missing-image">[Image: x]</span>`, "unsupported URL scheme"},
+		{"Windows path", `<img src="C:\Windows\win.ini" alt="x">`, `<span class="missing-image">[Image: x]</span>`, "skipped"},
+		{"UNC path", `<img src="\\server\share\a.png" alt="x">`, `<span class="missing-image">[Image: x]</span>`, "skipped"},
+		// Tracking pixels are dropped without a warning.
+		{"tracking pixel", `<p>a<img src="{S}/pixel.png">b</p>`, `<p>ab</p>`, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := process(t, s, tt.markup)
+			if len(r.images) != 0 {
+				t.Errorf("got %d images, want 0", len(r.images))
+			}
+			if r.body != tt.want {
+				t.Errorf("body = %q, want %q", r.body, tt.want)
+			}
+			switch {
+			case tt.warning == "" && len(r.warnings) != 0:
+				t.Errorf("warnings = %v, want none", r.warnings)
+			case tt.warning != "" && (len(r.warnings) != 1 || !strings.Contains(r.warnings[0], tt.warning)):
+				t.Errorf("warnings = %v, want one containing %q", r.warnings, tt.warning)
+			}
+		})
+	}
+}
+
+func TestLargeDownloadsAreRefused(t *testing.T) {
+	s := newServer(t)
+	doc, _ := html.Parse(strings.NewReader(`<body><img src="` + s.URL + `/photo.jpg" alt="x"></body>`))
+	page, _ := url.Parse(s.URL)
+	var warnings []string
+	p := New(s.Client(), "test-agent")
+	p.MaxBytes = 1000
+	p.Warn = func(msg string) { warnings = append(warnings, msg) }
+	images, err := p.Process(context.Background(), doc, page)
+	if err != nil || len(images) != 0 || len(warnings) != 1 || !strings.Contains(warnings[0], "larger than") {
+		t.Errorf("images = %d, err = %v, warnings = %v", len(images), err, warnings)
+	}
+}
+
+func TestCancelled(t *testing.T) {
+	s := newServer(t)
+	doc, _ := html.Parse(strings.NewReader(`<body><img src="` + s.URL + `/photo.jpg"></body>`))
+	page, _ := url.Parse(s.URL)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := New(s.Client(), "test-agent").Process(ctx, doc, page); err == nil {
+		t.Error("Process succeeded after cancellation, want an error")
+	}
+}
+
+func TestDecodeRefusesLocalFiles(t *testing.T) {
+	page, _ := url.Parse("https://example.com/article")
+	p := New(http.DefaultClient, "test-agent")
+	for _, src := range []string{"file:///etc/hosts", "/etc/hosts", `C:\Windows\win.ini`} {
+		if _, err := p.Decode(context.Background(), src, page); err == nil {
+			t.Errorf("Decode(%q) succeeded", src)
+		}
+	}
+}

internal/assets/process.go

@@ -0,0 +1,187 @@
+package assets
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"image"
+	"image/color"
+	"image/draw"
+	_ "image/gif" // registers GIF decoding; Decode returns the first frame
+	"image/jpeg"
+	"image/png"
+	"sort"
+
+	"github.com/srwiley/oksvg"
+	"github.com/srwiley/rasterx"
+	xdraw "golang.org/x/image/draw"
+	_ "golang.org/x/image/webp" // registers WebP decoding
+
+	"git.monoexp.com/bajdev/rekopub/internal/device"
+)
+
+// maxPixels rejects images that would take too much memory to decode,
+// such as "decompression bombs" with a tiny file but huge dimensions.
+const maxPixels = 50_000_000
+
+// errTooSmall marks images of a couple of pixels, which are tracking pixels
+// or spacers rather than content.
+var errTooSmall = errors.New("image is a tracking pixel or spacer")
+
+// encoded is a processed image ready to go into the book.
+type encoded struct {
+	data      []byte
+	mediaType string
+	ext       string
+}
+
+// convert decodes data and returns it prepared for e-ink: transparency
+// flattened onto white, converted to grayscale, scaled down to fit the
+// profile, and encoded as JPEG (photos) or a 16-level PNG (line art).
+// WebP, GIF and SVG input are never passed through.
+func convert(data []byte, svg bool, p device.Profile) (*encoded, error) {
+	img, err := decodeAny(data, svg, p)
+	if err != nil {
+		return nil, err
+	}
+
+	gray := scaleToFit(ToGray(img), p.MaxImageWidth, p.MaxImageHeight)
+
+	var buf bytes.Buffer
+	if isLineArt(gray, p.GrayLevels) {
+		if err := png.Encode(&buf, quantize(gray, p.GrayLevels)); err != nil {
+			return nil, err
+		}
+		return &encoded{data: buf.Bytes(), mediaType: "image/png", ext: ".png"}, nil
+	}
+	if err := jpeg.Encode(&buf, gray, &jpeg.Options{Quality: p.JPEGQuality}); err != nil {
+		return nil, err
+	}
+	return &encoded{data: buf.Bytes(), mediaType: "image/jpeg", ext: ".jpg"}, nil
+}
+
+// decodeAny decodes image data in any supported format, rasterizing SVG,
+// and rejects tracking pixels and spacers.
+func decodeAny(data []byte, svg bool, p device.Profile) (image.Image, error) {
+	var img image.Image
+	var err error
+	if svg {
+		img, err = rasterizeSVG(data, p)
+	} else {
+		img, err = decode(data)
+	}
+	if err != nil {
+		return nil, err
+	}
+	if b := img.Bounds(); b.Dx() <= 2 || b.Dy() <= 2 {
+		return nil, errTooSmall
+	}
+	return img, nil
+}
+
+// decode decodes JPEG, PNG, GIF or WebP data, refusing images whose
+// dimensions are too large to decode safely.
+func decode(data []byte) (image.Image, error) {
+	cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
+	if err != nil {
+		return nil, fmt.Errorf("unsupported or damaged image: %w", err)
+	}
+	if cfg.Width*cfg.Height > maxPixels {
+		return nil, fmt.Errorf("%s image is too large (%d×%d)", format, cfg.Width, cfg.Height)
+	}
+	img, _, err := image.Decode(bytes.NewReader(data))
+	if err != nil {
+		return nil, fmt.Errorf("damaged %s image: %w", format, err)
+	}
+	return img, nil
+}
+
+// rasterizeSVG draws an SVG at up to twice its own size, bounded by the
+// profile. The e-readers don't render SVG reliably, so it becomes a bitmap.
+// oksvg never loads external resources.
+func rasterizeSVG(data []byte, p device.Profile) (img image.Image, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("damaged SVG image: %v", r)
+		}
+	}()
+	icon, err := oksvg.ReadIconStream(bytes.NewReader(data), oksvg.IgnoreErrorMode)
+	if err != nil {
+		return nil, fmt.Errorf("damaged SVG image: %w", err)
+	}
+	w, h := icon.ViewBox.W, icon.ViewBox.H
+	if w <= 0 || h <= 0 {
+		w, h = 800, 600
+	}
+	scale := min(2, float64(p.MaxImageWidth)/w, float64(p.MaxImageHeight)/h)
+	tw, th := max(1, int(w*scale)), max(1, int(h*scale))
+
+	icon.SetTarget(0, 0, float64(tw), float64(th))
+	rgba := image.NewRGBA(image.Rect(0, 0, tw, th))
+	scanner := rasterx.NewScannerGV(tw, th, rgba, rgba.Bounds())
+	icon.Draw(rasterx.NewDasher(tw, th, scanner), 1)
+	return rgba, nil
+}
+
+// ToGray composites img over white, so transparent areas don't turn black
+// on readers that ignore alpha, and converts it to 8-bit grayscale using
+// the BT.601 luma weights.
+func ToGray(img image.Image) *image.Gray {
+	b := img.Bounds()
+	flat := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
+	draw.Draw(flat, flat.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
+	draw.Draw(flat, flat.Bounds(), img, b.Min, draw.Over)
+	gray := image.NewGray(flat.Bounds())
+	draw.Draw(gray, gray.Bounds(), flat, image.Point{}, draw.Src)
+	return gray
+}
+
+// scaleToFit shrinks img to fit within maxW×maxH, keeping its aspect
+// ratio. Images are never enlarged.
+func scaleToFit(img *image.Gray, maxW, maxH int) *image.Gray {
+	w, h := img.Bounds().Dx(), img.Bounds().Dy()
+	scale := min(1, float64(maxW)/float64(w), float64(maxH)/float64(h))
+	if scale >= 1 {
+		return img
+	}
+	dst := image.NewGray(image.Rect(0, 0, max(1, int(float64(w)*scale+0.5)), max(1, int(float64(h)*scale+0.5))))
+	xdraw.CatmullRom.Scale(dst, dst.Bounds(), img, img.Bounds(), xdraw.Src, nil)
+	return dst
+}
+
+// lineArtCoverage is the share of pixels that must fall on the most common
+// gray levels for an image to count as line art (diagrams, charts,
+// screenshots) rather than a photo.
+const lineArtCoverage = 0.9
+
+// isLineArt reports whether most of img's pixels use just a few distinct
+// gray values, as diagrams and screenshots do. Photos spread across many.
+func isLineArt(img *image.Gray, levels int) bool {
+	var hist [256]int
+	for _, v := range img.Pix {
+		hist[v]++
+	}
+	counts := hist[:]
+	sort.Sort(sort.Reverse(sort.IntSlice(counts)))
+	top := 0
+	for _, c := range counts[:levels] {
+		top += c
+	}
+	return float64(top) >= lineArtCoverage*float64(len(img.Pix))
+}
+
+// quantize maps img onto levels evenly spaced grays, black to white. With
+// 16 levels this matches the panels exactly and the PNG encoder writes
+// compact 4-bit pixels.
+func quantize(img *image.Gray, levels int) *image.Paletted {
+	pal := make(color.Palette, levels)
+	step := 255 / (levels - 1)
+	for i := range pal {
+		pal[i] = color.Gray{Y: uint8(i * step)}
+	}
+	out := image.NewPaletted(img.Bounds(), pal)
+	for i, v := range img.Pix {
+		out.Pix[i] = uint8((int(v) + step/2) / step)
+	}
+	return out
+}

internal/build/build.go

@@ -0,0 +1,172 @@
+// Package build packages an extracted article as an EPUB.
+package build
+
+import (
+	"bytes"
+	_ "embed"
+	"encoding/base64"
+	"fmt"
+	"html/template"
+	"strings"
+	"time"
+
+	epub "github.com/go-shiori/go-epub"
+	"github.com/google/uuid"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+
+	"git.monoexp.com/bajdev/rekopub/internal/assets"
+	"git.monoexp.com/bajdev/rekopub/internal/clean"
+	"git.monoexp.com/bajdev/rekopub/internal/extract"
+)
+
+//go:embed chapter.tmpl
+var chapterTmpl string
+
+//go:embed style.css
+var styleCSS []byte
+
+var chapter = template.Must(template.New("chapter").Parse(chapterTmpl))
+
+// DisplayDate formats a publication date as shown in the byline and on the
+// cover, or returns "" for the zero time.
+func DisplayDate(t time.Time) string {
+	if t.IsZero() {
+		return ""
+	}
+	return t.Format("2 January 2006")
+}
+
+// Build returns the EPUB for a, whose cleaned content is body (from
+// clean.Content) with images already processed into images. cover is a
+// JPEG cover image, or nil for none.
+func Build(a *extract.Article, body *html.Node, images []assets.Image, cover []byte) ([]byte, error) {
+	// go-epub stages files in a temporary folder that it removes after each
+	// write. Don't switch it to epub.MemoryFS: on Windows that loses every
+	// file except mimetype, producing an empty book.
+	e, err := epub.NewEpub(a.Title)
+	if err != nil {
+		return nil, err
+	}
+	// Derive the identifier from the URL, so rebuilding the same page gives
+	// the same book identity and readers treat it as the same book.
+	id := "urn:uuid:" + uuid.NewSHA1(uuid.NameSpaceURL, []byte(a.URL)).String()
+	e.SetIdentifier(id)
+	e.SetLang(a.Language)
+	if a.Author != "" {
+		e.SetAuthor(a.Author)
+	}
+	if a.Description != "" {
+		e.SetDescription(a.Description)
+	}
+
+	css, err := e.AddCSS("data:text/css;base64,"+base64.StdEncoding.EncodeToString(styleCSS), "style.css")
+	if err != nil {
+		return nil, fmt.Errorf("adding stylesheet: %w", err)
+	}
+	if cover != nil {
+		p, err := e.AddImage("data:image/jpeg;base64,"+base64.StdEncoding.EncodeToString(cover), "cover.jpg")
+		if err != nil {
+			return nil, fmt.Errorf("adding cover: %w", err)
+		}
+		if err := e.SetCover(p, ""); err != nil {
+			return nil, fmt.Errorf("adding cover: %w", err)
+		}
+	}
+	if err := addImages(e, body, images); err != nil {
+		return nil, err
+	}
+	headings := anchorHeadings(body)
+	chapter, err := chapterBody(a, body)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := e.AddSection(chapter, a.Title, chapterFile, css); err != nil {
+		return nil, err
+	}
+
+	var raw bytes.Buffer
+	if _, err := e.WriteTo(&raw); err != nil {
+		return nil, err
+	}
+	return finish(raw.Bytes(), id, a, headings)
+}
+
+// addImages stores each image in the book and points the <img> elements,
+// whose src is the image's filename, at the stored file.
+func addImages(e *epub.Epub, body *html.Node, images []assets.Image) error {
+	paths := map[string]string{}
+	for _, img := range images {
+		src := "data:" + img.MediaType + ";base64," + base64.StdEncoding.EncodeToString(img.Data)
+		p, err := e.AddImage(src, img.Filename)
+		if err != nil {
+			return fmt.Errorf("adding image %s: %w", img.Filename, err)
+		}
+		paths[img.Filename] = p
+	}
+	// Only images processed by the assets package may be in the book; any
+	// other <img> would point at a remote resource, so it is removed.
+	var unknown []*html.Node
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.DataAtom == atom.Img {
+			known := false
+			for i, a := range n.Attr {
+				if p, ok := paths[a.Val]; ok && a.Key == "src" {
+					n.Attr[i].Val, known = p, true
+				}
+			}
+			if !known {
+				unknown = append(unknown, n)
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(body)
+	for _, n := range unknown {
+		n.Parent.RemoveChild(n)
+	}
+	return nil
+}
+
+// chapterBody renders the chapter: a header with the title, byline and
+// source link, followed by the article content.
+func chapterBody(a *extract.Article, body *html.Node) (string, error) {
+	content, err := clean.Render(body)
+	if err != nil {
+		return "", err
+	}
+	var b strings.Builder
+	err = chapter.Execute(&b, struct {
+		Lang, Title, Byline, URL string
+		Content                  template.HTML
+	}{
+		Lang:    a.Language,
+		Title:   a.Title,
+		Byline:  byline(a),
+		URL:     a.URL,
+		Content: template.HTML(content), // sanitized and checked by clean.Content
+	})
+	if err != nil {
+		return "", fmt.Errorf("rendering chapter: %w", err)
+	}
+	return b.String(), nil
+}
+
+// byline joins the author, site name and publication date, skipping any
+// that are unknown.
+func byline(a *extract.Article) string {
+	var parts []string
+	if a.Author != "" {
+		parts = append(parts, a.Author)
+	}
+	if a.SiteName != "" {
+		parts = append(parts, a.SiteName)
+	}
+	if d := DisplayDate(a.Published); d != "" {
+		parts = append(parts, d)
+	}
+	return strings.Join(parts, " · ")
+}

internal/build/build_test.go

@@ -0,0 +1,302 @@
+package build
+
+import (
+	"bytes"
+	"image"
+	imgjpeg "image/jpeg"
+	imgpng "image/png"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/net/html"
+
+	"git.monoexp.com/bajdev/rekopub/internal/assets"
+	"git.monoexp.com/bajdev/rekopub/internal/clean"
+	"git.monoexp.com/bajdev/rekopub/internal/epubtest"
+	"git.monoexp.com/bajdev/rekopub/internal/extract"
+)
+
+func testArticle(t *testing.T, body string) *extract.Article {
+	t.Helper()
+	doc, err := html.Parse(strings.NewReader("<body>" + body + "</body>"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &extract.Article{
+		URL:         "https://example.com/posts/1?a=1&b=2",
+		Title:       "Tom & Jerry <3",
+		Author:      "Ada Lovelace",
+		SiteName:    "Example News",
+		Description: "A test article.",
+		Language:    "en",
+		Published:   time.Date(2025, 3, 14, 9, 30, 0, 0, time.UTC),
+		Content:     doc.FirstChild.LastChild, // html > body
+	}
+}
+
+func buildBytes(t *testing.T, a *extract.Article) []byte {
+	t.Helper()
+	base, err := url.Parse(a.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, err := clean.Content(a.Content, base)
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := Build(a, body, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return data
+}
+
+func TestBuild(t *testing.T) {
+	a := testArticle(t, `<h1>Section</h1><p>First &amp; <em>second</em>.<br>Line two.</p><hr><ul><li>One</li></ul><script>evil()</script>`)
+	data := buildBytes(t, a)
+	book := epubtest.Open(t, data)
+	book.AssertWellFormed(t)
+
+	opf := book.File(t, ".opf")
+	for _, want := range []string{
+		"<dc:title>Tom &amp; Jerry &lt;3</dc:title>",
+		"<dc:language>en</dc:language>",
+		"Ada Lovelace",
+		"A test article.",
+		"urn:uuid:",
+	} {
+		if !strings.Contains(opf, want) {
+			t.Errorf("OPF is missing %q:\n%s", want, opf)
+		}
+	}
+
+	chapter := book.File(t, "chapter.xhtml")
+	for _, want := range []string{
+		`<div lang="en" xml:lang="en">`,
+		`<link rel="stylesheet" type="text/css" href="../css/style.css"`,
+		"<h1>Tom &amp; Jerry &lt;3</h1>",
+		`<p class="byline">Ada Lovelace · Example News · 14 March 2025</p>`,
+		`<a href="https://example.com/posts/1?a=1&amp;b=2">`,
+		`<h2 id="sec-1">Section</h2>`, // shifted down from <h1>, anchored for the contents
+		"First &amp; <em>second</em>.<br/>Line two.",
+		"<hr/>",
+		"<li>One</li>",
+	} {
+		if !strings.Contains(chapter, want) {
+			t.Errorf("chapter is missing %q:\n%s", want, chapter)
+		}
+	}
+
+	if strings.Contains(chapter, "evil()") {
+		t.Errorf("chapter contains a script:\n%s", chapter)
+	}
+	if css := book.File(t, "style.css"); !strings.Contains(css, "border-left: 3px solid #000") {
+		t.Errorf("stylesheet missing or unexpected:\n%s", css)
+	}
+
+	path := filepath.Join(t.TempDir(), "book.epub")
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	epubtest.EPUBCheck(t, path)
+}
+
+func TestBuildIdentifierIsStable(t *testing.T) {
+	id := regexp.MustCompile(`urn:uuid:[0-9a-f-]{36}`)
+	a := testArticle(t, "<p>text</p>")
+	first := id.FindString(epubtest.Open(t, buildBytes(t, a)).File(t, ".opf"))
+	second := id.FindString(epubtest.Open(t, buildBytes(t, a)).File(t, ".opf"))
+	if first == "" || first != second {
+		t.Errorf("identifiers %q and %q, want the same non-empty identifier", first, second)
+	}
+
+	a.URL = "https://example.com/other"
+	if other := id.FindString(epubtest.Open(t, buildBytes(t, a)).File(t, ".opf")); other == first {
+		t.Errorf("different URLs gave the same identifier %q", other)
+	}
+}
+
+func TestByline(t *testing.T) {
+	date := time.Date(2025, 1, 2, 0, 0, 0, 0, time.UTC)
+	tests := []struct {
+		author, site string
+		date         time.Time
+		want         string
+	}{
+		{"A", "S", date, "A · S · 2 January 2025"},
+		{"", "S", date, "S · 2 January 2025"},
+		{"A", "", time.Time{}, "A"},
+		{"", "", time.Time{}, ""},
+	}
+	for _, tt := range tests {
+		got := byline(&extract.Article{Author: tt.author, SiteName: tt.site, Published: tt.date})
+		if got != tt.want {
+			t.Errorf("byline(%q, %q, %v) = %q, want %q", tt.author, tt.site, tt.date, got, tt.want)
+		}
+	}
+}
+
+func TestChapterWithoutByline(t *testing.T) {
+	a := testArticle(t, "<p>text</p>")
+	a.Author, a.SiteName, a.Published = "", "", time.Time{}
+	chapter := epubtest.Open(t, buildBytes(t, a)).File(t, "chapter.xhtml")
+	if strings.Contains(chapter, "byline") {
+		t.Errorf("chapter has a byline paragraph:\n%s", chapter)
+	}
+}
+
+func TestBuildWithImages(t *testing.T) {
+	var png bytes.Buffer
+	img := image.NewGray(image.Rect(0, 0, 40, 30))
+	if err := imgpng.Encode(&png, img); err != nil {
+		t.Fatal(err)
+	}
+	a := testArticle(t, "")
+	body, err := clean.Content(a.Content, mustURL(t, a.URL))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// As left by the assets package: src is the image's filename. The second
+	// image wasn't processed and must not end up in the book.
+	frag, err := html.ParseFragment(strings.NewReader(`<figure><img src="abc123.png" alt="A chart"/></figure><p>x <img src="https://remote.example/a.png" alt=""/></p>`), body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, n := range frag {
+		body.AppendChild(n)
+	}
+
+	data, err := Build(a, body, []assets.Image{{Filename: "abc123.png", MediaType: "image/png", Data: png.Bytes()}}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	book := epubtest.Open(t, data)
+	book.AssertWellFormed(t)
+	if got := book.File(t, "images/abc123.png"); got != png.String() {
+		t.Errorf("stored image differs from the original")
+	}
+	chapter := book.File(t, "chapter.xhtml")
+	if !strings.Contains(chapter, `<img src="../images/abc123.png" alt="A chart"/>`) {
+		t.Errorf("chapter doesn't reference the stored image:\n%s", chapter)
+	}
+	if strings.Contains(chapter, "remote.example") {
+		t.Errorf("chapter references a remote image:\n%s", chapter)
+	}
+	if opf := book.File(t, ".opf"); !strings.Contains(opf, `href="images/abc123.png"`) || !strings.Contains(opf, `media-type="image/png"`) {
+		t.Errorf("OPF manifest is missing the image:\n%s", opf)
+	}
+
+	path := filepath.Join(t.TempDir(), "book.epub")
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	epubtest.EPUBCheck(t, path)
+}
+
+func mustURL(t *testing.T, raw string) *url.URL {
+	t.Helper()
+	u, err := url.Parse(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return u
+}
+
+func TestBuildMetadataCoverAndContents(t *testing.T) {
+	a := testArticle(t, `<p>Intro.</p><h2>First <em>part</em></h2><p>a</p><h3>Detail</h3><p>b</p><h2>Second part</h2><p>c</p><h4>Too deep</h4>`)
+	var jpg bytes.Buffer
+	if err := imgjpeg.Encode(&jpg, image.NewGray(image.Rect(0, 0, 30, 40)), nil); err != nil {
+		t.Fatal(err)
+	}
+	body, err := clean.Content(a.Content, mustURL(t, a.URL))
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := Build(a, body, nil, jpg.Bytes())
+	if err != nil {
+		t.Fatal(err)
+	}
+	book := epubtest.Open(t, data)
+	book.AssertWellFormed(t)
+
+	opf := book.File(t, ".opf")
+	for _, want := range []string{
+		"<dc:publisher>Example News</dc:publisher>",
+		"<dc:source>https://example.com/posts/1?a=1&amp;b=2</dc:source>",
+		"<dc:date>2025-03-14</dc:date>",
+		`href="images/cover.jpg" media-type="image/jpeg" properties="cover-image"`,
+		`<itemref idref="cover.xhtml">`,
+	} {
+		if !strings.Contains(opf, want) {
+			t.Errorf("OPF is missing %q:\n%s", want, opf)
+		}
+	}
+	if got := book.File(t, "images/cover.jpg"); got != jpg.String() {
+		t.Errorf("stored cover differs from the original")
+	}
+
+	chapter := book.File(t, "chapter.xhtml")
+	for _, want := range []string{`<h2 id="sec-1">First <em>part</em></h2>`, `<h3 id="sec-2">Detail</h3>`, `<h2 id="sec-3">Second part</h2>`, "<h4>Too deep</h4>"} {
+		if !strings.Contains(chapter, want) {
+			t.Errorf("chapter is missing %q", want)
+		}
+	}
+
+	nav := book.File(t, "nav.xhtml")
+	navWant := `<li><a href="xhtml/chapter.xhtml">Tom &amp; Jerry &lt;3</a>
+        <ol>
+          <li><a href="xhtml/chapter.xhtml#sec-1">First part</a>
+            <ol>
+              <li><a href="xhtml/chapter.xhtml#sec-2">Detail</a></li>
+            </ol>
+          </li>
+          <li><a href="xhtml/chapter.xhtml#sec-3">Second part</a></li>
+        </ol>
+        </li>`
+	squash := func(s string) string { return strings.Join(strings.Fields(s), " ") }
+	if !strings.Contains(squash(nav), squash(navWant)) {
+		t.Errorf("nav document:\n%s\nwant it to contain:\n%s", nav, navWant)
+	}
+
+	ncx := book.File(t, "toc.ncx")
+	id := regexp.MustCompile(`urn:uuid:[0-9a-f-]{36}`).FindString(opf)
+	for _, want := range []string{
+		`<meta name="dtb:uid" content="` + id + `"/>`,
+		`<meta name="dtb:depth" content="3"/>`,
+		`<navPoint id="navpoint-1" playOrder="1">`,
+		`<navPoint id="navpoint-4" playOrder="4">`,
+		`<content src="xhtml/chapter.xhtml#sec-2"/>`,
+		`<navLabel><text>Second part</text></navLabel>`,
+	} {
+		if !strings.Contains(ncx, want) {
+			t.Errorf("NCX is missing %q:\n%s", want, ncx)
+		}
+	}
+
+	path := filepath.Join(t.TempDir(), "book.epub")
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	epubtest.EPUBCheck(t, path)
+}
+
+func TestBuildWithoutHeadingsOrDate(t *testing.T) {
+	a := testArticle(t, "<p>Just text.</p>")
+	a.SiteName, a.Published = "", time.Time{}
+	book := epubtest.Open(t, buildBytes(t, a))
+	opf := book.File(t, ".opf")
+	if strings.Contains(opf, "dc:publisher") || strings.Contains(opf, "dc:date") {
+		t.Errorf("OPF has empty publisher or date:\n%s", opf)
+	}
+	if !strings.Contains(book.File(t, "nav.xhtml"), `<li><a href="xhtml/chapter.xhtml">Tom &amp; Jerry &lt;3</a></li>`) {
+		t.Errorf("nav should list only the article:\n%s", book.File(t, "nav.xhtml"))
+	}
+	if !strings.Contains(book.File(t, "toc.ncx"), `<meta name="dtb:depth" content="1"/>`) {
+		t.Errorf("NCX depth should be 1")
+	}
+}

internal/build/chapter.tmpl

@@ -0,0 +1,8 @@
+<div lang="{{.Lang}}" xml:lang="{{.Lang}}">
+<h1>{{.Title}}</h1>
+{{- if .Byline}}
+<p class="byline">{{.Byline}}</p>
+{{- end}}
+<p class="source"><a href="{{.URL}}">{{.URL}}</a></p>
+{{.Content}}
+</div>

internal/build/finish.go

@@ -0,0 +1,273 @@
+package build
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+
+	"git.monoexp.com/bajdev/rekopub/internal/extract"
+)
+
+// chapterFile is the chapter's filename; go-epub stores it under xhtml/.
+const chapterFile = "chapter.xhtml"
+
+// Paths go-epub uses inside the book. finish fails if any is missing, so a
+// change in go-epub's layout shows up in tests rather than in broken books.
+const (
+	opfPath = "EPUB/package.opf"
+	navPath = "EPUB/nav.xhtml"
+	ncxPath = "EPUB/toc.ncx"
+)
+
+// heading is a section heading listed in the table of contents.
+type heading struct {
+	level int // 2 or 3
+	text  string
+	id    string
+}
+
+// anchorHeadings gives every <h2> and <h3> in body an id, so the table of
+// contents can link to it, and returns them in order. The sanitizer removed
+// the page's own ids, so these can't clash.
+func anchorHeadings(body *html.Node) []heading {
+	var hs []heading
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode && (n.DataAtom == atom.H2 || n.DataAtom == atom.H3) {
+			if text := strings.Join(strings.Fields(textOf(n)), " "); text != "" {
+				id := "sec-" + strconv.Itoa(len(hs)+1)
+				n.Attr = append(n.Attr, html.Attribute{Key: "id", Val: id})
+				level := 2
+				if n.DataAtom == atom.H3 {
+					level = 3
+				}
+				hs = append(hs, heading{level: level, text: text, id: id})
+			}
+			return
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(body)
+	return hs
+}
+
+// finish rewrites the files go-epub gets wrong or can't express:
+//
+//   - the OPF gains dc:publisher (the site), dc:source (the URL) and
+//     dc:date (publication date);
+//   - the EPUB 3 nav document and the NCX list the article's sections.
+//     go-epub can only list whole files, and its NCX has no dtb:uid, which
+//     Kobo's RMSDK renderer reads for plain EPUBs.
+func finish(book []byte, id string, a *extract.Article, headings []heading) ([]byte, error) {
+	return rewriteZip(book, map[string]func([]byte) []byte{
+		opfPath: func(opf []byte) []byte { return addMetadata(opf, a) },
+		navPath: func([]byte) []byte { return navDocument(a, headings) },
+		ncxPath: func([]byte) []byte { return ncxDocument(id, a, headings) },
+	})
+}
+
+// rewriteZip copies the zip archive data, replacing the named entries with
+// the output of their edit functions. Other entries, including the
+// uncompressed mimetype that must come first, are copied byte for byte.
+func rewriteZip(data []byte, edits map[string]func([]byte) []byte) ([]byte, error) {
+	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		return nil, err
+	}
+	var out bytes.Buffer
+	zw := zip.NewWriter(&out)
+	done := map[string]bool{}
+	for _, f := range zr.File {
+		edit, ok := edits[f.Name]
+		if !ok {
+			if err := zw.Copy(f); err != nil {
+				return nil, err
+			}
+			continue
+		}
+		rc, err := f.Open()
+		if err != nil {
+			return nil, err
+		}
+		old, err := io.ReadAll(rc)
+		rc.Close()
+		if err != nil {
+			return nil, err
+		}
+		w, err := zw.CreateHeader(&zip.FileHeader{Name: f.Name, Method: zip.Deflate, Modified: f.Modified})
+		if err != nil {
+			return nil, err
+		}
+		if _, err := w.Write(edit(old)); err != nil {
+			return nil, err
+		}
+		done[f.Name] = true
+	}
+	for name := range edits {
+		if !done[name] {
+			return nil, fmt.Errorf("book is missing %s", name)
+		}
+	}
+	if err := zw.Close(); err != nil {
+		return nil, err
+	}
+	return out.Bytes(), nil
+}
+
+// addMetadata inserts publisher, source and date elements into the OPF
+// metadata.
+func addMetadata(opf []byte, a *extract.Article) []byte {
+	var extra strings.Builder
+	element := func(name, value string) {
+		if value != "" {
+			fmt.Fprintf(&extra, "    <%s>%s</%s>\n", name, esc(value), name)
+		}
+	}
+	element("dc:publisher", a.SiteName)
+	element("dc:source", a.URL)
+	if !a.Published.IsZero() {
+		element("dc:date", a.Published.UTC().Format("2006-01-02"))
+	}
+	s := string(opf)
+	marker := `<meta property="dcterms:modified">`
+	i := strings.Index(s, marker)
+	if i < 0 {
+		i = strings.Index(s, "</metadata>")
+	}
+	if i < 0 {
+		return opf
+	}
+	// Insert at the start of the marker's line, keeping the indentation.
+	line := strings.LastIndex(s[:i], "\n") + 1
+	return []byte(s[:line] + extra.String() + s[line:])
+}
+
+// tocEntry is a node in the table of contents tree.
+type tocEntry struct {
+	text, href string
+	children   []*tocEntry
+}
+
+// toc returns the table of contents: the article, with its <h2> sections
+// and their <h3> subsections beneath it. Paths are relative to the EPUB
+// folder, where the nav document and NCX live.
+func toc(a *extract.Article, headings []heading) *tocEntry {
+	href := "xhtml/" + chapterFile
+	root := &tocEntry{text: a.Title, href: href}
+	var section *tocEntry
+	for _, h := range headings {
+		e := &tocEntry{text: h.text, href: href + "#" + h.id}
+		if h.level == 3 && section != nil {
+			section.children = append(section.children, e)
+			continue
+		}
+		root.children = append(root.children, e)
+		if h.level == 2 {
+			section = e
+		}
+	}
+	return root
+}
+
+func navDocument(a *extract.Article, headings []heading) []byte {
+	var b strings.Builder
+	var list func(es []*tocEntry, indent string)
+	list = func(es []*tocEntry, indent string) {
+		b.WriteString(indent + "<ol>\n")
+		for _, e := range es {
+			fmt.Fprintf(&b, "%s  <li><a href=\"%s\">%s</a>", indent, esc(e.href), esc(e.text))
+			if len(e.children) > 0 { // an empty <ol> is invalid
+				b.WriteString("\n")
+				list(e.children, indent+"    ")
+				b.WriteString(indent + "  ")
+			}
+			b.WriteString("</li>\n")
+		}
+		b.WriteString(indent + "</ol>\n")
+	}
+	lang := esc(a.Language)
+	fmt.Fprintf(&b, `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops" lang="%s" xml:lang="%s">
+  <head>
+    <title>%s</title>
+  </head>
+  <body>
+    <nav epub:type="toc" id="toc">
+      <h1>Contents</h1>
+`, lang, lang, esc(a.Title))
+	list([]*tocEntry{toc(a, headings)}, "      ")
+	b.WriteString("    </nav>\n  </body>\n</html>\n")
+	return []byte(b.String())
+}
+
+func ncxDocument(id string, a *extract.Article, headings []heading) []byte {
+	root := toc(a, headings)
+	depth := 1
+	for _, e := range root.children {
+		depth = max(depth, 2)
+		if len(e.children) > 0 {
+			depth = 3
+		}
+	}
+	var b strings.Builder
+	fmt.Fprintf(&b, `<?xml version="1.0" encoding="UTF-8"?>
+<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1" xml:lang="%s">
+  <head>
+    <meta name="dtb:uid" content="%s"/>
+    <meta name="dtb:depth" content="%d"/>
+    <meta name="dtb:totalPageCount" content="0"/>
+    <meta name="dtb:maxPageNumber" content="0"/>
+  </head>
+  <docTitle>
+    <text>%s</text>
+  </docTitle>
+  <navMap>
+`, esc(a.Language), esc(id), depth, esc(a.Title))
+	order := 0
+	var point func(e *tocEntry, indent string)
+	point = func(e *tocEntry, indent string) {
+		order++
+		fmt.Fprintf(&b, "%s<navPoint id=\"navpoint-%d\" playOrder=\"%d\">\n", indent, order, order)
+		fmt.Fprintf(&b, "%s  <navLabel><text>%s</text></navLabel>\n", indent, esc(e.text))
+		fmt.Fprintf(&b, "%s  <content src=\"%s\"/>\n", indent, esc(e.href))
+		for _, c := range e.children {
+			point(c, indent+"  ")
+		}
+		fmt.Fprintf(&b, "%s</navPoint>\n", indent)
+	}
+	point(root, "    ")
+	b.WriteString("  </navMap>\n</ncx>\n")
+	return []byte(b.String())
+}
+
+// esc escapes s for XML text and attribute values.
+func esc(s string) string {
+	var b strings.Builder
+	xml.EscapeText(&b, []byte(s))
+	return b.String()
+}
+
+func textOf(n *html.Node) string {
+	var b strings.Builder
+	var rec func(*html.Node)
+	rec = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			b.WriteString(n.Data)
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			rec(c)
+		}
+	}
+	rec(n)
+	return b.String()
+}

internal/build/style.css

@@ -0,0 +1,138 @@
+/*
+ * Stylesheet for grayscale e-ink readers (reMarkable 2, Kobo Clara HD).
+ *
+ * - Pure black only: no colours, no gray text, no background shading.
+ *   Gray text looks faint on e-ink and light fills turn into dithered noise.
+ * - Font family, size, line spacing and margins are left to the reader's
+ *   own settings; sizes below are relative (em) only.
+ * - Simple CSS only: Kobo's RMSDK renderer and the reMarkable ignore or
+ *   mis-render flexbox, grid, calc() and custom properties.
+ */
+
+body {
+  font-family: serif;
+  color: #000;
+}
+
+h1, h2, h3, h4, h5, h6 {
+  color: #000;
+  font-weight: bold;
+  line-height: 1.2;
+  text-align: left;
+  page-break-after: avoid;
+}
+
+h1 { font-size: 1.6em; margin: 0 0 0.4em 0; }
+h2 { font-size: 1.3em; margin: 1.4em 0 0.5em 0; }
+h3 { font-size: 1.15em; margin: 1.2em 0 0.4em 0; }
+h4, h5, h6 { font-size: 1em; margin: 1em 0 0.3em 0; }
+
+.byline {
+  font-style: italic;
+  margin: 0;
+  text-indent: 0;
+}
+
+.source {
+  font-size: 0.85em;
+  margin: 0.2em 0 1.8em 0;
+  text-indent: 0;
+  word-wrap: break-word;
+}
+
+/* Links are shown by an underline, not by colour. */
+a {
+  color: #000;
+  text-decoration: underline;
+}
+
+blockquote {
+  margin: 1em 0 1em 0.4em;
+  padding: 0 0 0 0.8em;
+  border-left: 3px solid #000;
+}
+
+/* E-ink readers can't scroll sideways, so long lines must wrap. */
+pre {
+  font-family: monospace;
+  font-size: 0.85em;
+  white-space: pre-wrap;
+  word-wrap: break-word;
+  margin: 1em 0;
+  padding: 0.4em 0 0.4em 0.6em;
+  border-left: 1px solid #000;
+}
+
+code {
+  font-family: monospace;
+}
+
+ul, ol {
+  margin: 0.8em 0;
+  padding-left: 1.6em;
+}
+
+dt {
+  font-weight: bold;
+}
+
+dd {
+  margin: 0 0 0.6em 1.4em;
+}
+
+table {
+  border-collapse: collapse;
+  margin: 1em 0;
+}
+
+th, td {
+  border: 1px solid #000;
+  padding: 0.2em 0.4em;
+  text-align: left;
+  vertical-align: top;
+}
+
+th {
+  font-weight: bold;
+}
+
+hr {
+  border: 0;
+  border-top: 1px solid #000;
+  margin: 1.5em 0;
+}
+
+img {
+  max-width: 100%;
+  height: auto;
+}
+
+/* Images are already grayscale and sized for the screen (see internal/assets). */
+figure {
+  margin: 1em 0;
+  text-align: center;
+  page-break-inside: avoid;
+}
+
+figure img {
+  display: block;
+  margin: 0 auto;
+}
+
+/* Stands in for an image that couldn't be downloaded. */
+.missing-image {
+  font-style: italic;
+}
+
+figcaption {
+  text-align: left;
+  font-style: italic;
+  font-size: 0.9em;
+  margin-top: 0.3em;
+}
+
+aside {
+  margin: 1em 0;
+  padding-left: 0.8em;
+  border-left: 1px solid #000;
+}

internal/clean/clean.go

@@ -0,0 +1,264 @@
+// Package clean turns extracted article markup into safe, well-formed XHTML
+// for an EPUB chapter.
+//
+// Prepare runs on the whole page before extraction; Content runs on the
+// extracted article, and Render serializes the result once images have been
+// processed.
+package clean
+
+import (
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/microcosm-cc/bluemonday"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// policy is the allow-list from PLAN.md §5.3. Anything not listed is
+// removed; the text inside removed elements is kept, except for elements
+// such as script and style whose content is dropped too.
+var policy = func() *bluemonday.Policy {
+	p := bluemonday.NewPolicy()
+	p.AllowElements(
+		"p", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "pre", "code",
+		"em", "strong", "b", "i", "u", "s", "sub", "sup", "small", "abbr", "cite", "q",
+		"ul", "ol", "li", "dl", "dt", "dd",
+		"table", "thead", "tbody", "tfoot", "tr", "th", "td",
+		"hr", "br", "span", "div", "aside", "figure", "figcaption",
+	)
+	p.AllowAttrs("href").OnElements("a")
+	p.AllowURLSchemes("http", "https", "mailto")
+	p.AllowRelativeURLs(true) // made absolute by resolveLinks
+	p.RequireParseableURLs(true)
+	p.AllowAttrs("src", "alt").OnElements("img")
+	p.AllowDataURIImages()
+	p.AllowAttrs("title").OnElements("abbr")
+	p.AllowAttrs("colspan", "rowspan").Matching(regexp.MustCompile(`^[1-9][0-9]{0,2}$`)).OnElements("th", "td")
+	p.AllowAttrs("lang").Matching(regexp.MustCompile(`^[a-zA-Z]{2,3}(-[a-zA-Z0-9]{1,8})*$`)).Globally()
+	p.AllowAttrs("dir").Matching(regexp.MustCompile(`^(ltr|rtl|auto)$`)).Globally()
+	return p
+}()
+
+// Content sanitizes the children of n, the extracted article, and returns
+// them under a new <body> node. Relative links are resolved against base,
+// headings are shifted so the article's top level is <h2> below the chapter
+// title, and standalone images are wrapped in <figure>s with their
+// captions.
+func Content(n *html.Node, base *url.URL) (*html.Node, error) {
+	restoreImageSources(n)
+	var raw strings.Builder
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if err := html.Render(&raw, c); err != nil {
+			return nil, err
+		}
+	}
+	safe := policy.Sanitize(raw.String())
+
+	// Re-parse the sanitized markup so the HTML parser repairs any invalid
+	// nesting, then edit the tree.
+	body := &html.Node{Type: html.ElementNode, Data: "body", DataAtom: atom.Body}
+	nodes, err := html.ParseFragment(strings.NewReader(safe), body)
+	if err != nil {
+		return nil, err
+	}
+	for _, c := range nodes {
+		body.AppendChild(c)
+	}
+	resolveLinks(body, base)
+	restoreCodeBlocks(body)
+	buildFigures(body)
+	shiftHeadings(body)
+	removeEmpty(body)
+	return body, nil
+}
+
+// Render serializes the children of body as an XHTML fragment. It fails
+// rather than return markup that isn't well-formed, because e-readers
+// refuse to open such books.
+func Render(body *html.Node) (string, error) {
+	var out strings.Builder
+	for c := body.FirstChild; c != nil; c = c.NextSibling {
+		if err := html.Render(&out, c); err != nil {
+			return "", err
+		}
+	}
+	if err := WellFormed("<div>" + out.String() + "</div>"); err != nil {
+		return "", fmt.Errorf("cleaned content is not well-formed XHTML: %w", err)
+	}
+	return out.String(), nil
+}
+
+// mangledScheme matches a mailto: or tel: link that go-trafilatura has
+// wrongly treated as relative and joined onto the page path, as in
+// "https://example.com/post/mailto:a@example.com".
+var mangledScheme = regexp.MustCompile(`/((?:mailto|tel):.*)$`)
+
+// resolveLinks makes every link href and image src absolute.
+func resolveLinks(n *html.Node, base *url.URL) {
+	walk(n, func(n *html.Node) {
+		key := "href"
+		switch n.DataAtom {
+		case atom.A:
+		case atom.Img:
+			key = "src"
+		default:
+			return
+		}
+		for i, a := range n.Attr {
+			if a.Key != key {
+				continue
+			}
+			href := a.Val
+			if m := mangledScheme.FindStringSubmatch(href); m != nil && key == "href" {
+				href = m[1]
+			}
+			if u, err := base.Parse(href); err == nil {
+				n.Attr[i].Val = u.String()
+			}
+		}
+	})
+}
+
+// blockParents are elements in which a <pre> may appear.
+var blockParents = map[atom.Atom]bool{
+	atom.Body: true, atom.Div: true, atom.Blockquote: true, atom.Li: true, atom.Dd: true,
+	atom.Td: true, atom.Th: true, atom.Figure: true, atom.Aside: true,
+}
+
+// restoreCodeBlocks turns go-trafilatura's block-level <code> elements back
+// into <pre><code>, so code keeps its line breaks and indentation. A <code>
+// counts as a block when it sits directly in a block container and holds
+// a line break or another <code>; inline code is left alone.
+func restoreCodeBlocks(n *html.Node) {
+	var blocks []*html.Node
+	walk(n, func(n *html.Node) {
+		if n.DataAtom == atom.Code && n.Parent != nil && blockParents[n.Parent.DataAtom] &&
+			(strings.Contains(text(n), "\n") || contains(n, atom.Code)) {
+			blocks = append(blocks, n)
+		}
+	})
+	for _, c := range blocks {
+		c.DataAtom, c.Data = atom.Pre, "pre"
+		// Unwrap nested <code> elements, then put a single one inside.
+		inner := &html.Node{Type: html.ElementNode, Data: "code", DataAtom: atom.Code}
+		for child := c.FirstChild; child != nil; child = c.FirstChild {
+			c.RemoveChild(child)
+			if child.DataAtom == atom.Code {
+				for gc := child.FirstChild; gc != nil; gc = child.FirstChild {
+					child.RemoveChild(gc)
+					inner.AppendChild(gc)
+				}
+				continue
+			}
+			inner.AppendChild(child)
+		}
+		c.AppendChild(inner)
+	}
+}
+
+var headingLevels = map[atom.Atom]int{atom.H1: 1, atom.H2: 2, atom.H3: 3, atom.H4: 4, atom.H5: 5, atom.H6: 6}
+var headingAtoms = []atom.Atom{0, atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6}
+
+// shiftHeadings moves all headings down by the same amount so that none is
+// an <h1>, which is reserved for the chapter title. Relative levels are
+// kept; anything pushed past <h6> stays at <h6>.
+func shiftHeadings(n *html.Node) {
+	top := 7
+	walk(n, func(n *html.Node) {
+		if level, ok := headingLevels[n.DataAtom]; ok {
+			top = min(top, level)
+		}
+	})
+	shift := 2 - top
+	if shift <= 0 {
+		return
+	}
+	walk(n, func(n *html.Node) {
+		if level, ok := headingLevels[n.DataAtom]; ok {
+			a := headingAtoms[min(level+shift, 6)]
+			n.DataAtom, n.Data = a, a.String()
+		}
+	})
+}
+
+// removeEmpty drops images left without a source (the sanitizer removes
+// unsafe ones), then paragraphs, headings and list items left with no text
+// and no image, such as those that only held a removed embed.
+func removeEmpty(n *html.Node) {
+	var empty []*html.Node
+	walk(n, func(n *html.Node) {
+		if n.DataAtom == atom.Img && strings.TrimSpace(attr(n, "src")) == "" {
+			empty = append(empty, n)
+		}
+	})
+	for _, e := range empty {
+		e.Parent.RemoveChild(e)
+	}
+	empty = nil
+	walk(n, func(n *html.Node) {
+		switch n.DataAtom {
+		case atom.P, atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6, atom.Li:
+			if strings.TrimSpace(text(n)) == "" && !contains(n, atom.Img) {
+				empty = append(empty, n)
+			}
+		}
+	})
+	for _, e := range empty {
+		if e.Parent != nil {
+			e.Parent.RemoveChild(e)
+		}
+	}
+}
+
+// contains reports whether n has a descendant element of type a.
+func contains(n *html.Node, a atom.Atom) bool {
+	found := false
+	walk(n, func(d *html.Node) { found = found || (d != n && d.DataAtom == a) })
+	return found
+}
+
+func walk(n *html.Node, f func(*html.Node)) {
+	if n.Type == html.ElementNode {
+		f(n)
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		walk(c, f)
+	}
+}
+
+func text(n *html.Node) string {
+	var b strings.Builder
+	var rec func(*html.Node)
+	rec = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			b.WriteString(n.Data)
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			rec(c)
+		}
+	}
+	rec(n)
+	return b.String()
+}
+
+// WellFormed reports whether s parses as strict XML using only the five
+// predefined entities, as EPUB content documents must.
+func WellFormed(s string) error {
+	d := xml.NewDecoder(strings.NewReader(s))
+	d.Strict = true
+	for {
+		_, err := d.Token()
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+	}
+}

internal/clean/clean_test.go

@@ -0,0 +1,164 @@
+package clean
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+var base, _ = url.Parse("https://example.com/posts/article")
+
+// body parses markup as the children of a <body>, like extracted content.
+func body(t *testing.T, markup string) *html.Node {
+	t.Helper()
+	doc, err := html.Parse(strings.NewReader("<body>" + markup + "</body>"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return doc.FirstChild.LastChild // html > body
+}
+
+func cleanString(t *testing.T, markup string) string {
+	t.Helper()
+	cleaned, err := Content(body(t, markup), base)
+	if err != nil {
+		t.Fatalf("Content(%q): %v", markup, err)
+	}
+	out, err := Render(cleaned)
+	if err != nil {
+		t.Fatalf("Render(%q): %v", markup, err)
+	}
+	if err := WellFormed("<div>" + out + "</div>"); err != nil {
+		t.Fatalf("Content(%q) = %q, not well-formed: %v", markup, out, err)
+	}
+	return out
+}
+
+func TestContentSanitizes(t *testing.T) {
+	tests := []struct {
+		name, in, want string
+	}{
+		{"keeps basic markup",
+			`<p>Hello <em>there</em> <strong>you</strong>.</p>`,
+			`<p>Hello <em>there</em> <strong>you</strong>.</p>`},
+		{"drops scripts and their content",
+			`<p>a</p><script>alert(1)</script><p>b</p>`,
+			`<p>a</p><p>b</p>`},
+		{"drops styles and their content",
+			`<style>p { color: red }</style><p>a</p>`,
+			`<p>a</p>`},
+		{"drops event handlers, classes, ids and inline styles",
+			`<p class="x" id="y" style="color:red" onclick="evil()">a</p>`,
+			`<p>a</p>`},
+		{"drops forms but keeps their text",
+			`<form action="/x"><p>Sign up</p><input name="email"><button>Go</button></form>`,
+			`<p>Sign up</p>Go`},
+		{"drops iframes and objects",
+			`<p>a</p><iframe src="https://ads.example.com"></iframe><object data="x.swf"></object>`,
+			`<p>a</p>`},
+		{"drops javascript links but keeps the text",
+			`<p><a href="javascript:alert(1)">click</a></p>`,
+			`<p>click</p>`},
+		{"makes relative links absolute",
+			`<p><a href="/about">about</a> <a href="other">other</a> <a href="#notes">notes</a></p>`,
+			`<p><a href="https://example.com/about">about</a> <a href="https://example.com/posts/other">other</a> <a href="https://example.com/posts/article#notes">notes</a></p>`},
+		{"keeps mailto links",
+			`<p><a href="mailto:a@example.com">mail</a></p>`,
+			`<p><a href="mailto:a@example.com">mail</a></p>`},
+		{"keeps valid table spans and drops invalid ones",
+			`<table><tr><td colspan="2" rowspan="0">a</td></tr></table>`,
+			`<table><tbody><tr><td colspan="2">a</td></tr></tbody></table>`},
+		{"keeps valid lang and dir, drops invalid",
+			`<p lang="fr" dir="rtl">a</p><p lang="not valid!" dir="sideways">b</p>`,
+			`<p lang="fr" dir="rtl">a</p><p>b</p>`},
+		{"keeps images in paragraphs and makes their src absolute",
+			`<p>a</p><p><img src="/x.png" alt="x" class="big" onerror="evil()"></p>`,
+			`<p>a</p><p><img src="https://example.com/x.png" alt="x"/></p>`},
+		{"drops images with unsafe sources",
+			`<p>a <img src="javascript:evil()" alt="x"></p>`,
+			`<p>a </p>`},
+		{"removes empty paragraphs",
+			`<p>a</p><p> </p><p><br></p><p>b</p>`,
+			`<p>a</p><p>b</p>`},
+		{"repairs mailto and tel links mangled by go-trafilatura",
+			`<p><a href="https://example.com/posts/mailto:a@example.com">mail</a> <a href="https://example.com/posts/article/tel:+15551234">call</a></p>`,
+			`<p><a href="mailto:a@example.com">mail</a> <a href="tel:+15551234">call</a></p>`},
+		{"restores code blocks from go-trafilatura's block code",
+			"<code><code>a = 1\nb = 2\n</code></code><p>after</p>",
+			"<pre><code>a = 1\nb = 2\n</code></pre><p>after</p>"},
+		{"restores multi-line code inside a list item",
+			"<ul><li><code>x\ny</code></li></ul>",
+			"<ul><li><pre><code>x\ny</code></pre></li></ul>"},
+		{"leaves inline code in a paragraph alone, even with a line break",
+			"<p>Run <code>make\nbuild</code> now.</p>",
+			"<p>Run <code>make\nbuild</code> now.</p>"},
+		{"keeps code blocks",
+			`<pre><code>if a &lt; b {
+	return
+}</code></pre>`,
+			"<pre><code>if a &lt; b {\n\treturn\n}</code></pre>"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := cleanString(t, tt.in); got != tt.want {
+				t.Errorf("got  %q\nwant %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestContentRepairsNesting(t *testing.T) {
+	// Each of these is invalid HTML that must still come out well-formed.
+	for _, in := range []string{
+		`<p>a<div>b</div>c</p>`,
+		`<ul><p>loose</p><li>item</li></ul>`,
+		`<a href="/a">outer <a href="/b">inner</a></a>`,
+		`<table><td>no row</td></table>`,
+		`<p>unclosed <em>emphasis<p>next`,
+		`<li>orphan item</li>`,
+	} {
+		cleanString(t, in) // fails the test if not well-formed
+	}
+}
+
+func TestContentEntities(t *testing.T) {
+	// &nbsp; and other HTML entities aren't defined in XML; they must come out
+	// as characters.
+	got := cleanString(t, `<p>a&nbsp;b &copy; &mdash; &lt;tag&gt; &amp;</p>`)
+	want := "<p>a b © — &lt;tag&gt; &amp;</p>"
+	if got != want {
+		t.Errorf("got %q, want %q", got, want)
+	}
+}
+
+func TestShiftHeadings(t *testing.T) {
+	tests := []struct{ in, want string }{
+		{`<h1>a</h1><h2>b</h2><h3>c</h3>`, `<h2>a</h2><h3>b</h3><h4>c</h4>`},
+		{`<h2>a</h2><h3>b</h3>`, `<h2>a</h2><h3>b</h3>`},
+		{`<h3>a</h3><h4>b</h4>`, `<h3>a</h3><h4>b</h4>`},
+		{`<h1>a</h1><h5>b</h5><h6>c</h6>`, `<h2>a</h2><h6>b</h6><h6>c</h6>`},
+		{`<p>no headings</p>`, `<p>no headings</p>`},
+	}
+	for _, tt := range tests {
+		if got := cleanString(t, tt.in); got != tt.want {
+			t.Errorf("%s: got %q, want %q", tt.in, got, tt.want)
+		}
+	}
+}
+
+func TestWellFormed(t *testing.T) {
+	good := []string{`<div><p>a</p><br/></div>`, `<p>&amp; &lt; &gt; &quot; &apos; &#160;</p>`}
+	bad := []string{`<p>unclosed`, `<p>a</div>`, `<p>&nbsp;</p>`, `<br>`}
+	for _, s := range good {
+		if err := WellFormed(s); err != nil {
+			t.Errorf("WellFormed(%q) = %v, want nil", s, err)
+		}
+	}
+	for _, s := range bad {
+		if err := WellFormed(s); err == nil {
+			t.Errorf("WellFormed(%q) = nil, want an error", s)
+		}
+	}
+}

internal/clean/embeds.go

@@ -0,0 +1,89 @@
+package clean
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// Prepare edits the downloaded page before the article is extracted:
+//
+//   - It makes every link absolute. go-trafilatura resolves relative links
+//     as if the page URL were a folder ("../faq" on /docs/config becomes
+//     /docs/faq instead of /faq), so they must be absolute before it runs.
+//   - It fixes images for go-trafilatura, which only looks at src; see
+//     prepareImages.
+//   - It replaces embedded YouTube and Vimeo players, which can't play on an
+//     e-reader and would otherwise vanish without a trace, with a paragraph
+//     linking to the video.
+func Prepare(doc *html.Node, base *url.URL) {
+	resolveLinks(doc, base)
+	prepareImages(doc, base)
+
+	var iframes []*html.Node
+	walk(doc, func(n *html.Node) {
+		if n.DataAtom == atom.Iframe {
+			iframes = append(iframes, n)
+		}
+	})
+	for _, f := range iframes {
+		link, ok := videoLink(attr(f, "src"), base)
+		if !ok || f.Parent == nil {
+			continue
+		}
+		label := strings.TrimSpace(attr(f, "title"))
+		if label == "" {
+			label = link
+		}
+		f.Parent.InsertBefore(videoParagraph(link, label), f)
+		f.Parent.RemoveChild(f)
+	}
+}
+
+var (
+	youTubeEmbed = regexp.MustCompile(`^(?:www\.)?youtube(?:-nocookie)?\.com$`)
+	youTubeID    = regexp.MustCompile(`^/embed/([A-Za-z0-9_-]{6,})`)
+	vimeoID      = regexp.MustCompile(`^/video/([0-9]+)`)
+)
+
+// videoLink turns an embedded player's URL into the video's watch page.
+func videoLink(src string, base *url.URL) (string, bool) {
+	u, err := base.Parse(src)
+	if err != nil {
+		return "", false
+	}
+	switch {
+	case youTubeEmbed.MatchString(u.Host):
+		if m := youTubeID.FindStringSubmatch(u.Path); m != nil {
+			return "https://www.youtube.com/watch?v=" + m[1], true
+		}
+	case u.Host == "player.vimeo.com":
+		if m := vimeoID.FindStringSubmatch(u.Path); m != nil {
+			return "https://vimeo.com/" + m[1], true
+		}
+	}
+	return "", false
+}
+
+// videoParagraph builds <p>▶ Video: <a href="link">label</a></p>.
+func videoParagraph(link, label string) *html.Node {
+	a := &html.Node{Type: html.ElementNode, Data: "a", DataAtom: atom.A,
+		Attr: []html.Attribute{{Key: "href", Val: link}}}
+	a.AppendChild(&html.Node{Type: html.TextNode, Data: label})
+	p := &html.Node{Type: html.ElementNode, Data: "p", DataAtom: atom.P}
+	p.AppendChild(&html.Node{Type: html.TextNode, Data: "▶ Video: "})
+	p.AppendChild(a)
+	return p
+}
+
+func attr(n *html.Node, key string) string {
+	for _, a := range n.Attr {
+		if a.Key == key {
+			return a.Val
+		}
+	}
+	return ""
+}

internal/clean/embeds_test.go

@@ -0,0 +1,64 @@
+package clean
+
+import (
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+func prepareString(t *testing.T, markup string) string {
+	t.Helper()
+	doc, err := html.Parse(strings.NewReader("<body>" + markup + "</body>"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	Prepare(doc, base)
+	restoreImageSources(doc) // show real URLs rather than stand-ins
+	var b strings.Builder
+	body := doc.FirstChild.LastChild
+	for c := body.FirstChild; c != nil; c = c.NextSibling {
+		html.Render(&b, c)
+	}
+	return b.String()
+}
+
+func TestPrepareReplacesVideoEmbeds(t *testing.T) {
+	tests := []struct{ name, in, want string }{
+		{"YouTube with title",
+			`<iframe src="https://www.youtube.com/embed/dQw4w9WgXcQ?si=abc" title="Never Gonna Give You Up"></iframe>`,
+			`<p>▶ Video: <a href="https://www.youtube.com/watch?v=dQw4w9WgXcQ">Never Gonna Give You Up</a></p>`},
+		{"YouTube without title",
+			`<iframe src="https://www.youtube.com/embed/dQw4w9WgXcQ"></iframe>`,
+			`<p>▶ Video: <a href="https://www.youtube.com/watch?v=dQw4w9WgXcQ">https://www.youtube.com/watch?v=dQw4w9WgXcQ</a></p>`},
+		{"YouTube no-cookie, protocol-relative",
+			`<iframe src="//www.youtube-nocookie.com/embed/abcDEF12345"></iframe>`,
+			`<p>▶ Video: <a href="https://www.youtube.com/watch?v=abcDEF12345">https://www.youtube.com/watch?v=abcDEF12345</a></p>`},
+		{"Vimeo",
+			`<iframe src="https://player.vimeo.com/video/76979871?h=8272103f6e" title="The New Vimeo Player"></iframe>`,
+			`<p>▶ Video: <a href="https://vimeo.com/76979871">The New Vimeo Player</a></p>`},
+		{"other iframes are left alone",
+			`<iframe src="https://ads.example.com/banner"></iframe>`,
+			`<iframe src="https://ads.example.com/banner"></iframe>`},
+		{"lookalike hosts are left alone",
+			`<iframe src="https://youtube.com.evil.example/embed/dQw4w9WgXcQ"></iframe>`,
+			`<iframe src="https://youtube.com.evil.example/embed/dQw4w9WgXcQ"></iframe>`},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := prepareString(t, tt.in); got != tt.want {
+				t.Errorf("got  %q\nwant %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestPrepareResolvesLinks(t *testing.T) {
+	// base is https://example.com/posts/article; relative links resolve
+	// against its folder, /posts/.
+	got := prepareString(t, `<p><a href="../faq#restart">a</a> <a href="other">b</a> <a href="/root">c</a> <a href="https://x.example/">d</a></p>`)
+	want := `<p><a href="https://example.com/faq#restart">a</a> <a href="https://example.com/posts/other">b</a> <a href="https://example.com/root">c</a> <a href="https://x.example/">d</a></p>`
+	if got != want {
+		t.Errorf("got  %q\nwant %q", got, want)
+	}
+}

internal/clean/images.go

@@ -0,0 +1,395 @@
+package clean
+
+import (
+	"encoding/base64"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// targetWidth is the image width worth downloading: the reMarkable 2's
+// screen width. Larger candidates only waste bandwidth.
+const targetWidth = 1404
+
+// lazySrcAttrs hold the real image URL on pages that lazy-load images.
+var lazySrcAttrs = []string{"data-src", "data-original", "data-lazy-src", "data-url", "data-hi-res-src"}
+var lazySrcsetAttrs = []string{"data-srcset", "data-lazy-srcset"}
+
+// prepareImages fixes images before extraction, because go-trafilatura only
+// looks at src:
+//
+//   - lazy-loading attributes and the best srcset or <picture> candidate
+//     are moved into src, and src is made absolute;
+//   - tracking pixels (width or height of 2 or less) are removed;
+//   - links that wrap nothing but an image, such as "click to zoom"
+//     lightbox links, are unwrapped: go-trafilatura drops an image inside
+//     such a link when it sits in a <div> or <p>, as on Ars Technica and
+//     many WordPress sites, and the link is useless on an e-reader;
+//   - each <figcaption> becomes an italic paragraph, which go-trafilatura
+//     keeps (it drops captions); Content turns it back into a caption;
+//   - every src is replaced by a stand-in URL (see protectSrc), because
+//     go-trafilatura drops images whose URL doesn't end in an image file
+//     extension; Content restores the real URL.
+func prepareImages(doc *html.Node, base *url.URL) {
+	var imgs, captions, pixels, imageLinks []*html.Node
+	walk(doc, func(n *html.Node) {
+		switch n.DataAtom {
+		case atom.A:
+			if contains(n, atom.Img) && strings.TrimSpace(text(n)) == "" {
+				imageLinks = append(imageLinks, n)
+			}
+		case atom.Img:
+			if isTrackingPixel(n) {
+				pixels = append(pixels, n)
+			} else {
+				imgs = append(imgs, n)
+			}
+		case atom.Figcaption:
+			captions = append(captions, n)
+		}
+	})
+	for _, n := range pixels {
+		n.Parent.RemoveChild(n)
+	}
+	for _, a := range imageLinks {
+		unwrap(a)
+	}
+	for _, n := range imgs {
+		src := bestSource(n)
+		if src == "" {
+			continue
+		}
+		if !strings.HasPrefix(src, "data:") {
+			u, err := base.Parse(src)
+			if err != nil {
+				continue
+			}
+			src = u.String()
+		}
+		setAttr(n, "src", protectSrc(src))
+	}
+	for _, c := range captions {
+		// <figcaption>x</figcaption> → <p><i>x</i></p>
+		i := &html.Node{Type: html.ElementNode, Data: "i", DataAtom: atom.I}
+		if hasBlock(c) {
+			// A paragraph can't hold blocks, so captions laid out with
+			// <div>s (as on Ars Technica) are reduced to their text.
+			caption := spacedText(c)
+			for child := c.FirstChild; child != nil; child = c.FirstChild {
+				c.RemoveChild(child)
+			}
+			i.AppendChild(&html.Node{Type: html.TextNode, Data: caption})
+		} else {
+			for child := c.FirstChild; child != nil; child = c.FirstChild {
+				c.RemoveChild(child)
+				i.AppendChild(child)
+			}
+		}
+		c.Data, c.DataAtom, c.Attr = "p", atom.P, nil
+		c.AppendChild(i)
+	}
+}
+
+// bestSource picks the URL to download for an <img>: the best candidate
+// from its srcset, its <picture> sources or lazy-loading attributes, else
+// its src. A placeholder data: URL in src loses to any real URL.
+func bestSource(img *html.Node) string {
+	var srcsets []string
+	if pic := img.Parent; pic != nil && pic.DataAtom == atom.Picture {
+		for s := pic.FirstChild; s != nil; s = s.NextSibling {
+			if s.DataAtom == atom.Source && usableType(attr(s, "type")) {
+				srcsets = append(srcsets, attr(s, "srcset"), attr(s, "data-srcset"))
+			}
+		}
+	}
+	srcsets = append(srcsets, attr(img, "srcset"))
+	for _, a := range lazySrcsetAttrs {
+		srcsets = append(srcsets, attr(img, a))
+	}
+
+	var candidates []candidate
+	for _, s := range srcsets {
+		candidates = append(candidates, parseSrcset(s)...)
+	}
+	if best := pickCandidate(candidates); best != "" {
+		return best
+	}
+	for _, a := range lazySrcAttrs {
+		if v := strings.TrimSpace(attr(img, a)); v != "" {
+			return v
+		}
+	}
+	return strings.TrimSpace(attr(img, "src"))
+}
+
+// usableType reports whether a <source> type is one we can decode. AVIF and
+// JPEG XL aren't supported, so their sources are skipped in favour of the
+// fallback.
+func usableType(t string) bool {
+	switch strings.ToLower(strings.TrimSpace(t)) {
+	case "", "image/jpeg", "image/jpg", "image/png", "image/gif", "image/webp", "image/svg+xml":
+		return true
+	}
+	return false
+}
+
+type candidate struct {
+	url     string
+	width   int     // from a "800w" descriptor, or 0
+	density float64 // from a "2x" descriptor, or 0
+}
+
+// parseSrcset splits a srcset attribute into candidates. URLs may contain
+// commas (as some image CDNs use), so candidates are split on the comma
+// that follows a descriptor or whitespace, as the HTML spec describes.
+func parseSrcset(s string) []candidate {
+	var out []candidate
+	for s = strings.TrimSpace(s); s != ""; s = strings.TrimLeft(s, " \t\n\r\f,") {
+		end := strings.IndexAny(s, " \t\n\r\f")
+		if end < 0 {
+			end = len(s)
+		}
+		u := s[:end]
+		s = s[end:]
+		var desc string
+		if strings.HasSuffix(u, ",") {
+			u = strings.TrimRight(u, ",")
+		} else {
+			comma := strings.IndexByte(s, ',')
+			if comma < 0 {
+				comma = len(s)
+			}
+			desc = strings.TrimSpace(s[:comma])
+			s = s[comma:]
+		}
+		if u == "" || strings.HasPrefix(u, "data:") {
+			continue
+		}
+		c := candidate{url: u}
+		switch {
+		case strings.HasSuffix(desc, "w"):
+			c.width, _ = strconv.Atoi(strings.TrimSuffix(desc, "w"))
+		case strings.HasSuffix(desc, "x"):
+			c.density, _ = strconv.ParseFloat(strings.TrimSuffix(desc, "x"), 64)
+		}
+		out = append(out, c)
+	}
+	return out
+}
+
+// pickCandidate chooses the smallest candidate at least targetWidth wide,
+// or the widest if none is; without width descriptors it takes the highest
+// density.
+func pickCandidate(cs []candidate) string {
+	var best *candidate
+	for i := range cs {
+		c := &cs[i]
+		if c.width == 0 {
+			continue
+		}
+		switch {
+		case best == nil:
+			best = c
+		case best.width < targetWidth:
+			if c.width > best.width {
+				best = c
+			}
+		case c.width >= targetWidth && c.width < best.width:
+			best = c
+		}
+	}
+	if best != nil {
+		return best.url
+	}
+	for i := range cs {
+		if best == nil || cs[i].density > best.density {
+			best = &cs[i]
+		}
+	}
+	if best != nil {
+		return best.url
+	}
+	return ""
+}
+
+// isTrackingPixel reports whether an <img> declares itself 2px or smaller.
+func isTrackingPixel(n *html.Node) bool {
+	for _, key := range []string{"width", "height"} {
+		if v, err := strconv.Atoi(strings.TrimSuffix(strings.TrimSpace(attr(n, key)), "px")); err == nil && v <= 2 {
+			return true
+		}
+	}
+	return false
+}
+
+// buildFigures wraps each standalone image (not inside a paragraph or
+// other text) in a <figure>, taking a following paragraph that is entirely
+// italic as its caption; prepareImages turned captions into exactly that.
+func buildFigures(body *html.Node) {
+	var standalone []*html.Node
+	walk(body, func(n *html.Node) {
+		if n.DataAtom == atom.Img && n.Parent != nil && blockParents[n.Parent.DataAtom] {
+			standalone = append(standalone, n)
+		}
+	})
+	for _, img := range standalone {
+		fig := &html.Node{Type: html.ElementNode, Data: "figure", DataAtom: atom.Figure}
+		img.Parent.InsertBefore(fig, img)
+		next := nextElement(img)
+		img.Parent.RemoveChild(img)
+		fig.AppendChild(img)
+		if next != nil && isCaption(next) {
+			i := firstElement(next)
+			next.Parent.RemoveChild(next)
+			caption := &html.Node{Type: html.ElementNode, Data: "figcaption", DataAtom: atom.Figcaption}
+			for c := i.FirstChild; c != nil; c = i.FirstChild {
+				i.RemoveChild(c)
+				caption.AppendChild(c)
+			}
+			fig.AppendChild(caption)
+		}
+	}
+}
+
+// isCaption reports whether p is a paragraph holding only one <i> or <em>.
+func isCaption(p *html.Node) bool {
+	if p.DataAtom != atom.P {
+		return false
+	}
+	i := firstElement(p)
+	if i == nil || (i.DataAtom != atom.I && i.DataAtom != atom.Em) || nextElement(i) != nil {
+		return false
+	}
+	for c := p.FirstChild; c != nil; c = c.NextSibling {
+		if c.Type == html.TextNode && strings.TrimSpace(c.Data) != "" {
+			return false
+		}
+	}
+	return strings.TrimSpace(text(i)) != ""
+}
+
+// nextElement returns the next sibling element of n, skipping whitespace.
+func nextElement(n *html.Node) *html.Node {
+	for s := n.NextSibling; s != nil; s = s.NextSibling {
+		if s.Type == html.ElementNode {
+			return s
+		}
+		if s.Type == html.TextNode && strings.TrimSpace(s.Data) != "" {
+			return nil
+		}
+	}
+	return nil
+}
+
+func firstElement(n *html.Node) *html.Node {
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if c.Type == html.ElementNode {
+			return c
+		}
+	}
+	return nil
+}
+
+// hasBlock reports whether n contains block-level elements.
+func hasBlock(n *html.Node) bool {
+	found := false
+	walk(n, func(d *html.Node) {
+		switch d.DataAtom {
+		case atom.Div, atom.P, atom.Ul, atom.Ol, atom.Table, atom.Blockquote, atom.Section, atom.Figure:
+			found = found || d != n
+		}
+	})
+	return found
+}
+
+// spacedText returns the text of n with a space at every element boundary
+// and whitespace collapsed, so "dialog.<span>Credit: KDE</span>" reads
+// "dialog. Credit: KDE".
+func spacedText(n *html.Node) string {
+	var b strings.Builder
+	var rec func(*html.Node)
+	rec = func(n *html.Node) {
+		switch n.Type {
+		case html.TextNode:
+			b.WriteString(n.Data)
+		case html.ElementNode:
+			b.WriteByte(' ')
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			rec(c)
+		}
+		if n.Type == html.ElementNode {
+			b.WriteByte(' ')
+		}
+	}
+	rec(n)
+	return strings.Join(strings.Fields(b.String()), " ")
+}
+
+// srcStandIn prefixes the stand-in URLs that carry image sources through
+// go-trafilatura. The .invalid domain is reserved and never resolves, so a
+// stand-in that somehow escaped could not reach anyone's server.
+const srcStandIn = "https://rekopub.invalid/img/"
+
+// protectSrc wraps an image URL in a stand-in ending in ".png".
+// go-trafilatura keeps an image only when filepath.Ext of its whole URL is
+// an image type, so it drops URLs with a query string
+// ("photo.jpg?utm_source=…", as on Wikipedia), extensionless CDN URLs and
+// data: URLs. The stand-in passes that check; restoreSrc undoes it.
+func protectSrc(src string) string {
+	return srcStandIn + base64.RawURLEncoding.EncodeToString([]byte(src)) + ".png"
+}
+
+// restoreSrc returns the original URL inside a stand-in made by
+// protectSrc, or src unchanged if it isn't one.
+func restoreSrc(src string) string {
+	enc, ok := strings.CutPrefix(src, srcStandIn)
+	if !ok {
+		return src
+	}
+	raw, err := base64.RawURLEncoding.DecodeString(strings.TrimSuffix(enc, ".png"))
+	if err != nil {
+		return ""
+	}
+	return string(raw)
+}
+
+// restoreImageSources replaces every stand-in src under n with its
+// original URL. It matches on the tag name: go-trafilatura creates its
+// output elements without setting DataAtom.
+func restoreImageSources(n *html.Node) {
+	walk(n, func(n *html.Node) {
+		if n.Data == "img" {
+			for i, a := range n.Attr {
+				if a.Key == "src" {
+					n.Attr[i].Val = restoreSrc(a.Val)
+				}
+			}
+		}
+	})
+}
+
+// unwrap replaces n with its children.
+func unwrap(n *html.Node) {
+	if n.Parent == nil {
+		return
+	}
+	for c := n.FirstChild; c != nil; c = n.FirstChild {
+		n.RemoveChild(c)
+		n.Parent.InsertBefore(c, n)
+	}
+	n.Parent.RemoveChild(n)
+}
+
+func setAttr(n *html.Node, key, val string) {
+	for i := range n.Attr {
+		if n.Attr[i].Key == key {
+			n.Attr[i].Val = val
+			return
+		}
+	}
+	n.Attr = append(n.Attr, html.Attribute{Key: key, Val: val})
+}

internal/clean/images_test.go

@@ -0,0 +1,159 @@
+package clean
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+func TestParseSrcset(t *testing.T) {
+	tests := []struct {
+		in   string
+		want []candidate
+	}{
+		{"a.jpg 400w, b.jpg 1500w", []candidate{{url: "a.jpg", width: 400}, {url: "b.jpg", width: 1500}}},
+		{"a.jpg, b.jpg 2x", []candidate{{url: "a.jpg"}, {url: "b.jpg", density: 2}}},
+		{"  a.jpg  1x ,\n b.jpg 2x  ", []candidate{{url: "a.jpg", density: 1}, {url: "b.jpg", density: 2}}},
+		// Commas inside URLs, as some image CDNs use.
+		{"https://cdn.example/w_400,h_300/a.jpg 400w, https://cdn.example/w_1600,h_1200/a.jpg 1600w",
+			[]candidate{{url: "https://cdn.example/w_400,h_300/a.jpg", width: 400}, {url: "https://cdn.example/w_1600,h_1200/a.jpg", width: 1600}}},
+		{"single.jpg", []candidate{{url: "single.jpg"}}},
+		{"data:image/gif;base64,R0lG 1x, real.jpg 2x", []candidate{{url: "real.jpg", density: 2}}},
+		{"", nil},
+	}
+	for _, tt := range tests {
+		if got := parseSrcset(tt.in); !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("parseSrcset(%q) = %+v, want %+v", tt.in, got, tt.want)
+		}
+	}
+}
+
+func TestPickCandidate(t *testing.T) {
+	tests := []struct {
+		srcset, want string
+	}{
+		{"a 400w, b 1500w, c 3000w", "b"}, // smallest that is at least 1404 wide
+		{"c 3000w, b 1500w, a 400w", "b"}, // order doesn't matter
+		{"a 400w, b 800w", "b"},           // none wide enough: the widest
+		{"a 1x, b 2x, c 1.5x", "b"},       // densities: the highest
+		{"a, b", "a"},                     // no descriptors: the first
+		{"", ""},
+	}
+	for _, tt := range tests {
+		if got := pickCandidate(parseSrcset(tt.srcset)); got != tt.want {
+			t.Errorf("pickCandidate(%q) = %q, want %q", tt.srcset, got, tt.want)
+		}
+	}
+}
+
+func TestPrepareImages(t *testing.T) {
+	tests := []struct{ name, in, want string }{
+		{"relative src made absolute",
+			`<img src="photo.jpg" alt="p">`,
+			`<img src="https://example.com/posts/photo.jpg" alt="p"/>`},
+		{"lazy data-src replaces placeholder",
+			`<img src="data:image/gif;base64,R0lGODlhAQABAAAAACw=" data-src="/img/real.png" alt="p">`,
+			`<img src="https://example.com/img/real.png" data-src="/img/real.png" alt="p"/>`},
+		{"data-original",
+			`<img data-original="/img/real.png">`,
+			`<img data-original="/img/real.png" src="https://example.com/img/real.png"/>`},
+		{"srcset picks the right width",
+			`<img src="/s.jpg" srcset="/s.jpg 400w, /m.jpg 1500w, /l.jpg 3000w">`,
+			`<img src="https://example.com/m.jpg" srcset="/s.jpg 400w, /m.jpg 1500w, /l.jpg 3000w"/>`},
+		{"picture skips AVIF sources and uses WebP",
+			`<picture><source srcset="/p.avif" type="image/avif"><source srcset="/p-800.webp 800w, /p-1600.webp 1600w" type="image/webp"><img src="/p.jpg"></picture>`,
+			`<picture><source srcset="/p.avif" type="image/avif"/><source srcset="/p-800.webp 800w, /p-1600.webp 1600w" type="image/webp"/><img src="https://example.com/p-1600.webp"/></picture>`},
+		{"picture with only AVIF falls back to the img",
+			`<picture><source srcset="/p.avif" type="image/avif"><img src="/p.jpg"></picture>`,
+			`<picture><source srcset="/p.avif" type="image/avif"/><img src="https://example.com/p.jpg"/></picture>`},
+		{"tracking pixels are removed",
+			`<p>a<img src="https://t.example/p.gif" width="1" height="1">b<img src="/x.gif" height="2px"></p>`,
+			`<p>ab</p>`},
+		{"captions laid out with divs are reduced to text",
+			`<figure><img src="/a.jpg"><figcaption class="hidden"><div class="caption"><div class="icon"></div><div class="caption-content">The new dialog.<span class="credit">Credit: KDE</span></div></div></figcaption></figure>`,
+			`<figure><img src="https://example.com/a.jpg"/><p><i>The new dialog. Credit: KDE</i></p></figure>`},
+		{"zoom links around images are unwrapped",
+			`<div><a href="/full.jpg" class="cursor-zoom-in"><img src="/small.jpg" alt="x"></a></div>`,
+			`<div><img src="https://example.com/small.jpg" alt="x"/></div>`},
+		{"links with text around images are kept",
+			`<p><a href="/page"><img src="/icon.png" alt=""> Read more</a></p>`,
+			`<p><a href="https://example.com/page"><img src="https://example.com/icon.png" alt=""/> Read more</a></p>`},
+		{"figcaption becomes an italic paragraph",
+			`<figure><img src="/a.jpg"><figcaption class="c">A <b>bold</b> caption</figcaption></figure>`,
+			`<figure><img src="https://example.com/a.jpg"/><p><i>A <b>bold</b> caption</i></p></figure>`},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := prepareString(t, tt.in); got != tt.want {
+				t.Errorf("got  %q\nwant %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestBuildFigures(t *testing.T) {
+	tests := []struct{ name, in, want string }{
+		{"standalone image with caption",
+			`<img src="https://example.com/a.jpg" alt="A"><p><i>The caption</i></p><p>Text.</p>`,
+			`<figure><img src="https://example.com/a.jpg" alt="A"/><figcaption>The caption</figcaption></figure><p>Text.</p>`},
+		{"em caption",
+			`<img src="https://example.com/a.jpg"><p><em>Caption</em></p>`,
+			`<figure><img src="https://example.com/a.jpg"/><figcaption>Caption</figcaption></figure>`},
+		{"standalone image without caption",
+			`<img src="https://example.com/a.jpg"><p>Just a paragraph with <i>some</i> italics.</p>`,
+			`<figure><img src="https://example.com/a.jpg"/></figure><p>Just a paragraph with <i>some</i> italics.</p>`},
+		{"image inside a paragraph stays inline",
+			`<p>Look <img src="https://example.com/a.jpg"> here.</p>`,
+			`<p>Look <img src="https://example.com/a.jpg"/> here.</p>`},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := cleanString(t, tt.in); got != tt.want {
+				t.Errorf("got  %q\nwant %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestStandInSources(t *testing.T) {
+	for _, src := range []string{
+		"https://thumb.example.org/a/Photo.jpg/500px-Photo.jpg?utm_source=en.wikipedia.org&utm_campaign=parser",
+		"https://images.example.org/media/12345",
+		"data:image/png;base64,iVBORw0KGgo=",
+		"https://example.com/plain.png",
+	} {
+		standIn := protectSrc(src)
+		if !strings.HasPrefix(standIn, "https://rekopub.invalid/img/") || !strings.HasSuffix(standIn, ".png") {
+			t.Errorf("protectSrc(%q) = %q, want a rekopub.invalid URL ending in .png", src, standIn)
+		}
+		if got := restoreSrc(standIn); got != src {
+			t.Errorf("restoreSrc(protectSrc(%q)) = %q", src, got)
+		}
+	}
+	if got := restoreSrc("https://example.com/a.png"); got != "https://example.com/a.png" {
+		t.Errorf("restoreSrc changed an ordinary URL: %q", got)
+	}
+	if got := restoreSrc("https://rekopub.invalid/img/!!!.png"); got != "" {
+		t.Errorf("restoreSrc of a damaged stand-in = %q, want empty", got)
+	}
+}
+
+func TestPrepareUsesStandIns(t *testing.T) {
+	doc, err := html.Parse(strings.NewReader(`<body><img src="/a.jpg?x=1"></body>`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	Prepare(doc, base)
+	var src string
+	walk(doc, func(n *html.Node) {
+		if n.DataAtom == atom.Img {
+			src = attr(n, "src")
+		}
+	})
+	if !strings.HasPrefix(src, srcStandIn) || restoreSrc(src) != "https://example.com/a.jpg?x=1" {
+		t.Errorf("src after Prepare = %q", src)
+	}
+}

internal/cover/cover.go

@@ -0,0 +1,219 @@
+// Package cover draws book covers for e-ink readers.
+package cover
+
+import (
+	"bytes"
+	"image"
+	"image/color"
+	"image/draw"
+	"image/jpeg"
+	"strings"
+	"sync"
+
+	xdraw "golang.org/x/image/draw"
+	"golang.org/x/image/font"
+	"golang.org/x/image/font/gofont/gobold"
+	"golang.org/x/image/font/gofont/goregular"
+	"golang.org/x/image/font/opentype"
+	"golang.org/x/image/math/fixed"
+
+	"git.monoexp.com/bajdev/rekopub/internal/assets"
+	"git.monoexp.com/bajdev/rekopub/internal/device"
+)
+
+// Details is the text shown on a cover.
+type Details struct {
+	Title    string
+	SiteName string // may be empty
+	Date     string // already formatted; may be empty
+}
+
+// Render draws a grayscale JPEG cover at the profile's cover size. With a
+// photo, the photo fills the top of the cover and the title sits on white
+// below it; without one, the cover is typographic. Either way the title is
+// large, because library thumbnails on e-ink readers are small.
+func Render(d Details, photo image.Image, p device.Profile) ([]byte, error) {
+	w, h := p.CoverWidth, p.CoverHeight
+	img := image.NewGray(image.Rect(0, 0, w, h))
+	draw.Draw(img, img.Bounds(), image.White, image.Point{}, draw.Src)
+
+	margin := w / 14
+	textWidth := w - 2*margin
+	sub := subtitle(d)
+
+	if photo != nil {
+		photoHeight := h * 3 / 5
+		drawPhoto(img, image.Rect(0, 0, w, photoHeight), photo)
+		top := photoHeight + margin*2/3
+		bottom := h - margin
+		if sub != "" {
+			bottom -= drawLines(img, regular(), []string{sub}, 46, margin, bottom-46, color.Black, true)
+			bottom -= margin / 3
+		}
+		face, lines := fitTitle(d.Title, textWidth, bottom-top, 104, 52)
+		drawWrapped(img, face, lines, margin, top)
+	} else {
+		// A black band across the top with the site name, a large title, and
+		// the date at the bottom.
+		band := h / 9
+		draw.Draw(img, image.Rect(0, 0, w, band), image.Black, image.Point{}, draw.Src)
+		if d.SiteName != "" {
+			name := truncate(regular(), 54, d.SiteName, textWidth)
+			drawLines(img, regular(), []string{name}, 54, margin, band/2+18, color.White, false)
+		}
+		top := band + margin*3/2
+		bottom := h - margin*2
+		face, lines := fitTitle(d.Title, textWidth, bottom-top, 140, 64)
+		end := drawWrapped(img, face, lines, margin, top)
+		rule := end + margin*3/4
+		draw.Draw(img, image.Rect(margin, rule, margin+w/4, rule+8), image.Black, image.Point{}, draw.Src)
+		if d.Date != "" {
+			drawLines(img, regular(), []string{d.Date}, 46, margin, rule+margin, color.Black, false)
+		}
+	}
+
+	var buf bytes.Buffer
+	if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 85}); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// subtitle is "Site · Date", skipping whichever is missing.
+func subtitle(d Details) string {
+	var parts []string
+	for _, s := range []string{d.SiteName, d.Date} {
+		if s = strings.TrimSpace(s); s != "" {
+			parts = append(parts, s)
+		}
+	}
+	return strings.Join(parts, " · ")
+}
+
+// drawPhoto scales photo to cover r completely, cropping the overflow
+// evenly, and draws it in grayscale.
+func drawPhoto(dst *image.Gray, r image.Rectangle, photo image.Image) {
+	gray := assets.ToGray(photo)
+	sw, sh := gray.Bounds().Dx(), gray.Bounds().Dy()
+	scale := max(float64(r.Dx())/float64(sw), float64(r.Dy())/float64(sh))
+	cw, ch := int(float64(r.Dx())/scale), int(float64(r.Dy())/scale)
+	x0, y0 := (sw-cw)/2, (sh-ch)/2
+	xdraw.CatmullRom.Scale(dst, r, gray, image.Rect(x0, y0, x0+cw, y0+ch), xdraw.Src, nil)
+}
+
+// fitTitle picks the largest font size, from maxSize down to minSize, at
+// which the title wraps into the given box. At the smallest size, lines
+// that don't fit are dropped and the last kept line ends with "…".
+func fitTitle(title string, width, height int, maxSize, minSize float64) (font.Face, []string) {
+	title = strings.Join(strings.Fields(title), " ")
+	if title == "" {
+		title = "Untitled"
+	}
+	for size := maxSize; ; size -= 4 {
+		face := faceFor(bold(), size)
+		lines := wrap(face, title, width)
+		lineHeight := int(size * 1.18)
+		if len(lines)*lineHeight <= height || size-4 < minSize {
+			if keep := max(1, height/lineHeight); len(lines) > keep {
+				lines = lines[:keep]
+				lines[keep-1] = truncate(bold(), size, lines[keep-1]+" …", width)
+			}
+			return face, lines
+		}
+	}
+}
+
+// wrap breaks text into lines no wider than width, splitting words that
+// are too long on their own.
+func wrap(face font.Face, text string, width int) []string {
+	var lines []string
+	line := ""
+	for _, word := range strings.Fields(text) {
+		for font.MeasureString(face, word).Ceil() > width {
+			// Split an overlong word at the widest prefix that fits.
+			n := len([]rune(word)) - 1
+			for n > 1 && font.MeasureString(face, string([]rune(word)[:n])).Ceil() > width {
+				n--
+			}
+			if line != "" {
+				lines = append(lines, line)
+				line = ""
+			}
+			lines = append(lines, string([]rune(word)[:n]))
+			word = string([]rune(word)[n:])
+		}
+		candidate := strings.TrimSpace(line + " " + word)
+		if line != "" && font.MeasureString(face, candidate).Ceil() > width {
+			lines = append(lines, line)
+			candidate = word
+		}
+		line = candidate
+	}
+	if line != "" {
+		lines = append(lines, line)
+	}
+	return lines
+}
+
+// truncate shortens s with "…" until it fits within width.
+func truncate(f *opentype.Font, size float64, s string, width int) string {
+	face := faceFor(f, size)
+	if font.MeasureString(face, s).Ceil() <= width {
+		return s
+	}
+	r := []rune(strings.TrimSuffix(s, " …"))
+	for len(r) > 1 && font.MeasureString(face, string(r)+"…").Ceil() > width {
+		r = r[:len(r)-1]
+	}
+	return strings.TrimSpace(string(r)) + "…"
+}
+
+// drawWrapped draws lines starting with the first line's top at y, and
+// returns the y just below the last line.
+func drawWrapped(dst *image.Gray, face font.Face, lines []string, x, y int) int {
+	m := face.Metrics()
+	lineHeight := (m.Ascent + m.Descent).Ceil() * 118 / 100
+	baseline := y + m.Ascent.Ceil()
+	for _, line := range lines {
+		(&font.Drawer{Dst: dst, Src: image.Black, Face: face, Dot: fixed.P(x, baseline)}).DrawString(line)
+		baseline += lineHeight
+	}
+	return baseline - lineHeight + m.Descent.Ceil()
+}
+
+// drawLines draws single lines at baseline y in the given font and colour,
+// and returns the height used. With fromBottom, y is the bottom edge.
+func drawLines(dst *image.Gray, f *opentype.Font, lines []string, size float64, x, y int, c color.Color, fromBottom bool) int {
+	face := faceFor(f, size)
+	m := face.Metrics()
+	if fromBottom {
+		y -= m.Descent.Ceil()
+	}
+	for _, line := range lines {
+		(&font.Drawer{Dst: dst, Src: image.NewUniform(c), Face: face, Dot: fixed.P(x, y)}).DrawString(line)
+	}
+	return (m.Ascent + m.Descent).Ceil()
+}
+
+var (
+	bold    = sync.OnceValue(func() *opentype.Font { return mustParse(gobold.TTF) })
+	regular = sync.OnceValue(func() *opentype.Font { return mustParse(goregular.TTF) })
+)
+
+// mustParse parses an embedded font; it can only fail if the font data is
+// corrupt, which is a build problem.
+func mustParse(ttf []byte) *opentype.Font {
+	f, err := opentype.Parse(ttf)
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
+func faceFor(f *opentype.Font, size float64) font.Face {
+	face, err := opentype.NewFace(f, &opentype.FaceOptions{Size: size, DPI: 72, Hinting: font.HintingFull})
+	if err != nil {
+		panic(err) // only fails for invalid options, which are constant here
+	}
+	return face
+}

internal/cover/cover_test.go

@@ -0,0 +1,149 @@
+package cover
+
+import (
+	"bytes"
+	"image"
+	"image/color"
+	"image/jpeg"
+	"strings"
+	"testing"
+
+	"git.monoexp.com/bajdev/rekopub/internal/device"
+)
+
+func decode(t *testing.T, data []byte) *image.Gray {
+	t.Helper()
+	img, err := jpeg.Decode(bytes.NewReader(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	gray, ok := img.(*image.Gray)
+	if !ok {
+		t.Fatalf("cover decodes as %T, want a grayscale JPEG", img)
+	}
+	if b := gray.Bounds(); b.Dx() != 1404 || b.Dy() != 1872 {
+		t.Fatalf("cover is %dx%d, want 1404x1872", b.Dx(), b.Dy())
+	}
+	return gray
+}
+
+// darkShare is the share of pixels in r darker than mid-gray.
+func darkShare(img *image.Gray, r image.Rectangle) float64 {
+	dark, total := 0, 0
+	for y := r.Min.Y; y < r.Max.Y; y++ {
+		for x := r.Min.X; x < r.Max.X; x++ {
+			if img.GrayAt(x, y).Y < 128 {
+				dark++
+			}
+			total++
+		}
+	}
+	return float64(dark) / float64(total)
+}
+
+func TestPlainCover(t *testing.T) {
+	data, err := Render(Details{Title: "A Short Title", SiteName: "Example News", Date: "14 March 2025"}, nil, device.EInk)
+	if err != nil {
+		t.Fatal(err)
+	}
+	img := decode(t, data)
+	if s := darkShare(img, image.Rect(0, 0, 1404, 150)); s < 0.8 {
+		t.Errorf("top band is %.0f%% dark, want a black band", s*100)
+	}
+	if s := darkShare(img, image.Rect(100, 300, 1300, 700)); s < 0.05 {
+		t.Errorf("title area is %.1f%% dark, want visible text", s*100)
+	}
+	if s := darkShare(img, image.Rect(0, 1700, 1404, 1872)); s > 0.01 {
+		t.Errorf("bottom edge is %.1f%% dark, want it empty", s*100)
+	}
+}
+
+func TestPhotoCover(t *testing.T) {
+	// A wide black photo with a white stripe in the middle: cropping to the
+	// cover's shape must keep the centre.
+	photo := image.NewRGBA(image.Rect(0, 0, 3000, 1000))
+	for y := 0; y < 1000; y++ {
+		for x := 0; x < 3000; x++ {
+			c := color.RGBA{0, 0, 0, 255}
+			if x > 1400 && x < 1600 {
+				c = color.RGBA{255, 255, 255, 255}
+			}
+			photo.Set(x, y, c)
+		}
+	}
+	data, err := Render(Details{Title: "Photo Story", SiteName: "Example", Date: "1 May 2025"}, photo, device.EInk)
+	if err != nil {
+		t.Fatal(err)
+	}
+	img := decode(t, data)
+	if v := img.GrayAt(702, 500).Y; v < 200 {
+		t.Errorf("photo centre is %d, want the white stripe", v)
+	}
+	if v := img.GrayAt(100, 500).Y; v > 50 {
+		t.Errorf("photo edge is %d, want black", v)
+	}
+	if s := darkShare(img, image.Rect(100, 1150, 1300, 1450)); s < 0.02 {
+		t.Errorf("title area is %.1f%% dark, want visible text", s*100)
+	}
+}
+
+func TestTransparentPhotoIsOnWhite(t *testing.T) {
+	photo := image.NewNRGBA(image.Rect(0, 0, 800, 600)) // fully transparent
+	data, err := Render(Details{Title: "T"}, photo, device.EInk)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v := decode(t, data).GrayAt(702, 500).Y; v < 240 {
+		t.Errorf("transparent photo became %d, want white", v)
+	}
+}
+
+func TestLongTitlesFit(t *testing.T) {
+	long := strings.Repeat("Remarkably long headline words ", 20)
+	for _, title := range []string{
+		long,
+		"Supercalifragilisticexpialidociousandthensomemorelettersontop",
+		"",
+		"日本語のタイトル",
+	} {
+		for _, photo := range []image.Image{nil, image.NewGray(image.Rect(0, 0, 600, 400))} {
+			if _, err := Render(Details{Title: title, SiteName: strings.Repeat("Site ", 30)}, photo, device.EInk); err != nil {
+				t.Errorf("Render(%q): %v", title, err)
+			}
+		}
+	}
+}
+
+func TestFitTitle(t *testing.T) {
+	// Short titles get the largest size; long ones shrink, and lines never
+	// exceed the width or the box.
+	face, lines := fitTitle("Hi", 1200, 800, 140, 64)
+	if face.Metrics().Height.Ceil() < 140 || len(lines) != 1 {
+		t.Errorf("short title: height %d, lines %q", face.Metrics().Height.Ceil(), lines)
+	}
+	title := strings.Repeat("word ", 200)
+	face, lines = fitTitle(title, 1200, 800, 140, 64)
+	if len(lines)*int(float64(face.Metrics().Height.Ceil())*1.0) > 800*1.2 {
+		t.Errorf("long title overflows: %d lines", len(lines))
+	}
+	if !strings.HasSuffix(lines[len(lines)-1], "…") {
+		t.Errorf("truncated title should end with an ellipsis: %q", lines[len(lines)-1])
+	}
+	if fitted, _ := fitTitle("", 1200, 800, 140, 64); fitted == nil {
+		t.Error("empty title should still render")
+	}
+}
+
+func TestSubtitle(t *testing.T) {
+	tests := map[Details]string{
+		{SiteName: "Site", Date: "1 May 2025"}: "Site · 1 May 2025",
+		{SiteName: "Site"}:                     "Site",
+		{Date: "1 May 2025"}:                   "1 May 2025",
+		{}:                                     "",
+	}
+	for d, want := range tests {
+		if got := subtitle(d); got != want {
+			t.Errorf("subtitle(%+v) = %q, want %q", d, got, want)
+		}
+	}
+}

internal/device/profile.go

@@ -0,0 +1,28 @@
+// Package device holds the output settings for the target e-readers.
+package device
+
+// Profile describes how images are prepared for a reading device.
+type Profile struct {
+	// MaxImageWidth and MaxImageHeight bound every image; larger images
+	// are scaled down to fit, keeping their aspect ratio.
+	MaxImageWidth, MaxImageHeight int
+	// CoverWidth and CoverHeight are the size of the generated cover.
+	CoverWidth, CoverHeight int
+	// GrayLevels is the number of shades the panel shows. Line art is
+	// quantized to this many levels.
+	GrayLevels int
+	// JPEGQuality is used for photos.
+	JPEGQuality int
+}
+
+// EInk targets the reMarkable 2 (1404×1872) and the Kobo Clara HD
+// (1072×1448). Both are 3:4 grayscale panels with 16 shades, so images are
+// sized for the larger screen and the Kobo scales them down.
+var EInk = Profile{
+	MaxImageWidth:  1404,
+	MaxImageHeight: 1872,
+	CoverWidth:     1404,
+	CoverHeight:    1872,
+	GrayLevels:     16,
+	JPEGQuality:    80,
+}

internal/epubtest/epubtest.go

@@ -0,0 +1,104 @@
+// Package epubtest has test helpers for checking generated EPUB files.
+package epubtest
+
+import (
+	"archive/zip"
+	"bytes"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.monoexp.com/bajdev/rekopub/internal/clean"
+)
+
+// Book is an opened EPUB.
+type Book struct {
+	Files map[string]string // zip entry name → content
+	Order []string          // zip entry names in archive order
+}
+
+// Open reads an EPUB and checks the container basics every reader relies
+// on: "mimetype" is the first entry, stored uncompressed, and
+// META-INF/container.xml exists.
+func Open(t *testing.T, data []byte) *Book {
+	t.Helper()
+	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		t.Fatalf("not a zip file: %v", err)
+	}
+	b := &Book{Files: map[string]string{}}
+	for _, f := range zr.File {
+		rc, err := f.Open()
+		if err != nil {
+			t.Fatal(err)
+		}
+		content, err := io.ReadAll(rc)
+		rc.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+		b.Files[f.Name] = string(content)
+		b.Order = append(b.Order, f.Name)
+	}
+
+	if len(zr.File) == 0 || zr.File[0].Name != "mimetype" {
+		t.Errorf("first zip entry = %v, want mimetype", b.Order)
+	} else if zr.File[0].Method != zip.Store {
+		t.Errorf("mimetype is compressed, want stored")
+	}
+	if got := b.Files["mimetype"]; got != "application/epub+zip" {
+		t.Errorf("mimetype = %q", got)
+	}
+	if _, ok := b.Files["META-INF/container.xml"]; !ok {
+		t.Errorf("META-INF/container.xml missing")
+	}
+	return b
+}
+
+// File returns the content of the single entry whose name ends in suffix.
+func (b *Book) File(t *testing.T, suffix string) string {
+	t.Helper()
+	var found []string
+	for _, name := range b.Order {
+		if strings.HasSuffix(name, suffix) {
+			found = append(found, name)
+		}
+	}
+	if len(found) != 1 {
+		t.Fatalf("entries ending in %q: %v (all: %v)", suffix, found, b.Order)
+	}
+	return b.Files[found[0]]
+}
+
+// AssertWellFormed checks that every XHTML, OPF and NCX file in the book is
+// well-formed XML, which reading systems require.
+func (b *Book) AssertWellFormed(t *testing.T) {
+	t.Helper()
+	for _, name := range b.Order {
+		switch filepath.Ext(name) {
+		case ".xhtml", ".opf", ".ncx", ".xml":
+			if err := clean.WellFormed(b.Files[name]); err != nil {
+				t.Errorf("%s is not well-formed XML: %v", name, err)
+			}
+		}
+	}
+}
+
+// EPUBCheck runs epubcheck on the book at path and fails the test if it
+// reports errors. Set EPUBCHECK_JAR to the path of epubcheck.jar to enable
+// it; otherwise the check is skipped. CI always sets it.
+func EPUBCheck(t *testing.T, path string) {
+	t.Helper()
+	jar := os.Getenv("EPUBCHECK_JAR")
+	if jar == "" {
+		t.Log("EPUBCHECK_JAR not set; skipping epubcheck")
+		return
+	}
+	out, err := exec.Command("java", "-jar", jar, "--quiet", path).CombinedOutput()
+	if err != nil {
+		t.Errorf("epubcheck failed: %v\n%s", err, out)
+	}
+}

internal/extract/extract.go

@@ -0,0 +1,129 @@
+// Package extract finds the readable article in a web page, along with its
+// metadata.
+package extract
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	trafilatura "github.com/markusmobius/go-trafilatura"
+	"golang.org/x/net/html"
+	"golang.org/x/text/language"
+)
+
+// minTextLen is the shortest article text, in characters, worth turning
+// into a book. Anything shorter is usually a JavaScript shell, an error page
+// or a cookie wall.
+const minTextLen = 200
+
+// ErrNoContent reports that no readable article was found.
+var ErrNoContent = errors.New("no readable article content found (the page may need JavaScript, which isn't supported yet)")
+
+// Article is the extracted article.
+type Article struct {
+	URL         string // final page URL
+	Title       string
+	Author      string
+	SiteName    string
+	Description string
+	Language    string // BCP 47 code, "en" when unknown
+	Published   time.Time
+	ImageURL    string     // the page's preview image, for the cover; may be empty
+	Content     *html.Node // the article body; its children are the content
+}
+
+// Extract finds the article in doc, a parsed page downloaded from pageURL.
+// It may modify doc.
+func Extract(doc *html.Node, pageURL *url.URL) (*Article, error) {
+	// Read the page's own metadata first: extraction modifies doc.
+	meta := readPageMeta(doc, pageURL)
+
+	res, err := trafilatura.ExtractDocument(doc, trafilatura.Options{
+		OriginalURL:     pageURL,
+		EnableFallback:  true,
+		IncludeLinks:    true,
+		IncludeImages:   true,
+		ExcludeComments: true,
+		// The default date search also scans body text and invents dates
+		// from stray numbers (an image width of 2000 became 1 January 2000).
+		// Fast mode only trusts metadata, URLs and date-like markup.
+		HtmlDateMode: trafilatura.Fast,
+	})
+	if err != nil || res.ContentNode == nil ||
+		utf8.RuneCountInString(strings.TrimSpace(res.ContentText)) < minTextLen {
+		return nil, ErrNoContent
+	}
+
+	// go-trafilatura's metadata comes first; the page's own declarations
+	// fill the gaps.
+	m := res.Metadata
+	a := &Article{
+		URL:         pageURL.String(),
+		Title:       first(m.Title, meta.Title, pageURL.Hostname()),
+		Author:      first(m.Author, meta.Author),
+		SiteName:    first(m.Sitename, meta.SiteName),
+		Description: first(m.Description, meta.Description),
+		Language:    languageTag(m.Language),
+		Published:   m.Date,
+		ImageURL:    meta.Image,
+		Content:     res.ContentNode,
+	}
+	if a.Published.IsZero() {
+		a.Published = meta.Published
+	}
+	removeLeadingTitle(a.Content, a.Title)
+	return a, nil
+}
+
+// languageTag returns s as a well-formed BCP 47 tag ("en_US" → "en-US"),
+// or "en" when s is missing or invalid.
+func languageTag(s string) string {
+	tag, err := language.Parse(strings.ReplaceAll(strings.TrimSpace(s), "_", "-"))
+	if err != nil || tag == language.Und {
+		return "en"
+	}
+	return tag.String()
+}
+
+// removeLeadingTitle drops the first element of the content when it is a
+// heading repeating the title, which the book already shows.
+func removeLeadingTitle(content *html.Node, title string) {
+	first := content.FirstChild
+	for first != nil && first.Type == html.TextNode && strings.TrimSpace(first.Data) == "" {
+		first = first.NextSibling
+	}
+	if first == nil || first.Type != html.ElementNode {
+		return
+	}
+	// Match on the tag name: go-trafilatura creates some output elements
+	// without setting DataAtom.
+	switch first.Data {
+	case "h1", "h2":
+		if normalize(textContent(first)) == normalize(title) {
+			content.RemoveChild(first)
+		}
+	}
+}
+
+func textContent(n *html.Node) string {
+	var b strings.Builder
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			b.WriteString(n.Data)
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(n)
+	return b.String()
+}
+
+// normalize lowercases s and collapses whitespace, for comparing titles.
+func normalize(s string) string {
+	return strings.Join(strings.Fields(strings.ToLower(s)), " ")
+}

internal/extract/extract_test.go

@@ -0,0 +1,154 @@
+package extract
+
+import (
+	"errors"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/net/html"
+)
+
+func extractFile(t *testing.T, name, pageURL string) (*Article, error) {
+	t.Helper()
+	data, err := os.ReadFile("../../testdata/pages/" + name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	u, err := url.Parse(pageURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return Extract(parse(t, string(data)), u)
+}
+
+func parse(t *testing.T, page string) *html.Node {
+	t.Helper()
+	doc, err := html.Parse(strings.NewReader(page))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return doc
+}
+
+func render(t *testing.T, n *html.Node) string {
+	t.Helper()
+	var b strings.Builder
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if err := html.Render(&b, c); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return b.String()
+}
+
+func TestExtractBlog(t *testing.T) {
+	a, err := extractFile(t, "blog.html", "https://reading.example.com/posts/eink")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	checks := []struct{ field, got, want string }{
+		{"Title", a.Title, "Why E-Ink Is Easier on the Eyes"},
+		{"Author", a.Author, "Ada Lovelace"},
+		{"SiteName", a.SiteName, "The Reading Desk"},
+		{"Language", a.Language, "en"},
+		{"URL", a.URL, "https://reading.example.com/posts/eink"},
+	}
+	for _, c := range checks {
+		if c.got != c.want {
+			t.Errorf("%s = %q, want %q", c.field, c.got, c.want)
+		}
+	}
+	if !strings.HasPrefix(a.Description, "A look at how electronic paper") {
+		t.Errorf("Description = %q", a.Description)
+	}
+	if want := time.Date(2025, 3, 14, 0, 0, 0, 0, time.UTC); !a.Published.Equal(want) {
+		t.Errorf("Published = %v, want %v", a.Published, want)
+	}
+
+	content := render(t, a.Content)
+	for _, want := range []string{
+		"Electronic paper does not shine light into your eyes.",
+		"<h2>How the display works</h2>",
+		"<blockquote>",
+		"<li>Readable in direct sunlight</li>",
+		"hard to beat &amp; well worth",
+		// Links are made absolute.
+		`<a href="https://reading.example.com/guides/choosing-an-ereader">`,
+	} {
+		if !strings.Contains(content, want) {
+			t.Errorf("content is missing %q", want)
+		}
+	}
+	for _, unwanted := range []string{
+		"Subscribe to our newsletter", // navigation
+		"We use cookies",              // cookie banner
+		"Related posts",               // sidebar
+		"Great article, thanks!",      // comments
+		"All rights reserved",         // footer
+		"window.dataLayer",            // script
+		"<h1>",                        // title repeated in the content
+	} {
+		if strings.Contains(content, unwanted) {
+			t.Errorf("content contains %q", unwanted)
+		}
+	}
+}
+
+func TestExtractNoContent(t *testing.T) {
+	pages := map[string]string{
+		"JavaScript shell": `<!DOCTYPE html><html><head><title>App</title></head>
+			<body><div id="root"></div><script src="/app.js"></script></body></html>`,
+		"too short": `<html><head><title>Short</title></head>
+			<body><article><p>Just a sentence.</p></article></body></html>`,
+		"empty": ``,
+	}
+	u, _ := url.Parse("https://example.com/")
+	for name, page := range pages {
+		if _, err := Extract(parse(t, page), u); !errors.Is(err, ErrNoContent) {
+			t.Errorf("%s: error = %v, want ErrNoContent", name, err)
+		}
+	}
+}
+
+func TestLanguageTag(t *testing.T) {
+	tests := map[string]string{
+		"en": "en", "en-US": "en-US", "en_GB": "en-GB", " fr ": "fr", "pt-br": "pt-BR",
+		"": "en", "not a language": "en", "und": "en",
+	}
+	for in, want := range tests {
+		if got := languageTag(in); got != want {
+			t.Errorf("languageTag(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestRemoveLeadingTitle(t *testing.T) {
+	tests := []struct {
+		name, body, title, want string
+	}{
+		{"matching h1", "<h1>The Title</h1><p>text</p>", "The Title", "<p>text</p>"},
+		{"case and spacing", "\n <h1>  the   TITLE </h1><p>text</p>", "The Title", "<p>text</p>"},
+		{"matching h2", "<h2>The Title</h2><p>text</p>", "The Title", "<p>text</p>"},
+		{"different heading", "<h1>Other</h1><p>text</p>", "The Title", "<h1>Other</h1><p>text</p>"},
+		{"not first", "<p>intro</p><h1>The Title</h1>", "The Title", "<p>intro</p><h1>The Title</h1>"},
+		{"paragraph", "<p>The Title</p>", "The Title", "<p>The Title</p>"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			doc, err := html.Parse(strings.NewReader("<body>" + tt.body + "</body>"))
+			if err != nil {
+				t.Fatal(err)
+			}
+			body := doc.FirstChild.LastChild // html > body
+			removeLeadingTitle(body, tt.title)
+			got := strings.TrimSpace(render(t, body))
+			if got != tt.want {
+				t.Errorf("got %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

internal/extract/meta.go

@@ -0,0 +1,219 @@
+package extract
+
+import (
+	"encoding/json"
+	"net/url"
+	"strings"
+	"time"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// pageMeta is metadata the page declares about itself, used to fill in
+// anything go-trafilatura didn't find, and for the cover image, which it
+// doesn't look for reliably.
+type pageMeta struct {
+	Title, Author, SiteName, Description, Image string
+	Published                                   time.Time
+}
+
+// articleTypes are the schema.org types whose JSON-LD describes an article.
+var articleTypes = map[string]bool{
+	"Article": true, "NewsArticle": true, "BlogPosting": true, "TechArticle": true,
+	"ReportageNewsArticle": true, "AnalysisNewsArticle": true, "OpinionNewsArticle": true,
+	"ScholarlyArticle": true, "Report": true,
+}
+
+// readPageMeta collects Open Graph, article: and Twitter meta tags and
+// schema.org JSON-LD from doc. JSON-LD wins over meta tags, and meta tags
+// over <title>. Relative image URLs are resolved against base.
+func readPageMeta(doc *html.Node, base *url.URL) pageMeta {
+	tags := map[string]string{} // first value of each property/name, lowercased key
+	var title string
+	var ld []map[string]any
+	walk(doc, func(n *html.Node) {
+		switch n.DataAtom {
+		case atom.Meta:
+			key := strings.ToLower(attr(n, "property"))
+			if key == "" {
+				key = strings.ToLower(attr(n, "name"))
+			}
+			if v := strings.TrimSpace(attr(n, "content")); key != "" && v != "" {
+				if _, seen := tags[key]; !seen {
+					tags[key] = v
+				}
+			}
+		case atom.Title:
+			if title == "" {
+				title = strings.TrimSpace(textContent(n))
+			}
+		case atom.Script:
+			if strings.Contains(strings.ToLower(attr(n, "type")), "ld+json") {
+				ld = append(ld, articleObjects(textContent(n))...)
+			}
+		}
+	})
+
+	m := pageMeta{
+		Title:       first(tags["og:title"], tags["twitter:title"], title),
+		Author:      first(tags["article:author"], tags["author"], tags["twitter:creator"]),
+		SiteName:    first(tags["og:site_name"], tags["application-name"]),
+		Description: first(tags["og:description"], tags["description"], tags["twitter:description"]),
+		Image:       first(tags["og:image"], tags["og:image:url"], tags["twitter:image"], tags["twitter:image:src"]),
+		Published:   parseDate(first(tags["article:published_time"], tags["date"], tags["dc.date"])),
+	}
+	// An article:author that is a profile URL isn't a name.
+	if strings.HasPrefix(m.Author, "http") {
+		m.Author = ""
+	}
+
+	for _, o := range ld {
+		if s := str(o["headline"]); s != "" {
+			m.Title = s
+		}
+		if s := names(o["author"]); s != "" {
+			m.Author = s
+		}
+		if s := names(o["publisher"]); s != "" && m.SiteName == "" {
+			m.SiteName = s
+		}
+		if s := str(o["description"]); s != "" && m.Description == "" {
+			m.Description = s
+		}
+		if s := imageURL(o["image"]); s != "" && m.Image == "" {
+			m.Image = s
+		}
+		if t := parseDate(str(o["datePublished"])); !t.IsZero() {
+			m.Published = t
+		}
+	}
+
+	if m.Image != "" {
+		if u, err := base.Parse(m.Image); err == nil && (u.Scheme == "http" || u.Scheme == "https") {
+			m.Image = u.String()
+		} else {
+			m.Image = ""
+		}
+	}
+	return m
+}
+
+// articleObjects returns the article objects in a JSON-LD script, looking
+// inside arrays and @graph. Invalid JSON yields nothing.
+func articleObjects(script string) []map[string]any {
+	var v any
+	if json.Unmarshal([]byte(script), &v) != nil {
+		return nil
+	}
+	var out []map[string]any
+	var visit func(any)
+	visit = func(v any) {
+		switch v := v.(type) {
+		case []any:
+			for _, e := range v {
+				visit(e)
+			}
+		case map[string]any:
+			if isArticle(v["@type"]) {
+				out = append(out, v)
+			}
+			visit(v["@graph"])
+		}
+	}
+	visit(v)
+	return out
+}
+
+func isArticle(t any) bool {
+	switch t := t.(type) {
+	case string:
+		return articleTypes[t]
+	case []any:
+		for _, e := range t {
+			if s, ok := e.(string); ok && articleTypes[s] {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// names reads an author or publisher: a string, an object with a name, or
+// a list of those, joined with ", ".
+func names(v any) string {
+	switch v := v.(type) {
+	case string:
+		return strings.TrimSpace(v)
+	case map[string]any:
+		return str(v["name"])
+	case []any:
+		var out []string
+		for _, e := range v {
+			if s := names(e); s != "" {
+				out = append(out, s)
+			}
+		}
+		return strings.Join(out, ", ")
+	}
+	return ""
+}
+
+// imageURL reads a schema.org image: a URL, an ImageObject, or a list.
+func imageURL(v any) string {
+	switch v := v.(type) {
+	case string:
+		return strings.TrimSpace(v)
+	case map[string]any:
+		return first(str(v["url"]), str(v["contentUrl"]))
+	case []any:
+		for _, e := range v {
+			if s := imageURL(e); s != "" {
+				return s
+			}
+		}
+	}
+	return ""
+}
+
+func str(v any) string {
+	s, _ := v.(string)
+	return strings.TrimSpace(s)
+}
+
+func first(values ...string) string {
+	for _, v := range values {
+		if v = strings.TrimSpace(v); v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+// parseDate accepts the ISO 8601 forms pages use for publication dates.
+func parseDate(s string) time.Time {
+	for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05Z0700", "2006-01-02T15:04:05", "2006-01-02T15:04", "2006-01-02"} {
+		if t, err := time.Parse(layout, strings.TrimSpace(s)); err == nil {
+			return t
+		}
+	}
+	return time.Time{}
+}
+
+func attr(n *html.Node, key string) string {
+	for _, a := range n.Attr {
+		if a.Key == key {
+			return a.Val
+		}
+	}
+	return ""
+}
+
+func walk(n *html.Node, f func(*html.Node)) {
+	if n.Type == html.ElementNode {
+		f(n)
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		walk(c, f)
+	}
+}

internal/extract/meta_test.go

@@ -0,0 +1,93 @@
+package extract
+
+import (
+	"net/url"
+	"testing"
+	"time"
+)
+
+func TestReadPageMetaTags(t *testing.T) {
+	base, _ := url.Parse("https://news.example/2025/story")
+	m := readPageMeta(parse(t, `<html><head>
+		<title>Fallback Title | News</title>
+		<meta property="og:title" content="The Story">
+		<meta property="og:site_name" content="Example News">
+		<meta property="og:image" content="/img/lead.jpg">
+		<meta property="og:image" content="/img/second.jpg">
+		<meta name="author" content="Ada Lovelace">
+		<meta name="description" content="What happened.">
+		<meta property="article:published_time" content="2025-03-14T09:30:00+01:00">
+	</head><body></body></html>`), base)
+
+	want := pageMeta{
+		Title: "The Story", Author: "Ada Lovelace", SiteName: "Example News", Description: "What happened.",
+		Image:     "https://news.example/img/lead.jpg",
+		Published: time.Date(2025, 3, 14, 9, 30, 0, 0, time.FixedZone("", 3600)),
+	}
+	if m.Title != want.Title || m.Author != want.Author || m.SiteName != want.SiteName ||
+		m.Description != want.Description || m.Image != want.Image || !m.Published.Equal(want.Published) {
+		t.Errorf("got  %+v\nwant %+v", m, want)
+	}
+}
+
+func TestReadPageMetaJSONLD(t *testing.T) {
+	base, _ := url.Parse("https://blog.example/post")
+	m := readPageMeta(parse(t, `<html><head>
+		<title>Title Tag</title>
+		<meta property="article:author" content="https://blog.example/authors/grace">
+		<script type="application/ld+json">{"@context":"https://schema.org","@graph":[
+			{"@type":"WebSite","name":"Not an article"},
+			{"@type":["BlogPosting","Article"],"headline":"JSON-LD Headline",
+			 "author":[{"@type":"Person","name":"Grace Hopper"},{"name":"Alan Turing"}],
+			 "publisher":{"@type":"Organization","name":"The Blog"},
+			 "image":{"@type":"ImageObject","url":"https://cdn.example/hero.png"},
+			 "datePublished":"2024-11-02"}
+		]}</script>
+		<script type="application/ld+json">this is not JSON</script>
+	</head><body></body></html>`), base)
+
+	if m.Title != "JSON-LD Headline" {
+		t.Errorf("Title = %q", m.Title)
+	}
+	if m.Author != "Grace Hopper, Alan Turing" {
+		t.Errorf("Author = %q (profile URLs must not count as names)", m.Author)
+	}
+	if m.SiteName != "The Blog" {
+		t.Errorf("SiteName = %q", m.SiteName)
+	}
+	if m.Image != "https://cdn.example/hero.png" {
+		t.Errorf("Image = %q", m.Image)
+	}
+	if !m.Published.Equal(time.Date(2024, 11, 2, 0, 0, 0, 0, time.UTC)) {
+		t.Errorf("Published = %v", m.Published)
+	}
+}
+
+func TestReadPageMetaImages(t *testing.T) {
+	base, _ := url.Parse("https://site.example/a")
+	tests := map[string]string{
+		`<meta name="twitter:image" content="https://site.example/t.jpg">`: "https://site.example/t.jpg",
+		`<meta property="og:image" content="//cdn.example/x.jpg">`:         "https://cdn.example/x.jpg",
+		`<meta property="og:image" content="data:image/png;base64,AAAA">`:  "",
+		`<meta property="og:image" content="javascript:alert(1)">`:         "",
+		``: "",
+	}
+	for head, want := range tests {
+		if got := readPageMeta(parse(t, "<html><head>"+head+"</head></html>"), base).Image; got != want {
+			t.Errorf("%s: Image = %q, want %q", head, got, want)
+		}
+	}
+}
+
+func TestExtractCoverImageAndFallbacks(t *testing.T) {
+	a, err := extractFile(t, "photo-essay.html", "https://letterpress.example/essays/print-shop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a.ImageURL != "https://letterpress.example/img/press.jpg" {
+		t.Errorf("ImageURL = %q", a.ImageURL)
+	}
+	if a.SiteName != "Letterpress Weekly" || a.Author != "Grace Hopper" {
+		t.Errorf("SiteName = %q, Author = %q", a.SiteName, a.Author)
+	}
+}

internal/fetch/errors.go

@@ -0,0 +1,68 @@
+package fetch
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+)
+
+// Explain turns a failed request's error into a message a reader can act
+// on, keeping the original error wrapped. Errors it doesn't recognise are
+// returned unchanged.
+func Explain(err error) error {
+	if err == nil {
+		return nil
+	}
+	var host string
+	var ue *url.Error
+	if errors.As(err, &ue) {
+		if u, perr := url.Parse(ue.URL); perr == nil {
+			host = u.Hostname()
+		}
+	}
+
+	var blocked *BlockedError
+	var dnsErr *net.DNSError
+	var certErr *tls.CertificateVerificationError
+	var unknownAuthority x509.UnknownAuthorityError
+	var hostnameErr x509.HostnameError
+	var invalidCert x509.CertificateInvalidError
+	switch {
+	case errors.As(err, &blocked):
+		return &explained{fmt.Sprintf("%s is a local or private address (%s); rekopub connects to those only when -URL is a local address too", orServer(host), blocked.Addr), err}
+	case errors.As(err, &dnsErr):
+		return &explained{fmt.Sprintf("can't find the server %s; check the address and your internet connection", dnsErr.Name), err}
+	case isRefused(err):
+		return &explained{fmt.Sprintf("%s refused the connection", orServer(host)), err}
+	case errors.As(err, &certErr), errors.As(err, &unknownAuthority), errors.As(err, &hostnameErr), errors.As(err, &invalidCert):
+		return &explained{fmt.Sprintf("%s's security certificate isn't valid, so the connection isn't safe", orServer(host)), err}
+	case isTimeout(err):
+		return &explained{fmt.Sprintf("%s took too long to respond", orServer(host)), err}
+	}
+	return err
+}
+
+func isTimeout(err error) bool {
+	var t interface{ Timeout() bool }
+	return errors.As(err, &t) && t.Timeout()
+}
+
+func orServer(host string) string {
+	if host == "" {
+		return "the server"
+	}
+	return host
+}
+
+// explained is an error with a friendlier message that still unwraps to
+// the original, so errors.Is and errors.As keep working.
+type explained struct {
+	msg string
+	err error
+}
+
+func (e *explained) Error() string { return e.msg }
+func (e *explained) Unwrap() error { return e.err }

internal/fetch/errors_test.go

@@ -0,0 +1,84 @@
+package fetch
+
+import (
+	"context"
+	"crypto/x509"
+	"errors"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+type timeoutErr struct{}
+
+func (timeoutErr) Error() string { return "i/o timeout" }
+func (timeoutErr) Timeout() bool { return true }
+
+func TestExplain(t *testing.T) {
+	wrap := func(inner error) error { return &url.Error{Op: "Get", URL: "https://news.example/story", Err: inner} }
+	tests := []struct {
+		name string
+		err  error
+		want string
+		is   error // must still match with errors.Is, when set
+	}{
+		{"DNS", wrap(&net.DNSError{Name: "news.example", Err: "no such host", IsNotFound: true}),
+			"can't find the server news.example; check the address and your internet connection", nil},
+		{"refused", wrap(&net.OpError{Op: "dial", Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}),
+			"news.example refused the connection", syscall.ECONNREFUSED},
+		{"certificate", wrap(x509.UnknownAuthorityError{}),
+			"news.example's security certificate isn't valid, so the connection isn't safe", nil},
+		{"timeout", wrap(timeoutErr{}), "news.example took too long to respond", nil},
+		{"unknown", wrap(errors.New("something odd")), `Get "https://news.example/story": something odd`, nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := Explain(tt.err)
+			if got.Error() != tt.want {
+				t.Errorf("Explain = %q, want %q", got, tt.want)
+			}
+			if tt.is != nil && !errors.Is(got, tt.is) {
+				t.Errorf("explained error no longer matches %v", tt.is)
+			}
+		})
+	}
+	if Explain(nil) != nil {
+		t.Error("Explain(nil) should be nil")
+	}
+}
+
+func TestFetchRefusedConnection(t *testing.T) {
+	// A port that was just closed refuses connections.
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := ln.Addr().String()
+	ln.Close()
+
+	f := testFetcher()
+	f.Retries = 0
+	_, err = f.Fetch(context.Background(), mustParse(t, "http://"+addr+"/page"))
+	if err == nil || !strings.Contains(err.Error(), "refused the connection") {
+		t.Errorf("error = %v, want a refused-connection message", err)
+	}
+}
+
+func TestStatusErrorHints(t *testing.T) {
+	tests := map[int]string{
+		http.StatusForbidden:       "returned 403 Forbidden; the site may need a login or may block automated downloads",
+		http.StatusNotFound:        "returned 404 Not Found; check the address",
+		http.StatusTooManyRequests: "returned 429 Too Many Requests; the site is limiting requests, so try again later",
+		http.StatusBadGateway:      "returned 502 Bad Gateway",
+	}
+	for code, want := range tests {
+		got := (&StatusError{URL: "https://news.example/story", Code: code}).Error()
+		if got != "https://news.example/story "+want {
+			t.Errorf("%d: %q", code, got)
+		}
+	}
+}

internal/fetch/fetch.go

@@ -0,0 +1,167 @@
+// Package fetch downloads a web page and decodes it to UTF-8.
+package fetch
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"golang.org/x/net/html/charset"
+)
+
+// DefaultUserAgent looks like a desktop browser, because some sites refuse
+// Go's default user agent, and says which tool is asking.
+const DefaultUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36 (compatible; rekopub)"
+
+// Page is a downloaded HTML page.
+type Page struct {
+	URL         *url.URL // final URL, after redirects
+	HTML        string   // decoded to UTF-8
+	ContentType string   // as sent by the server, or sniffed
+}
+
+// StatusError reports a response with a non-2xx status code.
+type StatusError struct {
+	URL  string
+	Code int
+}
+
+func (e *StatusError) Error() string {
+	msg := fmt.Sprintf("%s returned %d %s", e.URL, e.Code, http.StatusText(e.Code))
+	switch e.Code {
+	case http.StatusUnauthorized, http.StatusForbidden:
+		msg += "; the site may need a login or may block automated downloads"
+	case http.StatusNotFound, http.StatusGone:
+		msg += "; check the address"
+	case http.StatusTooManyRequests:
+		msg += "; the site is limiting requests, so try again later"
+	}
+	return msg
+}
+
+// Fetcher downloads pages. The zero value is not usable; call New.
+type Fetcher struct {
+	Client    *http.Client
+	UserAgent string
+	MaxBytes  int64         // largest page accepted
+	Retries   int           // extra attempts after a 429, 5xx or network error
+	Backoff   time.Duration // wait before the first retry; doubles each time
+}
+
+// New returns a Fetcher with the defaults from PLAN.md: a 60 second
+// timeout, a 20 MB size limit and two retries. Its client refuses local and
+// private addresses unless allowLocal is set; see IsLocal.
+func New(allowLocal bool) *Fetcher {
+	return &Fetcher{
+		Client:    newClient(allowLocal),
+		UserAgent: DefaultUserAgent,
+		MaxBytes:  20 << 20,
+		Retries:   2,
+		Backoff:   time.Second,
+	}
+}
+
+// maxRetryAfter caps how long a server's Retry-After header can make us wait.
+const maxRetryAfter = 30 * time.Second
+
+// Fetch downloads u and returns it decoded to UTF-8. It fails for non-2xx
+// responses, for content that isn't HTML and for pages over MaxBytes.
+func (f *Fetcher) Fetch(ctx context.Context, u *url.URL) (*Page, error) {
+	wait := f.Backoff
+	for attempt := 0; ; attempt++ {
+		page, retryAfter, err := f.fetchOnce(ctx, u)
+		if err == nil || retryAfter < 0 || attempt >= f.Retries {
+			return page, err
+		}
+		if retryAfter > 0 {
+			wait = min(retryAfter, maxRetryAfter)
+		}
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(wait):
+		}
+		wait *= 2
+	}
+}
+
+// fetchOnce makes a single attempt. retryAfter is negative when the error
+// is not worth retrying, zero to use the normal backoff, or the delay the
+// server asked for.
+func (f *Fetcher) fetchOnce(ctx context.Context, u *url.URL) (page *Page, retryAfter time.Duration, err error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, -1, err
+	}
+	req.Header.Set("User-Agent", f.UserAgent)
+	req.Header.Set("Accept", "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5")
+
+	resp, err := f.Client.Do(req)
+	if err != nil {
+		if ctx.Err() != nil {
+			return nil, -1, ctx.Err()
+		}
+		return nil, 0, Explain(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		err := &StatusError{URL: u.String(), Code: resp.StatusCode}
+		if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
+			return nil, parseRetryAfter(resp.Header.Get("Retry-After")), err
+		}
+		return nil, -1, err
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, f.MaxBytes+1))
+	if err != nil {
+		return nil, 0, fmt.Errorf("reading %s: %w", u, err)
+	}
+	if int64(len(body)) > f.MaxBytes {
+		return nil, -1, fmt.Errorf("%s is larger than %d MB", u, f.MaxBytes>>20)
+	}
+
+	// Sniff the content when the server sends no type or only the generic
+	// application/octet-stream, as some misconfigured servers do for HTML.
+	contentType := resp.Header.Get("Content-Type")
+	if mediaType, _, _ := mime.ParseMediaType(contentType); mediaType == "" || mediaType == "application/octet-stream" {
+		contentType = http.DetectContentType(body)
+	}
+	if !isHTML(contentType) {
+		return nil, -1, fmt.Errorf("%s is not a web page (content type %q)", u, contentType)
+	}
+
+	// charset.NewReader picks the encoding from the Content-Type header, a
+	// byte order mark or a <meta charset> tag, then converts to UTF-8.
+	r, err := charset.NewReader(bytes.NewReader(body), contentType)
+	if err != nil {
+		return nil, -1, fmt.Errorf("decoding %s: %w", u, err)
+	}
+	decoded, err := io.ReadAll(r)
+	if err != nil {
+		return nil, -1, fmt.Errorf("decoding %s: %w", u, err)
+	}
+
+	return &Page{URL: resp.Request.URL, HTML: string(decoded), ContentType: contentType}, 0, nil
+}
+
+func isHTML(contentType string) bool {
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	return err == nil && (mediaType == "text/html" || mediaType == "application/xhtml+xml")
+}
+
+// parseRetryAfter reads a Retry-After header given in seconds. HTTP dates
+// and missing values fall back to the normal backoff.
+func parseRetryAfter(v string) time.Duration {
+	secs, err := strconv.Atoi(v)
+	if err != nil || secs < 0 {
+		return 0
+	}
+	return time.Duration(secs) * time.Second
+}

internal/fetch/fetch_test.go

@@ -0,0 +1,264 @@
+package fetch
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// testFetcher returns a Fetcher with retries that don't slow tests down.
+func testFetcher() *Fetcher {
+	f := New(true)
+	f.Backoff = time.Millisecond
+	return f
+}
+
+func serve(t *testing.T, h http.HandlerFunc) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(h)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func mustParse(t *testing.T, raw string) *url.URL {
+	t.Helper()
+	u, err := url.Parse(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return u
+}
+
+func TestFetchHTML(t *testing.T) {
+	var gotUA, gotAccept string
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		gotUA, gotAccept = r.UserAgent(), r.Header.Get("Accept")
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		w.Write([]byte("<p>Héllo</p>"))
+	})
+
+	page, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL+"/a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if page.HTML != "<p>Héllo</p>" {
+		t.Errorf("HTML = %q", page.HTML)
+	}
+	if page.URL.String() != srv.URL+"/a" {
+		t.Errorf("URL = %q", page.URL)
+	}
+	if !strings.Contains(gotUA, "rekopub") {
+		t.Errorf("User-Agent = %q", gotUA)
+	}
+	if !strings.HasPrefix(gotAccept, "text/html") {
+		t.Errorf("Accept = %q", gotAccept)
+	}
+}
+
+func TestFetchFollowsRedirects(t *testing.T) {
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/old" {
+			http.Redirect(w, r, "/new", http.StatusMovedPermanently)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html")
+		w.Write([]byte("<p>moved</p>"))
+	})
+
+	page, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL+"/old"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if page.URL.Path != "/new" {
+		t.Errorf("final URL = %q, want path /new", page.URL)
+	}
+}
+
+func TestFetchNotFoundIsNotRetried(t *testing.T) {
+	var calls atomic.Int32
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		calls.Add(1)
+		http.NotFound(w, r)
+	})
+
+	_, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL))
+	var se *StatusError
+	if !errors.As(err, &se) || se.Code != http.StatusNotFound {
+		t.Fatalf("error = %v, want a 404 StatusError", err)
+	}
+	if !strings.Contains(err.Error(), "returned 404 Not Found") {
+		t.Errorf("error = %q", err)
+	}
+	if n := calls.Load(); n != 1 {
+		t.Errorf("server called %d times, want 1", n)
+	}
+}
+
+func TestFetchRetries(t *testing.T) {
+	for _, code := range []int{http.StatusTooManyRequests, http.StatusServiceUnavailable} {
+		var calls atomic.Int32
+		srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+			if calls.Add(1) < 3 {
+				w.Header().Set("Retry-After", "0")
+				w.WriteHeader(code)
+				return
+			}
+			w.Header().Set("Content-Type", "text/html")
+			w.Write([]byte("<p>ok</p>"))
+		})
+
+		page, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL))
+		if err != nil {
+			t.Fatalf("%d: %v", code, err)
+		}
+		if page.HTML != "<p>ok</p>" || calls.Load() != 3 {
+			t.Errorf("%d: HTML = %q after %d calls, want ok after 3", code, page.HTML, calls.Load())
+		}
+	}
+}
+
+func TestFetchGivesUpAfterRetries(t *testing.T) {
+	var calls atomic.Int32
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		calls.Add(1)
+		w.WriteHeader(http.StatusInternalServerError)
+	})
+
+	_, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL))
+	var se *StatusError
+	if !errors.As(err, &se) || se.Code != http.StatusInternalServerError {
+		t.Fatalf("error = %v, want a 500 StatusError", err)
+	}
+	if n := calls.Load(); n != 3 {
+		t.Errorf("server called %d times, want 3 (1 + 2 retries)", n)
+	}
+}
+
+func TestFetchRejectsNonHTML(t *testing.T) {
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/pdf")
+		w.Write([]byte("%PDF-1.7"))
+	})
+
+	_, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL))
+	if err == nil || !strings.Contains(err.Error(), "not a web page") {
+		t.Fatalf("error = %v, want a not-a-web-page error", err)
+	}
+}
+
+func TestFetchSniffsMissingContentType(t *testing.T) {
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		w.Header()["Content-Type"] = nil // stop net/http from adding one
+		w.Write([]byte("<!DOCTYPE html><html><body><p>sniffed</p></body></html>"))
+	})
+
+	page, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(page.HTML, "sniffed") {
+		t.Errorf("HTML = %q", page.HTML)
+	}
+}
+
+func TestFetchSniffsOctetStream(t *testing.T) {
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/octet-stream")
+		if r.URL.Path == "/page" {
+			w.Write([]byte("<!DOCTYPE html><html><body><p>sniffed</p></body></html>"))
+		} else {
+			w.Write([]byte("%PDF-1.7 binary"))
+		}
+	})
+
+	page, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL+"/page"))
+	if err != nil || !strings.Contains(page.HTML, "sniffed") {
+		t.Errorf("HTML page: err = %v", err)
+	}
+	if _, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL+"/file")); err == nil {
+		t.Error("binary file was accepted as a web page")
+	}
+}
+
+func TestFetchRejectsLargePages(t *testing.T) {
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		w.Write([]byte(strings.Repeat("x", 2<<20)))
+	})
+
+	f := testFetcher()
+	f.MaxBytes = 1 << 20
+	_, err := f.Fetch(context.Background(), mustParse(t, srv.URL))
+	if err == nil || !strings.Contains(err.Error(), "larger than 1 MB") {
+		t.Fatalf("error = %v, want a size error", err)
+	}
+}
+
+func TestFetchDecodesCharsets(t *testing.T) {
+	tests := []struct {
+		name        string
+		contentType string
+		body        []byte
+		want        string
+	}{
+		{
+			name:        "ISO-8859-1 from header",
+			contentType: "text/html; charset=iso-8859-1",
+			body:        []byte("<p>caf\xe9</p>"),
+			want:        "<p>café</p>",
+		},
+		{
+			name:        "Shift_JIS from meta tag",
+			contentType: "text/html",
+			body:        append([]byte(`<html><head><meta charset="shift_jis"></head><body><p>`), append([]byte("\x93\xfa\x96\x7b"), []byte("</p></body></html>")...)...),
+			want:        "<p>日本</p>",
+		},
+		{
+			name:        "windows-1252 quotes",
+			contentType: "text/html; charset=windows-1252",
+			body:        []byte("<p>\x93quoted\x94</p>"),
+			want:        "<p>“quoted”</p>",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", tt.contentType)
+				w.Write(tt.body)
+			})
+			page, err := testFetcher().Fetch(context.Background(), mustParse(t, srv.URL))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !strings.Contains(page.HTML, tt.want) {
+				t.Errorf("HTML = %q, want it to contain %q", page.HTML, tt.want)
+			}
+		})
+	}
+}
+
+func TestFetchCancelled(t *testing.T) {
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusServiceUnavailable)
+	})
+
+	f := testFetcher()
+	f.Backoff = time.Hour // cancellation must interrupt the wait
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	_, err := f.Fetch(ctx, mustParse(t, srv.URL))
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("error = %v, want context.DeadlineExceeded", err)
+	}
+	if time.Since(start) > 5*time.Second {
+		t.Errorf("Fetch took %v after cancellation", time.Since(start))
+	}
+}

internal/fetch/guard.go

@@ -0,0 +1,149 @@
+package fetch
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"syscall"
+	"time"
+
+	"golang.org/x/net/http/httpproxy"
+)
+
+// A web page decides which other addresses rekopub connects to: it can
+// redirect, and it names its images and cover. Without a guard, a page on
+// the internet could make rekopub send requests to the reader's own
+// computer or to devices on their network, such as a router's admin page.
+// So when the page itself is on the internet, connections to local and
+// private addresses are refused. The check is made on the address actually
+// dialled, after DNS lookup, so a hostname that resolves to a local address
+// is caught too.
+
+// BlockedError reports a refused connection to a local or private address.
+type BlockedError struct {
+	Addr netip.Addr
+}
+
+func (e *BlockedError) Error() string {
+	return fmt.Sprintf("connection to %s refused: it is a local or private address", e.Addr)
+}
+
+// IsLocal reports whether host is, or resolves to, an address that isn't on
+// the public internet, such as localhost or a server on the home network.
+// Local connections are allowed only when the page itself is local.
+func IsLocal(ctx context.Context, host string) bool {
+	if ip, err := netip.ParseAddr(host); err == nil {
+		return !isPublic(ip)
+	}
+	addrs, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host)
+	if err != nil {
+		return false
+	}
+	for _, ip := range addrs {
+		if !isPublic(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// nonPublic lists ranges that aren't on the public internet and that
+// netip.Addr's methods don't cover.
+var nonPublic = []netip.Prefix{
+	netip.MustParsePrefix("0.0.0.0/8"),     // "this network"
+	netip.MustParsePrefix("100.64.0.0/10"), // carrier-grade NAT, also used by Tailscale
+	netip.MustParsePrefix("192.0.0.0/24"),  // IETF protocol assignments
+	netip.MustParsePrefix("198.18.0.0/15"), // network benchmarking
+	netip.MustParsePrefix("240.0.0.0/4"),   // reserved
+}
+
+// isPublic reports whether ip is a public internet address. Loopback,
+// private, link-local (which includes cloud metadata services), multicast
+// and reserved addresses are not.
+func isPublic(ip netip.Addr) bool {
+	ip = ip.Unmap()
+	if !ip.IsGlobalUnicast() || ip.IsPrivate() {
+		return false
+	}
+	for _, p := range nonPublic {
+		if p.Contains(ip) {
+			return false
+		}
+	}
+	return true
+}
+
+// permitted decides whether a guarded client may connect to an address.
+// Tests replace it, because their servers are all on localhost.
+var permitted = func(ap netip.AddrPort) bool { return isPublic(ap.Addr()) }
+
+// newClient returns the HTTP client for pages and images. Unless
+// allowLocal is set, it refuses to connect to local and private addresses.
+func newClient(allowLocal bool) *http.Client {
+	t := http.DefaultTransport.(*http.Transport).Clone()
+	if !allowLocal {
+		t.DialContext = guardedDial(proxyAddrs())
+	}
+	return &http.Client{Timeout: 60 * time.Second, Transport: t}
+}
+
+// guardedDial returns a dial function that refuses local and private
+// addresses, except for the proxies in proxies: a proxy is the user's own
+// choice and is often on their network. Requests sent through a proxy are
+// dialled by the proxy, so the guard can't check them.
+func guardedDial(proxies map[string]bool) func(ctx context.Context, network, addr string) (net.Conn, error) {
+	// The same settings as http.DefaultTransport.
+	plain := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}
+	guarded := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second,
+		Control: func(_, address string, _ syscall.RawConn) error {
+			ap, err := netip.ParseAddrPort(address)
+			if err != nil {
+				return err
+			}
+			if !permitted(ap) {
+				return &BlockedError{Addr: ap.Addr().Unmap()}
+			}
+			return nil
+		}}
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		if proxies[addr] {
+			return plain.DialContext(ctx, network, addr)
+		}
+		return guarded.DialContext(ctx, network, addr)
+	}
+}
+
+// proxyAddrs returns the host:port of the proxies set in the environment
+// (HTTP_PROXY and HTTPS_PROXY), which http.DefaultTransport uses.
+func proxyAddrs() map[string]bool {
+	cfg := httpproxy.FromEnvironment()
+	addrs := map[string]bool{}
+	for _, raw := range []string{cfg.HTTPProxy, cfg.HTTPSProxy} {
+		if raw == "" {
+			continue
+		}
+		u, err := url.Parse(raw)
+		if err != nil || u.Host == "" {
+			// A proxy may be given without a scheme, as "proxy:3128".
+			if u, err = url.Parse("http://" + raw); err != nil {
+				continue
+			}
+		}
+		port := u.Port()
+		if port == "" {
+			switch u.Scheme {
+			case "https":
+				port = "443"
+			case "socks5", "socks5h":
+				port = "1080"
+			default:
+				port = "80"
+			}
+		}
+		addrs[net.JoinHostPort(u.Hostname(), port)] = true
+	}
+	return addrs
+}

internal/fetch/guard_test.go

@@ -0,0 +1,124 @@
+package fetch
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/netip"
+	"strings"
+	"sync/atomic"
+	"testing"
+)
+
+func TestIsPublic(t *testing.T) {
+	tests := map[string]bool{
+		"93.184.215.14":        true,
+		"2606:4700:4700::1111": true,
+		"127.0.0.1":            false, // loopback
+		"::1":                  false,
+		"10.1.2.3":             false, // private networks
+		"172.16.0.1":           false,
+		"192.168.1.1":          false,
+		"fd12:3456::1":         false,
+		"169.254.169.254":      false, // link-local: cloud metadata services
+		"fe80::1":              false,
+		"100.100.1.1":          false, // carrier-grade NAT, Tailscale
+		"0.0.0.0":              false,
+		"::":                   false,
+		"224.0.0.1":            false, // multicast
+		"255.255.255.255":      false,
+		"::ffff:192.168.1.1":   false, // IPv4-mapped
+	}
+	for addr, want := range tests {
+		if got := isPublic(netip.MustParseAddr(addr)); got != want {
+			t.Errorf("isPublic(%s) = %v, want %v", addr, got, want)
+		}
+	}
+}
+
+func TestIsLocal(t *testing.T) {
+	ctx := context.Background()
+	for _, host := range []string{"localhost", "127.0.0.1", "::1", "192.168.1.20", "169.254.169.254"} {
+		if !IsLocal(ctx, host) {
+			t.Errorf("IsLocal(%q) = false, want true", host)
+		}
+	}
+	for _, host := range []string{"93.184.215.14", "2606:4700:4700::1111", "name.invalid"} {
+		if IsLocal(ctx, host) {
+			t.Errorf("IsLocal(%q) = true, want false", host)
+		}
+	}
+}
+
+func TestGuardBlocksLocalPage(t *testing.T) {
+	var hits atomic.Int32
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) { hits.Add(1) })
+
+	f := New(false)
+	f.Retries = 0
+	_, err := f.Fetch(context.Background(), mustParse(t, srv.URL+"/page"))
+	var blocked *BlockedError
+	if !errors.As(err, &blocked) {
+		t.Fatalf("error = %v, want a BlockedError", err)
+	}
+	if !strings.Contains(err.Error(), "is a local or private address (127.0.0.1)") {
+		t.Errorf("error = %q, want an explanation", err)
+	}
+	if hits.Load() != 0 {
+		t.Errorf("server received %d requests, want none", hits.Load())
+	}
+}
+
+func TestGuardBlocksRedirectToLocal(t *testing.T) {
+	var localHits atomic.Int32
+	local := serve(t, func(w http.ResponseWriter, r *http.Request) { localHits.Add(1) })
+	public := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, local.URL+"/admin", http.StatusFound)
+	})
+	// Treat the "public" server as being on the internet.
+	publicAddr := netip.MustParseAddrPort(strings.TrimPrefix(public.URL, "http://"))
+	defer func(old func(netip.AddrPort) bool) { permitted = old }(permitted)
+	permitted = func(ap netip.AddrPort) bool { return ap == publicAddr }
+
+	f := New(false)
+	f.Retries = 0
+	_, err := f.Fetch(context.Background(), mustParse(t, public.URL+"/story"))
+	var blocked *BlockedError
+	if !errors.As(err, &blocked) {
+		t.Fatalf("error = %v, want a BlockedError", err)
+	}
+	if localHits.Load() != 0 {
+		t.Errorf("local server received %d requests, want none", localHits.Load())
+	}
+}
+
+func TestFetchRefusesFileRedirect(t *testing.T) {
+	srv := serve(t, func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "file:///etc/hosts", http.StatusFound)
+	})
+	f := testFetcher()
+	f.Retries = 0
+	page, err := f.Fetch(context.Background(), mustParse(t, srv.URL+"/page"))
+	if err == nil || page != nil {
+		t.Fatalf("Fetch followed a redirect to a local file: page = %+v, err = %v", page, err)
+	}
+	if !strings.Contains(err.Error(), "unsupported protocol scheme") {
+		t.Errorf("error = %v", err)
+	}
+}
+
+func TestProxyAddrs(t *testing.T) {
+	// Set both spellings: which one wins depends on the x/net version.
+	for _, name := range []string{"HTTP_PROXY", "http_proxy"} {
+		t.Setenv(name, "http://10.0.0.2")
+	}
+	for _, name := range []string{"HTTPS_PROXY", "https_proxy"} {
+		t.Setenv(name, "proxy.lan:3128")
+	}
+	got := proxyAddrs()
+	for _, want := range []string{"10.0.0.2:80", "proxy.lan:3128"} {
+		if !got[want] {
+			t.Errorf("proxyAddrs() = %v, missing %s", got, want)
+		}
+	}
+}

internal/fetch/refused_other.go

@@ -0,0 +1,10 @@
+//go:build !windows
+
+package fetch
+
+import (
+	"errors"
+	"syscall"
+)
+
+func isRefused(err error) bool { return errors.Is(err, syscall.ECONNREFUSED) }

internal/fetch/refused_windows.go

@@ -0,0 +1,15 @@
+package fetch
+
+import (
+	"errors"
+	"syscall"
+)
+
+// wsaeconnrefused is Winsock's "connection refused" error, which Windows
+// returns instead of syscall.ECONNREFUSED. The syscall package doesn't
+// define it.
+const wsaeconnrefused syscall.Errno = 10061
+
+func isRefused(err error) bool {
+	return errors.Is(err, wsaeconnrefused) || errors.Is(err, syscall.ECONNREFUSED)
+}

internal/kepub/kepub.go

@@ -0,0 +1,36 @@
+// Package kepub converts a finished EPUB into a Kobo KEPUB.
+//
+// Kobo devices open files ending in .kepub.epub with their own WebKit-based
+// renderer instead of Adobe's RMSDK, which gives faster page turns, reading
+// statistics and more reliable highlights. kepubify does the conversion: it
+// wraps each sentence in a koboSpan (which the statistics and highlights
+// need) and adds Kobo's style hooks.
+package kepub
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/pgaskin/kepubify/v4/kepub"
+)
+
+// Convert returns epub converted to a KEPUB.
+func Convert(ctx context.Context, epub []byte) ([]byte, error) {
+	zr, err := zip.NewReader(bytes.NewReader(epub), int64(len(epub)))
+	if err != nil {
+		return nil, fmt.Errorf("reading the EPUB: %w", err)
+	}
+	c := kepub.NewConverterWithOptions(
+		// Hyphenation helps justified text on the Clara HD's 6" screen.
+		kepub.ConverterOptionHyphenate(true),
+		// Smartypants is left off: it rewrites quotes and dashes, which
+		// would change the article's text.
+	)
+	var out bytes.Buffer
+	if err := c.Convert(ctx, &out, zr); err != nil {
+		return nil, fmt.Errorf("converting to KEPUB: %w", err)
+	}
+	return out.Bytes(), nil
+}

internal/kepub/kepub_test.go

@@ -0,0 +1,120 @@
+package kepub
+
+import (
+	"bytes"
+	"context"
+	"image"
+	"image/jpeg"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/net/html"
+
+	"git.monoexp.com/bajdev/rekopub/internal/build"
+	"git.monoexp.com/bajdev/rekopub/internal/clean"
+	"git.monoexp.com/bajdev/rekopub/internal/epubtest"
+	"git.monoexp.com/bajdev/rekopub/internal/extract"
+)
+
+// testEPUB builds a small plain EPUB with a cover and two sections.
+func testEPUB(t *testing.T) []byte {
+	t.Helper()
+	doc, err := html.Parse(strings.NewReader(`<body><p>First sentence here. Second sentence follows it!</p>
+		<h2>A section</h2><p>Text in the section, with <em>emphasis</em> and a <a href="https://example.com/x">link</a>.</p>
+		<h2>Another</h2><ul><li>One item.</li><li>Two items.</li></ul></body>`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := &extract.Article{
+		URL: "https://example.com/post", Title: "KEPUB Test", Author: "Ada Lovelace",
+		SiteName: "Example", Language: "en", Published: time.Date(2025, 1, 2, 0, 0, 0, 0, time.UTC),
+		Content: doc.FirstChild.LastChild,
+	}
+	u, _ := url.Parse(a.URL)
+	body, err := clean.Content(a.Content, u)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var cover bytes.Buffer
+	if err := jpeg.Encode(&cover, image.NewGray(image.Rect(0, 0, 30, 40)), nil); err != nil {
+		t.Fatal(err)
+	}
+	data, err := build.Build(a, body, nil, cover.Bytes())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return data
+}
+
+var tags = regexp.MustCompile(`<[^>]+>`)
+
+// bodyText returns the visible text of a chapter's <body>, with whitespace
+// collapsed.
+func bodyText(chapter string) string {
+	if i := strings.Index(chapter, "<body"); i >= 0 {
+		chapter = chapter[i:]
+	}
+	return strings.Join(strings.Fields(tags.ReplaceAllString(chapter, " ")), " ")
+}
+
+func TestConvert(t *testing.T) {
+	plain := testEPUB(t)
+	data, err := Convert(context.Background(), plain)
+	if err != nil {
+		t.Fatal(err)
+	}
+	book := epubtest.Open(t, data) // also checks mimetype is first and stored
+	book.AssertWellFormed(t)
+	before := epubtest.Open(t, plain)
+
+	chapter := book.File(t, "chapter.xhtml")
+	if !strings.Contains(chapter, `<span class="koboSpan" id="kobo.1.1">`) {
+		t.Errorf("chapter has no koboSpan elements:\n%s", chapter)
+	}
+	// Sentences are spanned separately, so Kobo can count and highlight them.
+	if !strings.Contains(chapter, `>First sentence here. </span><span class="koboSpan"`) {
+		t.Errorf("sentences aren't split into spans:\n%s", chapter)
+	}
+	if !strings.Contains(chapter, "hyphens: auto") {
+		t.Errorf("hyphenation isn't enabled")
+	}
+	// The words must be exactly the same, just wrapped.
+	if got, want := bodyText(chapter), bodyText(before.File(t, "chapter.xhtml")); got != want {
+		t.Errorf("text changed by conversion:\n got %q\nwant %q", got, want)
+	}
+
+	opf := book.File(t, ".opf")
+	for _, want := range []string{`properties="cover-image"`, "<dc:publisher>Example</dc:publisher>", "<dc:source>https://example.com/post</dc:source>"} {
+		if !strings.Contains(opf, want) {
+			t.Errorf("OPF lost %q:\n%s", want, opf)
+		}
+	}
+	if !strings.Contains(book.File(t, "nav.xhtml"), "#sec-2") || !strings.Contains(book.File(t, "toc.ncx"), "dtb:uid") {
+		t.Errorf("table of contents changed by conversion")
+	}
+
+	path := filepath.Join(t.TempDir(), "book.kepub.epub")
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	epubtest.EPUBCheck(t, path)
+}
+
+func TestConvertRejectsNonEPUB(t *testing.T) {
+	if _, err := Convert(context.Background(), []byte("not a zip file")); err == nil {
+		t.Error("Convert accepted data that isn't an EPUB")
+	}
+}
+
+func TestConvertCancelled(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := Convert(ctx, testEPUB(t)); err == nil {
+		t.Error("Convert succeeded after cancellation")
+	}
+}

internal/pipeline/pipeline.go

@@ -0,0 +1,160 @@
+// Package pipeline turns a web page into a book: fetch, extract, clean,
+// process images and build the EPUB.
+package pipeline
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"image"
+	"io"
+	"net/url"
+	"strings"
+
+	"golang.org/x/net/html"
+
+	"git.monoexp.com/bajdev/rekopub/internal/assets"
+	"git.monoexp.com/bajdev/rekopub/internal/build"
+	"git.monoexp.com/bajdev/rekopub/internal/clean"
+	"git.monoexp.com/bajdev/rekopub/internal/cover"
+	"git.monoexp.com/bajdev/rekopub/internal/extract"
+	"git.monoexp.com/bajdev/rekopub/internal/fetch"
+	"git.monoexp.com/bajdev/rekopub/internal/kepub"
+)
+
+// Errors that Run wraps, so callers can choose an exit code.
+var (
+	ErrFetch     = errors.New("couldn't fetch the page")
+	ErrNoContent = extract.ErrNoContent
+)
+
+// Book is a finished book, ready to be saved.
+type Book struct {
+	// Title is the article title, used to name the file for folder targets.
+	Title string
+	// Content writes the complete EPUB.
+	Content io.WriterTo
+}
+
+// Options adjust a single conversion.
+type Options struct {
+	// BeforeImages, if set, is called with the article title once it is
+	// known and before any image is downloaded. An error stops the
+	// conversion and is returned unchanged. The CLI uses it to stop early
+	// when the output file already exists.
+	BeforeImages func(title string) error
+	// Warn, if set, receives problems that don't stop the conversion, such
+	// as an image that couldn't be downloaded.
+	Warn func(msg string)
+	// KEPUB converts the finished book to a Kobo KEPUB.
+	KEPUB bool
+}
+
+// Run converts the page at u into a book using the default settings.
+// Images and redirects may lead to local and private addresses only when
+// u itself is local.
+func Run(ctx context.Context, u *url.URL, opts Options) (*Book, error) {
+	return New(fetch.IsLocal(ctx, u.Hostname())).Run(ctx, u, opts)
+}
+
+// Pipeline holds the settings for conversions.
+type Pipeline struct {
+	Fetcher *fetch.Fetcher
+	Images  *assets.Processor
+}
+
+// New returns a Pipeline with the default settings. Pages and images share
+// one HTTP client, which connects to local and private addresses only if
+// allowLocal is set.
+func New(allowLocal bool) Pipeline {
+	f := fetch.New(allowLocal)
+	return Pipeline{Fetcher: f, Images: assets.New(f.Client, f.UserAgent)}
+}
+
+// Run converts the page at u into a book.
+func (p Pipeline) Run(ctx context.Context, u *url.URL, opts Options) (*Book, error) {
+	page, err := p.Fetcher.Fetch(ctx, u)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %w", ErrFetch, err)
+	}
+
+	doc, err := html.Parse(strings.NewReader(page.HTML))
+	if err != nil {
+		return nil, fmt.Errorf("parsing the page: %w", err)
+	}
+	clean.Prepare(doc, page.URL)
+	article, err := extract.Extract(doc, page.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	if opts.BeforeImages != nil {
+		if err := opts.BeforeImages(article.Title); err != nil {
+			return nil, err
+		}
+	}
+
+	body, err := clean.Content(article.Content, page.URL)
+	if err != nil {
+		return nil, fmt.Errorf("cleaning the article: %w", err)
+	}
+	images := *p.Images
+	images.Warn = opts.Warn
+	imgs, err := images.Process(ctx, body, page.URL)
+	if err != nil {
+		return nil, fmt.Errorf("processing images: %w", err)
+	}
+
+	photo, err := coverPhoto(ctx, &images, article, page.URL)
+	if err != nil {
+		return nil, fmt.Errorf("processing images: %w", err)
+	}
+	coverJPEG, err := cover.Render(cover.Details{
+		Title:    article.Title,
+		SiteName: article.SiteName,
+		Date:     build.DisplayDate(article.Published),
+	}, photo, images.Profile)
+	if err != nil {
+		return nil, fmt.Errorf("drawing the cover: %w", err)
+	}
+
+	data, err := build.Build(article, body, imgs, coverJPEG)
+	if err != nil {
+		return nil, fmt.Errorf("building the book: %w", err)
+	}
+	if opts.KEPUB {
+		if data, err = kepub.Convert(ctx, data); err != nil {
+			return nil, err
+		}
+	}
+	return &Book{Title: article.Title, Content: bytes.NewReader(data)}, nil
+}
+
+// minCoverPhoto is the smallest side, in pixels, of a preview image worth
+// putting on the cover. Smaller ones are usually site logos, and a plain
+// typographic cover looks better than a blown-up logo.
+const minCoverPhoto = 300
+
+// coverPhoto downloads the page's preview image for the cover. It returns
+// nil, and the cover is typographic, when there is no usable image; only
+// cancellation is an error.
+func coverPhoto(ctx context.Context, images *assets.Processor, a *extract.Article, page *url.URL) (image.Image, error) {
+	if a.ImageURL == "" {
+		return nil, nil
+	}
+	img, err := images.Decode(ctx, a.ImageURL, page)
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return nil, ctxErr
+	}
+	if err != nil {
+		if images.Warn != nil {
+			images.Warn(fmt.Sprintf("cover image %s skipped, using a plain cover: %v", a.ImageURL, err))
+		}
+		return nil, nil
+	}
+	if b := img.Bounds(); min(b.Dx(), b.Dy()) < minCoverPhoto {
+		return nil, nil
+	}
+	return img, nil
+}

internal/save/save.go

@@ -0,0 +1,247 @@
+// Package save decides where a book is written and writes it there safely.
+//
+// It implements the -SAVE rules from the README: folder versus file targets,
+// the .epub / .kepub.epub extension, creation of missing folders, and
+// overwrite protection unless -FORCE is given. Writes are atomic, so a failed
+// run never leaves a partial file behind.
+package save
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/unicode/norm"
+)
+
+const (
+	extEPUB  = ".epub"
+	extKEPUB = ".kepub.epub"
+
+	// maxSlugLen caps generated filenames well below filesystem limits.
+	maxSlugLen = 80
+
+	// fallbackName is used when a title produces an empty slug, for example
+	// a title written entirely in a non-Latin script.
+	fallbackName = "article"
+)
+
+// ErrExists reports that the output file already exists and overwriting was
+// not allowed.
+var ErrExists = errors.New("already exists")
+
+// ErrKEPUBName reports a .kepub.epub output path without KEPUB output
+// enabled. Kobo would open such a plain EPUB in its KEPUB renderer.
+var ErrKEPUBName = errors.New("a .kepub.epub file needs -KEPUB (or use a .epub filename)")
+
+// Target is a resolved -SAVE value. The final filename is known up front for
+// file targets; for folder targets it depends on the article title.
+type Target struct {
+	file string // absolute file path, set for file targets
+	dir  string // absolute folder, set for folder targets
+	ext  string // extension used for folder targets
+
+	// Note is a message for the user about an adjustment made to the path,
+	// or empty.
+	Note string
+}
+
+// Resolve interprets the -SAVE argument. kepub selects the .kepub.epub
+// extension.
+func Resolve(arg string, kepub bool) (Target, error) {
+	if arg == "" {
+		return Target{}, errors.New("empty path")
+	}
+	arg, err := expandHome(arg)
+	if err != nil {
+		return Target{}, err
+	}
+
+	ext := extEPUB
+	if kepub {
+		ext = extKEPUB
+	}
+
+	if isFolder(arg) {
+		dir, err := filepath.Abs(arg)
+		if err != nil {
+			return Target{}, err
+		}
+		return Target{dir: dir, ext: ext}, nil
+	}
+
+	var note string
+	lower := strings.ToLower(arg)
+	switch {
+	case strings.HasSuffix(lower, extKEPUB):
+		if !kepub {
+			return Target{}, fmt.Errorf("%s: %w", arg, ErrKEPUBName)
+		}
+	case strings.HasSuffix(lower, extEPUB):
+		if kepub {
+			renamed := arg[:len(arg)-len(extEPUB)] + extKEPUB
+			note = fmt.Sprintf("saving as %s so Kobo opens it with the KEPUB renderer", filepath.Base(renamed))
+			arg = renamed
+		}
+	default:
+		arg += ext
+	}
+
+	file, err := filepath.Abs(arg)
+	if err != nil {
+		return Target{}, err
+	}
+	return Target{file: file, Note: note}, nil
+}
+
+// Known reports whether the output filename is known before the article
+// title is, which allows checking for an existing file before fetching.
+func (t Target) Known() bool {
+	return t.file != ""
+}
+
+// Path returns the output file path. title is only used for folder targets.
+func (t Target) Path(title string) string {
+	if t.file != "" {
+		return t.file
+	}
+	return filepath.Join(t.dir, Slug(title)+t.ext)
+}
+
+// isFolder reports whether arg names a folder: it ends in a path separator
+// or is an existing directory.
+func isFolder(arg string) bool {
+	if os.IsPathSeparator(arg[len(arg)-1]) {
+		return true
+	}
+	info, err := os.Stat(arg)
+	return err == nil && info.IsDir()
+}
+
+// expandHome replaces a leading "~" with the user's home directory, for
+// paths the shell did not expand (for example, quoted ones).
+func expandHome(arg string) (string, error) {
+	if arg != "~" && !(len(arg) > 1 && arg[0] == '~' && os.IsPathSeparator(arg[1])) {
+		return arg, nil
+	}
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", fmt.Errorf("expanding ~: %w", err)
+	}
+	return home + arg[1:], nil
+}
+
+// Slug turns a title into a safe filename stem: lowercase ASCII letters and
+// digits separated by single dashes, at most 80 characters. Accents are
+// removed ("Café" → "cafe"). It never contains path separators, so a title
+// cannot move the file out of the chosen folder.
+func Slug(title string) string {
+	var b strings.Builder
+	dash := false
+	for _, r := range norm.NFD.String(title) {
+		switch {
+		case unicode.Is(unicode.Mn, r):
+			// Combining accent left over from NFD; drop it.
+		case r < unicode.MaxASCII && (unicode.IsLetter(r) || unicode.IsDigit(r)):
+			if dash && b.Len() > 0 {
+				b.WriteByte('-')
+			}
+			dash = false
+			b.WriteRune(unicode.ToLower(r))
+		default:
+			dash = true
+		}
+	}
+	s := b.String()
+	if len(s) > maxSlugLen {
+		s = strings.TrimRight(s[:maxSlugLen], "-")
+	}
+	if s == "" {
+		return fallbackName
+	}
+	return s
+}
+
+// CheckExists returns an error wrapping ErrExists if path already exists.
+// It is the early check made before work is done; Write repeats the check
+// atomically.
+func CheckExists(path string) error {
+	info, err := os.Lstat(path)
+	switch {
+	case errors.Is(err, fs.ErrNotExist):
+		return nil
+	case err != nil:
+		return err
+	case info.IsDir():
+		return fmt.Errorf("%s is a folder", path)
+	default:
+		return fmt.Errorf("%s %w", path, ErrExists)
+	}
+}
+
+// Write writes src to path, creating missing folders. Unless force is set,
+// an existing file is never modified and an error wrapping ErrExists is
+// returned. The content is written to a temporary file and renamed into
+// place, so path never holds a partial book; on failure, nothing the run
+// created is left behind.
+func Write(path string, src io.WriterTo, force bool) (err error) {
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("creating folder: %w", err)
+	}
+
+	// Without force, claim the name first. O_EXCL fails if the file exists,
+	// including one created by another program since CheckExists ran.
+	reserved := false
+	if !force {
+		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
+		if errors.Is(err, fs.ErrExist) {
+			return fmt.Errorf("%s %w", path, ErrExists)
+		}
+		if err != nil {
+			return err
+		}
+		if err := f.Close(); err != nil {
+			os.Remove(path)
+			return err
+		}
+		reserved = true
+	}
+
+	tmp, err := os.CreateTemp(dir, ".rekopub-*.tmp")
+	if err != nil {
+		if reserved {
+			os.Remove(path)
+		}
+		return err
+	}
+	defer func() {
+		if err != nil {
+			tmp.Close()
+			os.Remove(tmp.Name())
+			if reserved {
+				os.Remove(path)
+			}
+		}
+	}()
+
+	if _, err := src.WriteTo(tmp); err != nil {
+		return err
+	}
+	if err := tmp.Sync(); err != nil {
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	// CreateTemp uses mode 0600; books should be readable like any other
+	// file. Best effort: FAT32 (a Kobo over USB) has no permissions.
+	_ = os.Chmod(tmp.Name(), 0o644)
+
+	return os.Rename(tmp.Name(), path)
+}

internal/save/save_test.go

@@ -0,0 +1,271 @@
+package save
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestResolve(t *testing.T) {
+	dir := t.TempDir()
+	existing := filepath.Join(dir, "existing")
+	if err := os.Mkdir(existing, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	sep := string(filepath.Separator)
+
+	tests := []struct {
+		name     string
+		arg      string
+		kepub    bool
+		wantFile string // set for file targets
+		wantDir  string // set for folder targets
+		wantNote bool
+	}{
+		{name: "epub file", arg: filepath.Join(dir, "a.epub"), wantFile: filepath.Join(dir, "a.epub")},
+		{name: "uppercase extension kept", arg: filepath.Join(dir, "a.EPUB"), wantFile: filepath.Join(dir, "a.EPUB")},
+		{name: "extension appended", arg: filepath.Join(dir, "a"), wantFile: filepath.Join(dir, "a.epub")},
+		{name: "new path without slash is a file", arg: filepath.Join(dir, "new"), wantFile: filepath.Join(dir, "new.epub")},
+		{name: "trailing slash is a folder", arg: filepath.Join(dir, "new") + sep, wantDir: filepath.Join(dir, "new")},
+		{name: "forward slash is a folder", arg: filepath.Join(dir, "new") + "/", wantDir: filepath.Join(dir, "new")},
+		{name: "existing folder", arg: existing, wantDir: existing},
+		{name: "kepub folder", arg: existing, kepub: true, wantDir: existing},
+		{name: "kepub file kept", arg: filepath.Join(dir, "a.kepub.epub"), kepub: true, wantFile: filepath.Join(dir, "a.kepub.epub")},
+		{name: "kepub renames epub", arg: filepath.Join(dir, "a.epub"), kepub: true, wantFile: filepath.Join(dir, "a.kepub.epub"), wantNote: true},
+		{name: "kepub extension appended", arg: filepath.Join(dir, "a"), kepub: true, wantFile: filepath.Join(dir, "a.kepub.epub")},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := Resolve(tt.arg, tt.kepub)
+			if err != nil {
+				t.Fatalf("Resolve(%q) error: %v", tt.arg, err)
+			}
+			if tt.wantFile != "" {
+				if !got.Known() || got.Path("ignored") != tt.wantFile {
+					t.Errorf("Resolve(%q) = %+v, want file %q", tt.arg, got, tt.wantFile)
+				}
+			} else if got.Known() || got.dir != tt.wantDir {
+				t.Errorf("Resolve(%q) = %+v, want folder %q", tt.arg, got, tt.wantDir)
+			}
+			if (got.Note != "") != tt.wantNote {
+				t.Errorf("Resolve(%q) note = %q, want note: %v", tt.arg, got.Note, tt.wantNote)
+			}
+		})
+	}
+}
+
+func TestResolveFolderNaming(t *testing.T) {
+	dir := t.TempDir()
+	for _, kepub := range []bool{false, true} {
+		got, err := Resolve(dir, kepub)
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := filepath.Join(dir, "hello-world.epub")
+		if kepub {
+			want = filepath.Join(dir, "hello-world.kepub.epub")
+		}
+		if p := got.Path("Hello, World!"); p != want {
+			t.Errorf("kepub=%v: Path = %q, want %q", kepub, p, want)
+		}
+	}
+}
+
+func TestResolveKEPUBNameWithoutFlag(t *testing.T) {
+	_, err := Resolve(filepath.Join(t.TempDir(), "a.kepub.epub"), false)
+	if !errors.Is(err, ErrKEPUBName) {
+		t.Fatalf("error = %v, want ErrKEPUBName", err)
+	}
+}
+
+func TestResolveHome(t *testing.T) {
+	home := t.TempDir()
+	t.Setenv("HOME", home)
+	t.Setenv("USERPROFILE", home) // os.UserHomeDir on Windows
+
+	got, err := Resolve("~/Books/a.epub", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := filepath.Join(home, "Books", "a.epub"); got.Path("") != want {
+		t.Errorf("Path = %q, want %q", got.Path(""), want)
+	}
+
+	got, err = Resolve("~", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Known() || got.dir != home {
+		t.Errorf("Resolve(~) = %+v, want folder %q", got, home)
+	}
+}
+
+func TestSlug(t *testing.T) {
+	tests := []struct{ in, want string }{
+		{"Hello, World!", "hello-world"},
+		{"  Leading and trailing  ", "leading-and-trailing"},
+		{"Café crème brûlée", "cafe-creme-brulee"},
+		{"Go 1.24: What's New?", "go-1-24-what-s-new"},
+		{"../../etc/passwd", "etc-passwd"},
+		{`a\b/c:d*e?f"g<h>i|j`, "a-b-c-d-e-f-g-h-i-j"},
+		{"日本語のタイトル", "article"},
+		{"", "article"},
+		{"---", "article"},
+		{strings.Repeat("ab ", 50), strings.TrimRight(strings.Repeat("ab-", 27)[:80], "-")},
+	}
+	for _, tt := range tests {
+		if got := Slug(tt.in); got != tt.want {
+			t.Errorf("Slug(%q) = %q, want %q", tt.in, got, tt.want)
+		}
+		if got := Slug(tt.in); len(got) > maxSlugLen || strings.ContainsAny(got, `/\`) {
+			t.Errorf("Slug(%q) = %q: too long or contains a separator", tt.in, got)
+		}
+	}
+}
+
+func TestCheckExists(t *testing.T) {
+	dir := t.TempDir()
+	file := filepath.Join(dir, "a.epub")
+	writeFile(t, file, "old")
+
+	if err := CheckExists(filepath.Join(dir, "missing.epub")); err != nil {
+		t.Errorf("missing file: %v", err)
+	}
+	if err := CheckExists(file); !errors.Is(err, ErrExists) {
+		t.Errorf("existing file: error = %v, want ErrExists", err)
+	}
+	if err := CheckExists(dir); err == nil || errors.Is(err, ErrExists) {
+		t.Errorf("folder: error = %v, want a non-ErrExists error", err)
+	}
+}
+
+func TestWriteCreatesFolders(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "a", "b", "c", "book.epub")
+
+	if err := Write(path, strings.NewReader("book"), false); err != nil {
+		t.Fatal(err)
+	}
+	assertContent(t, path, "book")
+	assertNoLeftovers(t, filepath.Dir(path), "book.epub")
+}
+
+func TestWriteRefusesExistingFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "book.epub")
+	writeFile(t, path, "old")
+	old := time.Now().Add(-time.Hour).Truncate(time.Second)
+	if err := os.Chtimes(path, old, old); err != nil {
+		t.Fatal(err)
+	}
+
+	err := Write(path, strings.NewReader("new"), false)
+	if !errors.Is(err, ErrExists) {
+		t.Fatalf("error = %v, want ErrExists", err)
+	}
+	assertContent(t, path, "old")
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !info.ModTime().Equal(old) {
+		t.Errorf("modification time changed: %v, want %v", info.ModTime(), old)
+	}
+	assertNoLeftovers(t, dir, "book.epub")
+}
+
+func TestWriteForceReplaces(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "book.epub")
+	writeFile(t, path, "old")
+
+	if err := Write(path, strings.NewReader("new"), true); err != nil {
+		t.Fatal(err)
+	}
+	assertContent(t, path, "new")
+	assertNoLeftovers(t, dir, "book.epub")
+}
+
+func TestWriteFailureKeepsOldFile(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "book.epub")
+	writeFile(t, path, "old")
+
+	err := Write(path, failingWriter{}, true)
+	if err == nil {
+		t.Fatal("Write succeeded, want error")
+	}
+	assertContent(t, path, "old")
+	assertNoLeftovers(t, dir, "book.epub")
+}
+
+func TestWriteFailureLeavesNothing(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "book.epub")
+
+	if err := Write(path, failingWriter{}, false); err == nil {
+		t.Fatal("Write succeeded, want error")
+	}
+	assertNoLeftovers(t, dir)
+}
+
+func TestWritePathThroughFile(t *testing.T) {
+	dir := t.TempDir()
+	blocker := filepath.Join(dir, "Books")
+	writeFile(t, blocker, "not a folder")
+
+	err := Write(filepath.Join(blocker, "book.epub"), strings.NewReader("book"), false)
+	if err == nil || errors.Is(err, ErrExists) {
+		t.Fatalf("error = %v, want a folder creation error", err)
+	}
+	assertContent(t, blocker, "not a folder")
+}
+
+// failingWriter writes part of a book and then fails, like a build error
+// half-way through.
+type failingWriter struct{}
+
+func (failingWriter) WriteTo(w io.Writer) (int64, error) {
+	n, _ := w.Write([]byte("partial"))
+	return int64(n), errors.New("build failed")
+}
+
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func assertContent(t *testing.T, path, want string) {
+	t.Helper()
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(got, []byte(want)) {
+		t.Errorf("%s contains %q, want %q", path, got, want)
+	}
+}
+
+// assertNoLeftovers checks that dir contains exactly the named files, so no
+// temp files or reservations were left behind.
+func assertNoLeftovers(t *testing.T, dir string, want ...string) {
+	t.Helper()
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for _, e := range entries {
+		got = append(got, e.Name())
+	}
+	if strings.Join(got, ",") != strings.Join(want, ",") {
+		t.Errorf("%s contains %v, want %v", dir, got, want)
+	}
+}

pull-releases/001-rekopub-import/README.md

@@ -0,0 +1,27 @@
+# Import the project as ReKoPub
+
+Branch `feature/rekopub-import` · no version change (the code is the same as the last imported commit)
+
+## What changed
+
+- **Whole project imported** from the previous repository's latest `main`: the CLI, `internal/` packages, tests and test data, `PLAN.md`, `SECURITY.md`, `build.cmd`, `test-examples.cmd`, the GoReleaser config and the GitHub workflow files. It's a single commit with no earlier history, so the old name only survives in the old repository.
+- **Renamed to ReKoPub.** The project is called ReKoPub in prose and `rekopub` in code: the binary, `cmd/rekopub/`, the usage text, `-VERSION` output, the user agent (`compatible; rekopub`), error messages, the internal stand-in host for image URLs (`rekopub.invalid`), the build scripts, GoReleaser archive names and `.gitignore`.
+- **New module path** `git.monoexp.com/bajdev/rekopub` (Gogs answers `?go-get=1` for it), so `go install git.monoexp.com/bajdev/rekopub/cmd/rekopub@latest` works. Clone and Releases links in the README point to `git.monoexp.com/bajdev/ReKoPub`.
+- **`LICENSE`**: the Gogs template's `<year> <copyright holders>` placeholder is replaced by the project's MIT licence, `Copyright (c) 2026 bajdev`.
+- **`README.md`**: the project README replaces the Gogs placeholder, keeping its "Turn any web article into a clean EPUB for your e-ink reader" tagline.
+- **`pull-releases/`**: this record and the index, started with this PR.
+
+## Notes
+
+These still describe GitHub and need a decision before the next release:
+
+- `.github/workflows/` (CI and Release) don't run on Gogs. The README's "Releasing" section and the "CI always runs them" line describe them.
+- `.goreleaser.yaml` takes its changelog from GitHub and creates a draft GitHub release. Gogs can't receive GoReleaser uploads, so archives would have to be built locally and attached to a Gogs release by hand.
+- `SECURITY.md` asks for reports through GitHub's Security tab, which Gogs doesn't have.
+
+## Testing
+
+- `gofmt -l .` is clean; `go vet ./...` passes; `go test ./...` passes (epubcheck checks skipped: no `EPUBCHECK_JAR`).
+- `go build -ldflags "-X main.version=dev" ./cmd/rekopub`, then `rekopub -VERSION` prints `rekopub dev` and `-h` shows `Usage: rekopub …`.
+- Converted `https://en.wikipedia.org/wiki/E_Ink` to `e-ink-wikipedia.epub` (309 KB). No file in the book contains the old name.
+- A case-insensitive search for the old name across the repository finds nothing.

pull-releases/README.md

@@ -0,0 +1,23 @@
+# Pull releases
+
+One folder per pull request, so each change has a written record. The record is
+added on the PR's own branch, so it is reviewed and merged with the change.
+
+```
+pull-releases/
+  NNN-branch-name/      NNN = next number in sequence, branch name without feature/
+    README.md           title (first line), what changed, notes, testing
+```
+
+To open the PR on Gogs, use the compare link for the branch, paste the first
+line of `README.md` as the title and the rest as the description.
+
+ReKoPub is a command-line tool with nothing to show in a browser, so records
+have no `desktop.png` or `mobile.png`. A record may include terminal output or
+files from a test run instead.
+
+## Index
+
+| # | Title | Branch |
+|---|---|---|
+| 001 | [Import the project as ReKoPub](001-rekopub-import/README.md) | `feature/rekopub-import` |

test-examples.cmd

@@ -0,0 +1,48 @@
+@echo off
+rem Builds rekopub.exe, then converts a set of real articles into
+rem test-examples\ for checking on the e-readers. Books from a previous run
+rem are replaced (-FORCE). A failure on one URL doesn't stop the others.
+rem
+rem Run it from Command Prompt or PowerShell:
+rem     test-examples.cmd          (Command Prompt)
+rem     .\test-examples.cmd        (PowerShell)
+rem Add "kepub" to make Kobo KEPUBs (.kepub.epub) instead of plain EPUBs:
+rem     test-examples.cmd kepub
+setlocal
+
+set KEPUB=
+if /i "%~1"=="kepub" set KEPUB=-KEPUB
+
+rem Work from the folder this script is in, wherever it was started from.
+cd /d "%~dp0"
+
+call build.cmd || exit /b 1
+echo.
+
+if not exist test-examples mkdir test-examples
+
+set /a ok=0, failed=0
+for %%U in (
+    "https://pluralistic.net/2026/09/12/god-in-the-box/"
+    "https://arstechnica.com/gadgets/2026/10/kde-turns-30-with-plasma-6-8-remote-upgrades-and-an-x11-farewell/"
+    "https://www.bbc.com/news/articles/c5ygyww0zk3o"
+    "https://en.wikipedia.org/wiki/BattleTech"
+    "https://www.wired.com/story/breadfruit-caribbean-pacific-climate-change-super-food/"
+) do call :convert %%U
+
+echo %ok% converted, %failed% failed. Books are in %CD%\test-examples\
+if %failed% gtr 0 exit /b 1
+exit /b 0
+
+rem Converts one URL and counts the result. The -SAVE folder is left
+rem unquoted: a quoted path ending in a backslash would swallow the quote.
+:convert
+echo === %~1
+rekopub.exe -URL "%~1" -SAVE test-examples\ -FORCE %KEPUB%
+if errorlevel 1 (
+    set /a failed+=1
+) else (
+    set /a ok+=1
+)
+echo.
+exit /b 0

testdata/pages/blog.html

@@ -0,0 +1,62 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>Why E-Ink Is Easier on the Eyes | The Reading Desk</title>
+  <meta property="og:title" content="Why E-Ink Is Easier on the Eyes">
+  <meta property="og:site_name" content="The Reading Desk">
+  <meta name="author" content="Ada Lovelace">
+  <meta name="description" content="A look at how electronic paper displays work and why long reading sessions feel different on them.">
+  <meta property="article:published_time" content="2025-03-14T09:30:00Z">
+  <script>window.dataLayer = [];</script>
+  <style>body { font-family: sans-serif; }</style>
+</head>
+<body>
+  <header class="site-header">
+    <a href="/" class="logo">The Reading Desk</a>
+    <nav>
+      <ul>
+        <li><a href="/">Home</a></li>
+        <li><a href="/reviews">Reviews</a></li>
+        <li><a href="/about">About us</a></li>
+        <li><a href="/subscribe">Subscribe to our newsletter</a></li>
+      </ul>
+    </nav>
+  </header>
+  <div class="cookie-banner">We use cookies to improve your experience. <button>Accept all</button></div>
+  <main>
+    <article>
+      <h1>Why E-Ink Is Easier on the Eyes</h1>
+      <p class="byline">By Ada Lovelace &middot; March 14, 2025</p>
+      <p>Electronic paper does not shine light into your eyes. Instead, it reflects the light around you, just like a printed page. That single difference explains most of why an hour with an e-reader feels so unlike an hour with a phone or a laptop.</p>
+      <h2>How the display works</h2>
+      <p>Each pixel of an E Ink panel holds millions of tiny capsules filled with black and white particles. A small electric charge pulls one colour to the surface and pushes the other down. Once the particles have moved, they stay put without using any power, which is why the screen can show a page for weeks on a single charge.</p>
+      <p>Because the image is made of real pigment rather than light, the contrast looks the same in bright sunlight as it does under a reading lamp. Glossy screens wash out outdoors; paper-like screens get easier to read.</p>
+      <h2>What it means for reading</h2>
+      <p>Researchers who compared reading on paper, e-ink and backlit tablets found that comprehension on e-ink was close to paper for long, linear texts. The differences grew for tasks that needed flipping back and forth, where physical pages still win.</p>
+      <iframe width="560" height="315" src="https://www.youtube.com/embed/dQw4w9WgXcQ" title="How E Ink works" allowfullscreen></iframe>
+      <blockquote><p>The best screen for a novel is the one you forget you are looking at.</p></blockquote>
+      <p>There are trade-offs. Page turns are slower, colour is limited or absent, and images can look muddy. For articles and books, though, the calm, steady page is hard to beat &amp; well worth the compromises.</p>
+      <ul>
+        <li>No backlight shining at your eyes</li>
+        <li>Readable in direct sunlight</li>
+        <li>Weeks of battery life</li>
+      </ul>
+      <p>Read more in <a href="/guides/choosing-an-ereader">our guide to choosing an e-reader</a>.</p>
+    </article>
+    <aside class="related">
+      <h3>Related posts</h3>
+      <ul><li><a href="/a">Ten best e-readers of the year</a></li><li><a href="/b">Is Kindle Unlimited worth it?</a></li></ul>
+    </aside>
+    <section class="comments">
+      <h3>3 comments</h3>
+      <div class="comment"><p>Great article, thanks!</p></div>
+      <div class="comment"><p>I switched to a Kobo last year and never looked back.</p></div>
+    </section>
+  </main>
+  <footer>
+    <p>&copy; 2025 The Reading Desk. All rights reserved.</p>
+    <a href="/privacy">Privacy policy</a> <a href="/terms">Terms</a>
+  </footer>
+</body>
+</html>

testdata/pages/docs.html

@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+<html lang="en_US">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+<title>Configuring the Sync Daemon &mdash; Docs</title>
+<meta property="og:site_name" content="Example Docs">
+</head>
+<body class="docs">
+<div class="sidebar"><ul><li><a href="/docs/install">Install</a></li><li><a href="/docs/config">Configure</a></li><li><a href="/docs/faq">FAQ</a></li></ul></div>
+<div class="content" role="main">
+<article id="main-article">
+<h1 id="top">Configuring the Sync Daemon</h1>
+<p onclick="track('intro')" style="font-size:18px;color:#666">The sync daemon reads a single configuration file at start-up. This page describes every option, the defaults, and a few common recipes&nbsp;&mdash; including how to keep battery use low on e-ink devices.</p>
+<h2>File location</h2>
+<p>The daemon looks for <code>config.toml</code> in these places, in order:</p>
+<ol>
+<li><code>$XDG_CONFIG_HOME/syncd/config.toml</code></li>
+<li><code>~/.config/syncd/config.toml</code></li>
+<li><code>/etc/syncd/config.toml</code><p>System-wide settings &amp; defaults.</p></li>
+</ol>
+<h2>Options</h2>
+<table class="options">
+<thead><tr><th>Option</th><th>Type</th><th>Default</th><th colspan="0">Notes</th></tr></thead>
+<tr><td><code>interval</code></td><td>duration</td><td>15m</td><td>How often to check for changes. Values under 1m are rounded up.</td></tr>
+<tr><td><code>wifi_only</code></td><td>bool</td><td>true</td><td>Skip syncing on mobile data.</td></tr>
+<tr><td rowspan="2"><code>paths</code></td><td>list</td><td>[]</td><td>Folders to watch &lt;relative to home&gt;.</td></tr>
+<tr><td colspan="3">May be empty, in which case nothing is synced.</td></tr>
+</table>
+<h3>Durations</h3>
+<p>Durations use Go syntax: <kbd>90s</kbd>, <kbd>15m</kbd> or <kbd>1h30m</kbd>. The &ldquo;quotes&rdquo; are optional.</p>
+<h4>Edge cases</h4>
+<dl><dt>Zero</dt><dd>Disables periodic sync; only manual sync runs.</dd><dt>Negative</dt><dd>Rejected at start-up with an error.</dd></dl>
+<h2>Example</h2>
+<pre class="language-toml"><code><span class="k">interval</span> = <span class="s">"30m"</span>
+<span class="k">wifi_only</span> = true
+<span class="k">paths</span> = ["Books", "Articles/very/long/path/that/would/overflow/a/small/e-ink/screen/if/it/did/not/wrap"]
+</code></pre>
+<div class="note"><p><b>Note:</b> restart the daemon after editing &ndash; it does not watch its own config file.<div>See also the <a href="../faq#restart">FAQ</a>.</div></p></div>
+<p>Questions? <a href="mailto:docs@example.com">Email the docs team</a> or <a href="javascript:openChat()">open chat</a>.</p>
+<form class="feedback"><p>Was this page helpful?</p><button>Yes</button><button>No</button></form>
+</article>
+</div>
+<footer><p>&copy; Example Inc.</p></footer>
+<script>analytics.page();</script>
+</body>
+</html>

testdata/pages/lightbox.html

@@ -0,0 +1,61 @@
+<!DOCTYPE html>
+<!-- Image markup modelled on WordPress lightbox themes (as used by Ars Technica): each image is wrapped in a zoom link inside nested divs in a <figure>, with the caption laid out in divs. go-trafilatura drops such images unless the link is unwrapped first. -->
+<html lang="en-US">
+<head>
+<meta charset="UTF-8">
+<title>KDE turns 30 with Plasma 6.8</title>
+<meta property="og:title" content="KDE turns 30 with Plasma 6.8">
+<meta property="og:site_name" content="Desktop Weekly">
+<meta property="og:image" content="/wp-content/uploads/2026/10/hero.jpg">
+<meta name="author" content="Jane Doe">
+</head>
+<body>
+<header class="site-header"><nav><a href="/">Desktop Weekly</a><a href="/news/">Gadgets</a></nav></header>
+<main>
+<article class="double-column h-entry post type-post">
+  <header>
+    <h1 class="font-serif text-[2rem]">KDE turns 30 with Plasma 6.8</h1>
+    <p class="text-gray-550">Remote upgrades and an X11 farewell.</p>
+  </header>
+  <div class="intro-image-container">
+    <figure class="intro-image">
+      <img width="1152" height="648" src="/wp-content/uploads/2026/10/hero-1152x648.jpg" class="intro-image" alt="" decoding="async" fetchpriority="high" srcset="/wp-content/uploads/2026/10/hero-1152x648.jpg 1152w, /wp-content/uploads/2026/10/hero-640x360.jpg 640w, /wp-content/uploads/2026/10/hero.jpg 1920w" sizes="(max-width: 1152px) 100vw, 1152px">
+      <figcaption><div class="caption">The Plasma 6.8 desktop.<span class="caption-credit">Credit: KDE</span></div></figcaption>
+    </figure>
+  </div>
+  <div class="post-content post-content-double">
+    <p>KDE, the community behind the Plasma desktop, turned 30 this week. To mark the occasion it shipped Plasma 6.8, a release that adds remote upgrades for managed fleets and finally drops the X11 session that has shipped alongside Wayland for several years.</p>
+    <figure class="ars-img-shortcode id-2123456 align-fullwidth">
+      <div>
+        <div class="ars-lightbox">
+          <div class="ars-lightbox-item">
+            <a data-pswp-width="1920" data-pswp-height="1080" data-pswp-srcset="/wp-content/uploads/2026/10/upgrade-dialog.png 1920w" data-cropped="true" href="/wp-content/uploads/2026/10/upgrade-dialog.png" class="cursor-zoom-in" target="_blank">
+              <img width="1024" height="576" src="/wp-content/uploads/2026/10/upgrade-dialog-1024x576.png" class="fullwidth full" alt="" decoding="async" loading="lazy" srcset="/wp-content/uploads/2026/10/upgrade-dialog-1024x576.png 1024w, /wp-content/uploads/2026/10/upgrade-dialog-640x360.png 640w, /wp-content/uploads/2026/10/upgrade-dialog-1536x864.png 1536w, /wp-content/uploads/2026/10/upgrade-dialog.png 1920w" sizes="(max-width: 1024px) 100vw, 1024px">
+            </a>
+            <div class="pswp-caption-content" id="caption-2123456">The new remote upgrade dialog.<div class="ars-gallery-caption-credit">Credit: KDE</div></div>
+          </div>
+        </div>
+      </div>
+      <figcaption class="hidden">
+        <div class="caption font-impact mb-4 mt-2 inline-flex flex-row">
+          <div class="caption-icon w-[10px] shrink-0"></div>
+          <div class="caption-content">The new remote upgrade dialog.<span class="caption-credit mt-2 text-xs">Credit: KDE</span></div>
+        </div>
+      </figcaption>
+    </figure>
+    <p>The remote upgrade feature is aimed at schools and companies that run KDE on many machines. Administrators can now stage an upgrade, test it on a handful of desktops and then roll it out to the rest without touching each computer.</p>
+    <h2>Goodbye, X11</h2>
+    <p>The bigger change for most people is the end of the X11 session. KDE says Wayland now covers the remaining gaps, from screen sharing to colour management, and that keeping both sessions working was slowing everything else down.</p>
+    <figure class="ars-img-shortcode id-2123457 align-center">
+      <div><div class="ars-lightbox"><div class="ars-lightbox-item">
+        <a href="/wp-content/uploads/2026/10/settings.jpg" class="cursor-zoom-in" target="_blank"><img width="640" height="480" src="/wp-content/uploads/2026/10/settings-640x480.jpg" class="center large" alt="System Settings in Plasma 6.8" decoding="async" loading="lazy"></a>
+      </div></div></div>
+      <figcaption class="hidden"><div class="caption"><div class="caption-content">System Settings has been reorganised.</div></div></figcaption>
+    </figure>
+    <p>Distributions that still need X11 can keep shipping older Plasma releases, but KDE will not backport new features to them. The project expects most users to notice nothing at all beyond a faster, smoother desktop.</p>
+  </div>
+</article>
+</main>
+<footer><p>&copy; 2026 Desktop Weekly</p></footer>
+</body>
+</html>

testdata/pages/photo-essay.html

@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>A Walk Through the Print Shop</title>
+  <meta property="og:site_name" content="Letterpress Weekly">
+  <meta name="author" content="Grace Hopper">
+  <meta property="og:image" content="/img/press.jpg">
+</head>
+<body>
+  <nav><a href="/">Home</a> <a href="/essays">Essays</a></nav>
+  <article>
+    <h1>A Walk Through the Print Shop</h1>
+    <p>The print shop on Mill Street has been running since 1894. Its owners still set type by hand, ink the rollers each morning and print posters, wedding invitations and the occasional book on machines older than anyone who works there.</p>
+    <figure>
+      <img src="../img/press.jpg" alt="The old printing press" width="2000" height="1200">
+      <figcaption>The 1890s press still runs every Friday.</figcaption>
+    </figure>
+    <p>Every job starts at the type cases. Each drawer holds a single typeface in a single size, with compartments for every letter, number and punctuation mark. The most used letters sit closest to the compositor's hand.</p>
+    <img src="data:image/gif;base64,R0lGODlhAQABAAAAACw=" data-src="/img/diagram.png" alt="How ink reaches the paper" class="lazyload">
+    <p>Once a page is set, it is locked into a frame called a chase and placed on the press bed. The rollers pass over the type, depositing a thin film of ink, and the paper is pressed against it for a crisp, slightly sunken impression.</p>
+    <picture>
+      <source srcset="/img/type.avif" type="image/avif">
+      <source srcset="/img/type.webp" type="image/webp">
+      <img src="/img/type-fallback.jpg" alt="Drawers of wooden type">
+    </picture>
+    <p>Not every image survives a move to e-ink: this paragraph has an inline <img src="/img/missing.png" alt="small press icon"> icon whose file has gone missing.</p>
+    <img src="https://tracker.example/p.gif" width="1" height="1" alt="">
+    <p>By late afternoon the shop is quiet again. The day's posters hang to dry on lines strung across the ceiling, and the type is distributed back into its cases, letter by letter, ready for tomorrow.</p>
+  </article>
+  <footer>&copy; Letterpress Weekly</footer>
+</body>
+</html>

testdata/pages/wiki.html

@@ -0,0 +1,38 @@
+<!DOCTYPE html>
+<!-- Markup modelled on MediaWiki (Wikipedia) article pages, with invented
+     text. Image URLs carry a tracking query string ("?utm_source=..."), and
+     one has no file extension: go-trafilatura dropped all such images until
+     clean.Prepare started carrying image URLs through it in stand-ins. -->
+<html class="client-nojs" lang="en" dir="ltr">
+<head>
+<meta charset="UTF-8">
+<title>Walking Robots (game) - Wikipedia</title>
+<meta property="og:title" content="Walking Robots (game)">
+<meta property="og:site_name" content="Wikipedia">
+</head>
+<body class="mediawiki ltr skin-vector-2022 action-view">
+<div class="mw-page-container"><div class="mw-page-container-inner">
+<header class="vector-header"><a href="/wiki/Main_Page" class="mw-logo"><img class="mw-logo-icon" src="/static/images/icons/enwiki-25.svg" alt="" height="50" width="50"></a></header>
+<div class="mw-content-container">
+<main id="content" class="mw-body">
+<h1 id="firstHeading" class="firstHeading mw-first-heading"><span class="mw-page-title-main">Walking Robots (game)</span></h1>
+<div id="bodyContent" class="vector-body">
+<div id="mw-content-text" class="mw-body-content"><div class="mw-content-ltr mw-parser-output" lang="en" dir="ltr">
+<section data-mw-section-id="0" id="mwAQ">
+<figure class="mw-default-size" typeof="mw:File/Thumb" id="mwCw"><a href="./File:Walking_Robots_logo.png" class="mw-file-description" id="mwDA"><img resource="./File:Walking_Robots_logo.png" src="/thumb/a/a6/Walking_Robots_logo.png/250px-Walking_Robots_logo.png?utm_source=en.wikipedia.org&amp;utm_campaign=parser&amp;utm_content=thumbnail" decoding="async" srcset="/upload/a/a6/Walking_Robots_logo.png?utm_source=en.wikipedia.org&amp;utm_campaign=parser&amp;utm_content=thumbnail_unscaled 2x" data-file-width="500" data-file-height="74" height="37" width="250" class="mw-file-element" id="mwDQ"></a><figcaption id="mwDg">The game's original logo</figcaption></figure>
+<p id="mwEA"><b>Walking Robots</b> is a tabletop wargame in which players command giant walking machines across a hex map. Each machine has a record sheet that tracks its armour, heat and weapons, and turns alternate between movement and combat so that every decision about speed has a cost later in the round.</p>
+<p id="mwEQ">The game was first published as a boxed set with cardboard counters and paper maps, and later editions added plastic miniatures, a campaign system and rules for vehicles and infantry. A thriving community of players paints its own miniatures and writes scenarios for local tournaments.</p>
+</section>
+<section data-mw-section-id="1" id="mwOw"><h2 id="Gameplay">Gameplay</h2>
+<figure class="mw-default-size mw-halign-right" typeof="mw:File/Thumb" id="mwPA"><a href="./File:Painted_miniature.jpg" class="mw-file-description" id="mwPQ"><img resource="./File:Painted_miniature.jpg" src="/thumb/9/99/Painted_miniature.jpg/250px-Painted_miniature.jpg?utm_source=en.wikipedia.org&amp;utm_campaign=parser&amp;utm_content=thumbnail" decoding="async" loading="lazy" srcset="/thumb/9/99/Painted_miniature.jpg/500px-Painted_miniature.jpg?utm_source=en.wikipedia.org&amp;utm_campaign=parser&amp;utm_content=thumbnail 2x" data-file-width="2304" data-file-height="3456" height="375" width="250" class="mw-file-element" id="mwPg"></a><figcaption id="mwPw">A painted miniature on a hex map</figcaption></figure>
+<p id="mwRA">Players move their machines one hex at a time, spending movement points that depend on the terrain. Woods and rubble cost more to cross, water slows everything down, and climbing a level costs an extra point. Firing generates heat, and a machine that runs too hot may shut down at the worst possible moment.</p>
+<p id="mwRQ">Combat is resolved with two six-sided dice. The attacker adds modifiers for range, movement and terrain, and a successful hit is assigned to a location such as an arm, a leg or the torso. Damage that breaks through the armour can destroy weapons, ammunition or the engine itself.</p>
+<figure class="mw-default-size" typeof="mw:File/Thumb" id="mwSA"><a href="./File:Record_sheet" class="mw-file-description"><img src="/media/record-sheet?width=500" decoding="async" height="300" width="400" class="mw-file-element"></a><figcaption>A machine's record sheet</figcaption></figure>
+<p id="mwSQ">Campaign rules link battles together, so that damaged machines must be repaired between games and pilots gain experience over time. Many groups play long campaigns that last for months, keeping detailed records of every battle and every salvaged machine.</p>
+</section>
+</div></div></div>
+</main>
+<footer class="mw-footer"><p>Text is available under the Creative Commons Attribution-ShareAlike License.</p><img src="/static/images/footer/wikimedia-button.svg" width="84" height="29" alt="Wikimedia Foundation"></footer>
+</div></div></div>
+</body>
+</html>