ReKoPub/internal/pipeline/pipeline.go

// Package pipeline turns a web page into a book: fetch, extract, clean,
// process images and build the EPUB.
package pipeline

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"image"
	"io"
	"net/url"
	"strings"

	"golang.org/x/net/html"

	"git.monoexp.com/bajdev/rekopub/internal/assets"
	"git.monoexp.com/bajdev/rekopub/internal/build"
	"git.monoexp.com/bajdev/rekopub/internal/clean"
	"git.monoexp.com/bajdev/rekopub/internal/cover"
	"git.monoexp.com/bajdev/rekopub/internal/extract"
	"git.monoexp.com/bajdev/rekopub/internal/fetch"
	"git.monoexp.com/bajdev/rekopub/internal/kepub"
)

// Errors that Run wraps, so callers can choose an exit code.
var (
	ErrFetch     = errors.New("couldn't fetch the page")
	ErrNoContent = extract.ErrNoContent
)

// Book is a finished book, ready to be saved.
type Book struct {
	// Title is the article title, used to name the file for folder targets.
	Title string
	// Content writes the complete EPUB.
	Content io.WriterTo
}

// Options adjust a single conversion.
type Options struct {
	// BeforeImages, if set, is called with the article title once it is
	// known and before any image is downloaded. An error stops the
	// conversion and is returned unchanged. The CLI uses it to stop early
	// when the output file already exists.
	BeforeImages func(title string) error
	// Warn, if set, receives problems that don't stop the conversion, such
	// as an image that couldn't be downloaded.
	Warn func(msg string)
	// KEPUB converts the finished book to a Kobo KEPUB.
	KEPUB bool
	// NoLocal refuses local and private addresses even when u itself is
	// one. A server that converts pages for other people sets it, so a
	// visitor can't make it connect to services on the server's network.
	NoLocal bool
}

// Run converts the page at u into a book using the default settings.
// Images and redirects may lead to local and private addresses only when
// u itself is local and opts.NoLocal isn't set.
func Run(ctx context.Context, u *url.URL, opts Options) (*Book, error) {
	return New(!opts.NoLocal && fetch.IsLocal(ctx, u.Hostname())).Run(ctx, u, opts)
}

// Pipeline holds the settings for conversions.
type Pipeline struct {
	Fetcher *fetch.Fetcher
	Images  *assets.Processor
}

// New returns a Pipeline with the default settings. Pages and images share
// one HTTP client, which connects to local and private addresses only if
// allowLocal is set.
func New(allowLocal bool) Pipeline {
	f := fetch.New(allowLocal)
	return Pipeline{Fetcher: f, Images: assets.New(f.Client, f.UserAgent)}
}

// Run converts the page at u into a book.
func (p Pipeline) Run(ctx context.Context, u *url.URL, opts Options) (*Book, error) {
	page, err := p.Fetcher.Fetch(ctx, u)
	if err != nil {
		return nil, fmt.Errorf("%w: %w", ErrFetch, err)
	}

	doc, err := html.Parse(strings.NewReader(page.HTML))
	if err != nil {
		return nil, fmt.Errorf("parsing the page: %w", err)
	}
	clean.Prepare(doc, page.URL)
	article, err := extract.Extract(doc, page.URL)
	if err != nil {
		return nil, err
	}

	if opts.BeforeImages != nil {
		if err := opts.BeforeImages(article.Title); err != nil {
			return nil, err
		}
	}

	body, err := clean.Content(article.Content, page.URL)
	if err != nil {
		return nil, fmt.Errorf("cleaning the article: %w", err)
	}
	images := *p.Images
	images.Warn = opts.Warn
	imgs, err := images.Process(ctx, body, page.URL)
	if err != nil {
		return nil, fmt.Errorf("processing images: %w", err)
	}

	photo, err := coverPhoto(ctx, &images, article, page.URL)
	if err != nil {
		return nil, fmt.Errorf("processing images: %w", err)
	}
	coverJPEG, err := cover.Render(cover.Details{
		Title:    article.Title,
		SiteName: article.SiteName,
		Date:     build.DisplayDate(article.Published),
	}, photo, images.Profile)
	if err != nil {
		return nil, fmt.Errorf("drawing the cover: %w", err)
	}

	data, err := build.Build(article, body, imgs, coverJPEG)
	if err != nil {
		return nil, fmt.Errorf("building the book: %w", err)
	}
	if opts.KEPUB {
		if data, err = kepub.Convert(ctx, data); err != nil {
			return nil, err
		}
	}
	return &Book{Title: article.Title, Content: bytes.NewReader(data)}, nil
}

// minCoverPhoto is the smallest side, in pixels, of a preview image worth
// putting on the cover. Smaller ones are usually site logos, and a plain
// typographic cover looks better than a blown-up logo.
const minCoverPhoto = 300

// coverPhoto downloads the page's preview image for the cover. It returns
// nil, and the cover is typographic, when there is no usable image; only
// cancellation is an error.
func coverPhoto(ctx context.Context, images *assets.Processor, a *extract.Article, page *url.URL) (image.Image, error) {
	if a.ImageURL == "" {
		return nil, nil
	}
	img, err := images.Decode(ctx, a.ImageURL, page)
	if ctxErr := ctx.Err(); ctxErr != nil {
		return nil, ctxErr
	}
	if err != nil {
		if images.Warn != nil {
			images.Warn(fmt.Sprintf("cover image %s skipped, using a plain cover: %v", a.ImageURL, err))
		}
		return nil, nil
	}
	if b := img.Bounds(); min(b.Dx(), b.Dy()) < minCoverPhoto {
		return nil, nil
	}
	return img, nil
}