go-fa-api/transport.go

package fa

import (
	"context"
	"errors"
	"io"
	"log/slog"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// transport is the SDK's http.RoundTripper. It is the only place where
// rate limiting, header injection, Cloudflare detection, and retries are
// enforced callers cannot bypass it because the *http.Client wired into
// Colly is built around it. Cookies live on the *http.Client's Jar, not here.
type transport struct {
	base       http.RoundTripper
	limiter    *rateLimiter
	userAgent  string
	maxRetries int
	logger     *slog.Logger
}

// defaultMaxRetries is the cap on automatic 429/5xx retries per request.
// Three retries with exponential backoff (1s/2s/4s) is enough to absorb
// short blips without masking real outages.
const defaultMaxRetries = 3

// RoundTrip implements http.RoundTripper. It gates on the rate limiter,
// injects the User-Agent header, retries on transient failures, and
// classifies Cloudflare challenges as non-retryable user-actionable errors.
func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
	if t.userAgent != "" && req.Header.Get("User-Agent") == "" {
		req.Header.Set("User-Agent", t.userAgent)
	}
	// FA serves Brotli-encoded HTML only to UAs that advertise it; the stdlib
	// transport handles gzip transparently when we don't set this header
	// ourselves, so leave it alone.

	var lastErr error
	for attempt := 0; attempt <= t.maxRetries; attempt++ {
		waitStart := time.Now()
		if err := t.limiter.wait(req.Context()); err != nil {
			return nil, err
		}
		waitMs := time.Since(waitStart).Milliseconds()

		// Each attempt needs an independent body reader; if the caller passed
		// one we trust them to have provided GetBody (stdlib does this for
		// the common cases). Without a body, this branch is a no-op.
		if attempt > 0 && req.GetBody != nil {
			body, err := req.GetBody()
			if err != nil {
				return nil, err
			}
			req.Body = body
		}

		rtStart := time.Now()
		resp, err := t.base.RoundTrip(req)
		durMs := time.Since(rtStart).Milliseconds()
		t.logRequest(req, resp, err, durMs, waitMs)
		if err != nil {
			lastErr = err
			if !isTransientNetErr(err) || attempt == t.maxRetries {
				return nil, err
			}
			t.sleepBackoff(attempt)
			continue
		}

		if isCloudflareChallenge(resp) {
			drainAndClose(resp.Body)
			return nil, ErrCloudflareChallenge
		}

		switch {
		case resp.StatusCode == http.StatusTooManyRequests:
			drainAndClose(resp.Body)
			if attempt == t.maxRetries {
				return nil, ErrRateLimited
			}
			t.sleepRetryAfter(resp, attempt)
			continue

		case resp.StatusCode >= 500 && resp.StatusCode <= 599:
			drainAndClose(resp.Body)
			if attempt == t.maxRetries {
				return nil, &HTTPError{StatusCode: resp.StatusCode, URL: req.URL.String()}
			}
			t.sleepBackoff(attempt)
			continue
		}

		return resp, nil
	}
	if lastErr != nil {
		return nil, lastErr
	}
	return nil, errors.New("fa: transport exhausted retries without a response")
}

// logRequest emits one structured slog record per HTTP round-trip so a
// consumer can trace request timings and rate-limit waits. Only the URL host
// is logged never the path or query to avoid leaking what was fetched.
func (t *transport) logRequest(req *http.Request, resp *http.Response, err error, durMs, waitMs int64) {
	if t.logger == nil {
		return
	}
	var host string
	if req != nil && req.URL != nil {
		host = req.URL.Host
	}
	status := 0
	if err == nil && resp != nil {
		status = resp.StatusCode
	}
	// InfoContext (not Info) so the request's context propagates to the slog
	// handler. A tracing consumer can carry an active span in that context
	// and nest this HTTP request as a child span of it.
	ctx := context.Background()
	if req != nil {
		ctx = req.Context()
	}
	t.logger.InfoContext(ctx, "fa.request",
		"host", host,
		"durationMs", durMs,
		"status", status,
		"rateWaitMs", waitMs,
	)
}

// sleepBackoff sleeps for an exponential interval based on attempt index.
// attempt is 0-based, so the sequence is 1s, 2s, 4s.
func (t *transport) sleepBackoff(attempt int) {
	d := time.Duration(1<<attempt) * time.Second
	if t.logger != nil {
		t.logger.Debug("fa: backoff", "attempt", attempt+1, "sleep", d)
	}
	time.Sleep(d)
}

// sleepRetryAfter honours the server's Retry-After header (seconds or HTTP
// date), capping at 60s to bound worst-case latency. Falls back to
// exponential backoff if the header is missing or unparseable.
func (t *transport) sleepRetryAfter(resp *http.Response, attempt int) {
	const cap = 60 * time.Second
	if h := resp.Header.Get("Retry-After"); h != "" {
		if secs, err := strconv.Atoi(strings.TrimSpace(h)); err == nil && secs >= 0 {
			d := time.Duration(secs) * time.Second
			if d > cap {
				d = cap
			}
			if t.logger != nil {
				t.logger.Debug("fa: retry-after", "sleep", d)
			}
			time.Sleep(d)
			return
		}
		if when, err := http.ParseTime(h); err == nil {
			d := time.Until(when)
			if d < 0 {
				d = time.Second
			}
			if d > cap {
				d = cap
			}
			time.Sleep(d)
			return
		}
	}
	t.sleepBackoff(attempt)
}

// isCloudflareChallenge inspects a response for the signatures Cloudflare
// emits when it interposes a challenge. We treat these as non-retryable
// because the SDK has no JS engine; the caller must refresh cf_clearance.
func isCloudflareChallenge(resp *http.Response) bool {
	if resp == nil {
		return false
	}
	if v := resp.Header.Get("cf-mitigated"); strings.EqualFold(v, "challenge") {
		return true
	}
	// Managed challenge / IUAM page: 403 or 503 with cf-ray and HTML body.
	if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusServiceUnavailable {
		if resp.Header.Get("cf-ray") != "" && strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
			return true
		}
	}
	return false
}

// isTransientNetErr returns true for the kinds of network errors that are
// reasonable to retry (timeouts, EOFs from broken keepalive connections).
// Anything else DNS failures, refused connections surfaces immediately.
func isTransientNetErr(err error) bool {
	if err == nil {
		return false
	}
	if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
		return true
	}
	var ne interface{ Timeout() bool }
	if errors.As(err, &ne) && ne.Timeout() {
		return true
	}
	return false
}

// drainAndClose flushes the response body so the underlying TCP connection
// can be returned to the pool. Failing to do this on a retry path leaks
// connections under load.
func drainAndClose(body io.ReadCloser) {
	if body == nil {
		return
	}
	_, _ = io.Copy(io.Discard, body)
	_ = body.Close()
}