package fa import ( "context" "errors" "io" "log/slog" "net/http" "strconv" "strings" "time" ) // transport is the SDK's http.RoundTripper. It is the only place where // rate limiting, header injection, Cloudflare detection, and retries are // enforced callers cannot bypass it because the *http.Client wired into // Colly is built around it. Cookies live on the *http.Client's Jar, not here. type transport struct { base http.RoundTripper limiter *rateLimiter userAgent string maxRetries int logger *slog.Logger } // defaultMaxRetries is the cap on automatic 429/5xx retries per request. // Three retries with exponential backoff (1s/2s/4s) is enough to absorb // short blips without masking real outages. const defaultMaxRetries = 3 // RoundTrip implements http.RoundTripper. It gates on the rate limiter, // injects the User-Agent header, retries on transient failures, and // classifies Cloudflare challenges as non-retryable user-actionable errors. func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { if ov := requestOverrideFrom(req.Context()); ov != nil { if ov.userAgent != "" { req.Header.Set("User-Agent", ov.userAgent) } if ov.touchesCookies() { // The cookie jar already wrote a Cookie header by the time we // reach RoundTrip; replace it wholesale so per-user creds win // over whatever the shared jar would inject. req.Header.Set("Cookie", ov.cookieHeader()) } } if t.userAgent != "" && req.Header.Get("User-Agent") == "" { req.Header.Set("User-Agent", t.userAgent) } // FA serves Brotli-encoded HTML only to UAs that advertise it; the stdlib // transport handles gzip transparently when we don't set this header // ourselves, so leave it alone. var lastErr error for attempt := 0; attempt <= t.maxRetries; attempt++ { waitStart := time.Now() if err := t.limiter.wait(req.Context()); err != nil { return nil, err } waitMs := time.Since(waitStart).Milliseconds() // Each attempt needs an independent body reader; if the caller passed // one we trust them to have provided GetBody (stdlib does this for // the common cases). Without a body, this branch is a no-op. if attempt > 0 && req.GetBody != nil { body, err := req.GetBody() if err != nil { return nil, err } req.Body = body } rtStart := time.Now() resp, err := t.base.RoundTrip(req) durMs := time.Since(rtStart).Milliseconds() t.logRequest(req, resp, err, durMs, waitMs) if err != nil { lastErr = err if !isTransientNetErr(err) || attempt == t.maxRetries { return nil, err } t.sleepBackoff(attempt) continue } if isCloudflareChallenge(resp) { drainAndClose(resp.Body) return nil, ErrCloudflareChallenge } switch { case resp.StatusCode == http.StatusTooManyRequests: drainAndClose(resp.Body) if attempt == t.maxRetries { return nil, ErrRateLimited } t.sleepRetryAfter(resp, attempt) continue case resp.StatusCode >= 500 && resp.StatusCode <= 599: drainAndClose(resp.Body) if attempt == t.maxRetries { return nil, &HTTPError{StatusCode: resp.StatusCode, URL: req.URL.String()} } t.sleepBackoff(attempt) continue } return resp, nil } if lastErr != nil { return nil, lastErr } return nil, errors.New("fa: transport exhausted retries without a response") } // logRequest emits one structured slog record per HTTP round-trip so a // consumer can trace request timings and rate-limit waits. Only the URL host // is logged never the path or query to avoid leaking what was fetched. func (t *transport) logRequest(req *http.Request, resp *http.Response, err error, durMs, waitMs int64) { if t.logger == nil { return } var host string if req != nil && req.URL != nil { host = req.URL.Host } status := 0 if err == nil && resp != nil { status = resp.StatusCode } // InfoContext (not Info) so the request's context propagates to the slog // handler. A tracing consumer can carry an active span in that context // and nest this HTTP request as a child span of it. ctx := context.Background() if req != nil { ctx = req.Context() } t.logger.InfoContext(ctx, "fa.request", "host", host, "durationMs", durMs, "status", status, "rateWaitMs", waitMs, ) } // sleepBackoff sleeps for an exponential interval based on attempt index. // attempt is 0-based, so the sequence is 1s, 2s, 4s. func (t *transport) sleepBackoff(attempt int) { d := time.Duration(1<= 0 { d := time.Duration(secs) * time.Second if d > cap { d = cap } if t.logger != nil { t.logger.Debug("fa: retry-after", "sleep", d) } time.Sleep(d) return } if when, err := http.ParseTime(h); err == nil { d := time.Until(when) if d < 0 { d = time.Second } if d > cap { d = cap } time.Sleep(d) return } } t.sleepBackoff(attempt) } // isCloudflareChallenge inspects a response for the signatures Cloudflare // emits when it interposes a challenge. We treat these as non-retryable // because the SDK has no JS engine; the caller must refresh cf_clearance. func isCloudflareChallenge(resp *http.Response) bool { if resp == nil { return false } if v := resp.Header.Get("cf-mitigated"); strings.EqualFold(v, "challenge") { return true } // Managed challenge / IUAM page: 403 or 503 with cf-ray and HTML body. if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusServiceUnavailable { if resp.Header.Get("cf-ray") != "" && strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { return true } } return false } // isTransientNetErr returns true for the kinds of network errors that are // reasonable to retry (timeouts, EOFs from broken keepalive connections). // Anything else DNS failures, refused connections surfaces immediately. func isTransientNetErr(err error) bool { if err == nil { return false } if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { return true } var ne interface{ Timeout() bool } if errors.As(err, &ne) && ne.Timeout() { return true } return false } // drainAndClose flushes the response body so the underlying TCP connection // can be returned to the pool. Failing to do this on a retry path leaks // connections under load. func drainAndClose(body io.ReadCloser) { if body == nil { return } _, _ = io.Copy(io.Discard, body) _ = body.Close() }