118 lines
3.6 KiB
Go
118 lines
3.6 KiB
Go
// priority demonstrates multi-level rate-limiter priority.
|
|
//
|
|
// A background gallery crawl runs continuously at fa.PriorityBackground
|
|
// while interactive submission fetches at fa.PriorityInteractive jump ahead
|
|
// of it even though the crawl started first and both share one global
|
|
// token bucket. (fa.PriorityNormal and fa.PriorityLow sit between the two;
|
|
// the same rule applies higher priority is always served first.)
|
|
//
|
|
// The example runs anonymously by default. If the FA_A / FA_B (and ideally
|
|
// CF_CLEARANCE + FA_UA) environment variables are set, it authenticates
|
|
// with them.
|
|
//
|
|
// Watch the timestamps in the output: once the interactive fetches are
|
|
// issued, they take the next tokens and the background crawl is pushed back.
|
|
//
|
|
// go run ./examples/priority <username> <submission-id> [<submission-id>...]
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
|
|
fa "git.anthrove.art/public/go-fa-api"
|
|
)
|
|
|
|
func main() {
|
|
if len(os.Args) < 3 {
|
|
log.Fatalf("usage: %s <username> <submission-id> [<submission-id>...]", os.Args[0])
|
|
}
|
|
user := os.Args[1]
|
|
ids := make([]fa.SubmissionID, 0, len(os.Args)-2)
|
|
for _, arg := range os.Args[2:] {
|
|
n, err := strconv.ParseInt(arg, 10, 64)
|
|
if err != nil {
|
|
log.Fatalf("invalid submission id %q: %v", arg, err)
|
|
}
|
|
ids = append(ids, fa.SubmissionID(n))
|
|
}
|
|
|
|
// Priority scheduling must be enabled at construction time; without
|
|
// WithPrioritizedRateLimiting the WithPriority markers below are inert.
|
|
opts := []fa.Option{
|
|
fa.WithUserAgent(envOr("FA_UA", "go-fa-api-example/0.1")),
|
|
fa.WithPrioritizedRateLimiting(true),
|
|
}
|
|
if a, b := os.Getenv("FA_A"), os.Getenv("FA_B"); a != "" && b != "" {
|
|
log.Printf("using FA_A/FA_B cookies for authenticated requests")
|
|
opts = append(opts,
|
|
fa.WithCookies(fa.Cookies{A: a, B: b}),
|
|
fa.WithCloudflare(fa.CFCookies{Clearance: os.Getenv("CF_CLEARANCE")}),
|
|
)
|
|
}
|
|
client := fa.New(opts...)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
start := time.Now()
|
|
since := func() string { return fmt.Sprintf("%5.1fs", time.Since(start).Seconds()) }
|
|
|
|
// Background crawler: walks the user's gallery at the lowest priority,
|
|
// keeping the shared token bucket busy so the priority effect is visible.
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
bgCtx := fa.WithBackgroundPriority(ctx)
|
|
n := 0
|
|
for sub, err := range client.Gallery(bgCtx, user, fa.ListOptions{}) {
|
|
if err != nil {
|
|
if ctx.Err() == nil { // not just our own cancel
|
|
log.Printf("[%s] background crawl stopped: %v", since(), err)
|
|
}
|
|
return
|
|
}
|
|
n++
|
|
fmt.Printf("[%s] background · gallery item %3d (%d) %s\n",
|
|
since(), n, sub.ID, sub.Title)
|
|
}
|
|
fmt.Printf("[%s] background · gallery exhausted after %d items\n", since(), n)
|
|
}()
|
|
|
|
// Give the crawler a head start so it is mid-crawl, holding the queue,
|
|
// when the interactive fetches arrive.
|
|
time.Sleep(1500 * time.Millisecond)
|
|
|
|
// Interactive fetches: the user is waiting on these. Each takes the very
|
|
// next token, ahead of the background crawl's pending page fetch.
|
|
for _, id := range ids {
|
|
ictx := fa.WithPriority(ctx, fa.PriorityInteractive)
|
|
t0 := time.Now()
|
|
sub, err := client.GetSubmission(ictx, id)
|
|
if err != nil {
|
|
log.Printf("[%s] interactive · GetSubmission(%d): %v", since(), id, err)
|
|
continue
|
|
}
|
|
fmt.Printf("[%s] INTERACTIVE · got (%d) %q in %.1fs jumped the queue\n",
|
|
since(), id, sub.Title, time.Since(t0).Seconds())
|
|
}
|
|
|
|
// Foreground work is done stop the background crawler and wait for it.
|
|
cancel()
|
|
wg.Wait()
|
|
fmt.Printf("[%s] done\n", since())
|
|
}
|
|
|
|
func envOr(key, fallback string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return fallback
|
|
}
|