// priority demonstrates multi-level rate-limiter priority. // // A background gallery crawl runs continuously at fa.PriorityBackground // while interactive submission fetches at fa.PriorityInteractive jump ahead // of it even though the crawl started first and both share one global // token bucket. (fa.PriorityNormal and fa.PriorityLow sit between the two; // the same rule applies higher priority is always served first.) // // The example runs anonymously by default. If the FA_A / FA_B (and ideally // CF_CLEARANCE + FA_UA) environment variables are set, it authenticates // with them. // // Watch the timestamps in the output: once the interactive fetches are // issued, they take the next tokens and the background crawl is pushed back. // // go run ./examples/priority [...] package main import ( "context" "fmt" "log" "os" "strconv" "sync" "time" fa "git.anthrove.art/public/go-fa-api" ) func main() { if len(os.Args) < 3 { log.Fatalf("usage: %s [...]", os.Args[0]) } user := os.Args[1] ids := make([]fa.SubmissionID, 0, len(os.Args)-2) for _, arg := range os.Args[2:] { n, err := strconv.ParseInt(arg, 10, 64) if err != nil { log.Fatalf("invalid submission id %q: %v", arg, err) } ids = append(ids, fa.SubmissionID(n)) } // Priority scheduling must be enabled at construction time; without // WithPrioritizedRateLimiting the WithPriority markers below are inert. opts := []fa.Option{ fa.WithUserAgent(envOr("FA_UA", "go-fa-api-example/0.1")), fa.WithPrioritizedRateLimiting(true), } if a, b := os.Getenv("FA_A"), os.Getenv("FA_B"); a != "" && b != "" { log.Printf("using FA_A/FA_B cookies for authenticated requests") opts = append(opts, fa.WithCookies(fa.Cookies{A: a, B: b}), fa.WithCloudflare(fa.CFCookies{Clearance: os.Getenv("CF_CLEARANCE")}), ) } client := fa.New(opts...) ctx, cancel := context.WithCancel(context.Background()) defer cancel() start := time.Now() since := func() string { return fmt.Sprintf("%5.1fs", time.Since(start).Seconds()) } // Background crawler: walks the user's gallery at the lowest priority, // keeping the shared token bucket busy so the priority effect is visible. var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() bgCtx := fa.WithBackgroundPriority(ctx) n := 0 for sub, err := range client.Gallery(bgCtx, user, fa.ListOptions{}) { if err != nil { if ctx.Err() == nil { // not just our own cancel log.Printf("[%s] background crawl stopped: %v", since(), err) } return } n++ fmt.Printf("[%s] background · gallery item %3d (%d) %s\n", since(), n, sub.ID, sub.Title) } fmt.Printf("[%s] background · gallery exhausted after %d items\n", since(), n) }() // Give the crawler a head start so it is mid-crawl, holding the queue, // when the interactive fetches arrive. time.Sleep(1500 * time.Millisecond) // Interactive fetches: the user is waiting on these. Each takes the very // next token, ahead of the background crawl's pending page fetch. for _, id := range ids { ictx := fa.WithPriority(ctx, fa.PriorityInteractive) t0 := time.Now() sub, err := client.GetSubmission(ictx, id) if err != nil { log.Printf("[%s] interactive · GetSubmission(%d): %v", since(), id, err) continue } fmt.Printf("[%s] INTERACTIVE · got (%d) %q in %.1fs jumped the queue\n", since(), id, sub.Title, time.Since(t0).Seconds()) } // Foreground work is done stop the background crawler and wait for it. cancel() wg.Wait() fmt.Printf("[%s] done\n", since()) } func envOr(key, fallback string) string { if v := os.Getenv(key); v != "" { return v } return fallback }