GalleryPage / ScrapsPage / FavoritesPage return a ListingPage struct carrying the page items, the 1-based page number, and a HasNext flag that mirrors FA's "next page" link. This lets external scrapers drive their own pagination loop (checkpoint resume, parallel workers, custom throttling) without re-implementing the page-walking code. The existing iter.Seq2-shaped methods now share the same per-page primitive internally so behaviour stays in lock-step.
116 lines
3.8 KiB
Go
116 lines
3.8 KiB
Go
package fa
|
|
|
|
import (
|
|
"context"
|
|
"iter"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
|
|
"git.anthrove.art/public/go-fa-api/internal/urls"
|
|
)
|
|
|
|
// Gallery iterates the submissions in a user's main gallery, newest first.
|
|
//
|
|
// Each yielded *Submission carries only the fields visible on the listing
|
|
// page: ID, Title, Author (for favorites), ThumbURL, Rating, and the Tags
|
|
// / CategorizedTags parsed from the figure's data-tags attribute. Call
|
|
// [Client.GetSubmission] with the ID to load the full record.
|
|
func (c *Client) Gallery(ctx context.Context, name string, opts ListOptions, reqOpts ...Option) iter.Seq2[*Submission, error] {
|
|
return c.listGallerySection(ctx, name, urls.Gallery, opts, reqOpts)
|
|
}
|
|
|
|
// Scraps iterates the user's scraps folder. Same yield shape as Gallery.
|
|
func (c *Client) Scraps(ctx context.Context, name string, opts ListOptions, reqOpts ...Option) iter.Seq2[*Submission, error] {
|
|
return c.listGallerySection(ctx, name, urls.Scraps, opts, reqOpts)
|
|
}
|
|
|
|
// Favorites iterates the user's favorited submissions. The yielded
|
|
// *Submission's Author field reflects the original artist (not the user
|
|
// whose favorites we are walking).
|
|
func (c *Client) Favorites(ctx context.Context, name string, opts ListOptions, reqOpts ...Option) iter.Seq2[*Submission, error] {
|
|
return c.listGallerySection(ctx, name, urls.Favorites, opts, reqOpts)
|
|
}
|
|
|
|
// GalleryPage fetches a single page of /gallery/{name}/ and returns the
|
|
// items along with whether more pages exist. Pages are 1-based; pass 0 or
|
|
// 1 for the first page. Use this when driving pagination manually
|
|
// (resuming from a checkpoint, distributing pages across workers); use
|
|
// [Client.Gallery] when you just want every item in order.
|
|
func (c *Client) GalleryPage(ctx context.Context, name string, page int, reqOpts ...Option) (*ListingPage, error) {
|
|
return c.fetchListingPage(ctx, name, page, urls.Gallery, reqOpts)
|
|
}
|
|
|
|
// ScrapsPage is the single-page counterpart to [Client.Scraps]. See
|
|
// [Client.GalleryPage] for usage notes.
|
|
func (c *Client) ScrapsPage(ctx context.Context, name string, page int, reqOpts ...Option) (*ListingPage, error) {
|
|
return c.fetchListingPage(ctx, name, page, urls.Scraps, reqOpts)
|
|
}
|
|
|
|
// FavoritesPage is the single-page counterpart to [Client.Favorites]. See
|
|
// [Client.GalleryPage] for usage notes.
|
|
func (c *Client) FavoritesPage(ctx context.Context, name string, page int, reqOpts ...Option) (*ListingPage, error) {
|
|
return c.fetchListingPage(ctx, name, page, urls.Favorites, reqOpts)
|
|
}
|
|
|
|
// fetchListingPage is the shared per-page primitive used by
|
|
// GalleryPage / ScrapsPage / FavoritesPage and the iterator engine.
|
|
func (c *Client) fetchListingPage(
|
|
ctx context.Context,
|
|
name string,
|
|
page int,
|
|
urlFn func(string, int) string,
|
|
reqOpts []Option,
|
|
) (*ListingPage, error) {
|
|
if page < 1 {
|
|
page = 1
|
|
}
|
|
out := &ListingPage{Page: page}
|
|
err := c.fetch(ctx, urlFn(name, page), func(doc *goquery.Document) error {
|
|
out.Items, out.HasNext = parseGalleryPage(doc, c.cfg.jsonListings)
|
|
return nil
|
|
}, reqOpts...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// listGallerySection is the shared engine for Gallery / Scraps / Favorites.
|
|
// urlFn picks the section-specific URL builder; the rest of the pagination
|
|
// machinery is identical across all three sections.
|
|
func (c *Client) listGallerySection(
|
|
ctx context.Context,
|
|
name string,
|
|
urlFn func(string, int) string,
|
|
opts ListOptions,
|
|
reqOpts []Option,
|
|
) iter.Seq2[*Submission, error] {
|
|
return func(yield func(*Submission, error) bool) {
|
|
page := opts.firstPage()
|
|
pagesFetched := 0
|
|
for {
|
|
if opts.reachedLimit(pagesFetched) {
|
|
return
|
|
}
|
|
lp, err := c.fetchListingPage(ctx, name, page, urlFn, reqOpts)
|
|
if err != nil {
|
|
yield(nil, err)
|
|
return
|
|
}
|
|
pagesFetched++
|
|
if len(lp.Items) == 0 {
|
|
return
|
|
}
|
|
for _, s := range lp.Items {
|
|
if !yield(s, nil) {
|
|
return
|
|
}
|
|
}
|
|
if !lp.HasNext {
|
|
return
|
|
}
|
|
page++
|
|
}
|
|
}
|
|
}
|