210 lines
5.5 KiB
Go
210 lines
5.5 KiB
Go
package furaffinaty
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"net/http/cookiejar"
|
|
"net/url"
|
|
"strings"
|
|
|
|
"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty/models"
|
|
"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty/utils"
|
|
"github.com/gocolly/colly/v2"
|
|
)
|
|
|
|
// Client is the main client for interacting with Furaffinity .
|
|
type Client struct {
|
|
RequestContext models.RequestContext
|
|
submissionIDs []string
|
|
submission []models.Submission
|
|
}
|
|
|
|
// NewClient creates a new Furaffinity client with the provided cookies.
|
|
func NewClient(username string, cookies models.Cookie, collyOptions []colly.CollectorOption) Client {
|
|
|
|
client := colly.NewCollector(collyOptions...)
|
|
|
|
client.UserAgent = fmt.Sprintf("Go-Furaffinity-SDK used by %s | (made by the Anthrove Team)", username)
|
|
|
|
err := client.Limit(&colly.LimitRule{
|
|
DomainGlob: "*",
|
|
Delay: models.RateLimit,
|
|
})
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
jar, err := cookiejar.New(nil)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
cookiesForJar := []*http.Cookie{
|
|
{
|
|
Name: "a",
|
|
Value: cookies.A,
|
|
Path: "/",
|
|
Domain: ".furaffinity.net",
|
|
Secure: true,
|
|
HttpOnly: true,
|
|
},
|
|
{
|
|
Name: "b",
|
|
Value: cookies.B,
|
|
Path: "/",
|
|
Domain: ".furaffinity.net",
|
|
Secure: true,
|
|
HttpOnly: true,
|
|
},
|
|
}
|
|
|
|
parsedUrl, _ := url.Parse(models.Endpoint)
|
|
jar.SetCookies(parsedUrl, cookiesForJar)
|
|
|
|
// Set Cookies
|
|
client.SetCookieJar(jar)
|
|
|
|
return Client{
|
|
RequestContext: models.RequestContext{
|
|
Client: client,
|
|
Host: models.Endpoint,
|
|
},
|
|
}
|
|
}
|
|
|
|
func (c *Client) GetSubmissionIDs() ([]string, error) {
|
|
|
|
submissionURL := fmt.Sprintf("%s/msg/submissions/", c.RequestContext.Host)
|
|
|
|
// Parse the HTML and get everything inside the figure tags into a struct
|
|
c.RequestContext.Client.OnHTML("figure", func(e *colly.HTMLElement) {
|
|
id := strings.Split(e.Attr("id"), "sid-")
|
|
c.submissionIDs = append(c.submissionIDs, id[1])
|
|
})
|
|
|
|
err := c.RequestContext.Client.Visit(submissionURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return c.submissionIDs, nil
|
|
}
|
|
|
|
func (c *Client) GetSubmissionIDsForFavorites(username string) ([]string, error) {
|
|
submissionURL := fmt.Sprintf("%s/favorites/%s", c.RequestContext.Host, username)
|
|
nextSubmissionURL := ""
|
|
|
|
// Set up the OnHTML callback for the "Next" button
|
|
c.RequestContext.Client.OnHTML("a.standard", func(e *colly.HTMLElement) {
|
|
nextURL := e.Attr("href")
|
|
if nextURL != "" {
|
|
nextSubmissionURL = fmt.Sprintf("%s%s", c.RequestContext.Host, nextURL)
|
|
}
|
|
})
|
|
|
|
for {
|
|
newSubmissionIDs := []string{}
|
|
|
|
// Parse the HTML and get everything inside the figure tags into a struct
|
|
c.RequestContext.Client.OnHTML("figure", func(e *colly.HTMLElement) {
|
|
id := strings.Split(e.Attr("id"), "sid-")
|
|
if len(id) > 1 {
|
|
newSubmissionIDs = append(newSubmissionIDs, id[1])
|
|
}
|
|
})
|
|
|
|
err := c.RequestContext.Client.Visit(submissionURL)
|
|
if err != nil {
|
|
log.Println(err)
|
|
break
|
|
}
|
|
|
|
log.Printf("Next Submission URL: %s", nextSubmissionURL)
|
|
|
|
if len(newSubmissionIDs) == 0 {
|
|
break // Exit loop if no new IDs found
|
|
}
|
|
|
|
c.submissionIDs = append(c.submissionIDs, newSubmissionIDs...)
|
|
|
|
if nextSubmissionURL == "" {
|
|
break // Exit loop if no next page found
|
|
}
|
|
|
|
submissionURL = nextSubmissionURL // Update to next page
|
|
}
|
|
|
|
return c.submissionIDs, nil
|
|
}
|
|
|
|
// GetSubmissions
|
|
// Set limit to 0 if you want all submissions
|
|
func (c *Client) GetSubmissions(limit int) ([]models.Submission, error) {
|
|
submissionLength := 0
|
|
|
|
if c.submissionIDs == nil {
|
|
log.Println("No IDs found, scraping IDs...")
|
|
_, err := c.GetSubmissionIDs()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if limit == 0 {
|
|
limit = len(c.submissionIDs)
|
|
submissionLength = len(c.submissionIDs)
|
|
} else {
|
|
submissionLength = limit
|
|
}
|
|
|
|
for i, id := range c.submissionIDs[:limit] {
|
|
log.Printf("Working on submission: %s | Number: %d, from: %d", id, i, submissionLength)
|
|
submissionURL := fmt.Sprintf("%s/view/%s", c.RequestContext.Host, id)
|
|
var submission models.Submission
|
|
|
|
submission.ID = id
|
|
submission.Url = submissionURL
|
|
|
|
c.RequestContext.Client.OnHTML("#submissionImg", func(e *colly.HTMLElement) {
|
|
submission.Image.DataFullviewSrc = e.Attr("data-fullview-src")
|
|
submission.Image.DataPreviewSrc = e.Attr("data-preview-src")
|
|
submission.Image.Src = e.Attr("src")
|
|
})
|
|
|
|
c.RequestContext.Client.OnHTML(".section-header", func(e *colly.HTMLElement) {
|
|
submission.Artist.AvatarURL = e.ChildAttr(".submission-user-icon", "src")
|
|
submission.Title = e.ChildText(".submission-title > h2:nth-child(1) > p:nth-child(1)")
|
|
submission.Artist.Name = e.ChildText(".submission-id-sub-container > a:nth-child(2) > strong:nth-child(1)")
|
|
|
|
parsedDate, err := utils.ParseDate(e.ChildAttr(".popup_date", "title"))
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
submission.Information.PostDate = parsedDate
|
|
})
|
|
c.RequestContext.Client.OnHTML(".info", func(e *colly.HTMLElement) {
|
|
submission.Information.Category.Name = e.ChildText("div:nth-child(1) > span.category-name")
|
|
submission.Information.Category.Type = e.ChildText("div:nth-child(1) > span.type-name")
|
|
submission.Information.Species = e.ChildText("div:nth-child(2) > span")
|
|
submission.Information.Gender = e.ChildText("div:nth-child(3) > span")
|
|
})
|
|
|
|
c.RequestContext.Client.OnHTML(".tags-row", func(e *colly.HTMLElement) {
|
|
e.ForEach(".tags", func(_ int, el *colly.HTMLElement) {
|
|
tag := el.ChildText("a")
|
|
submission.Tags = append(submission.Tags, tag)
|
|
})
|
|
})
|
|
|
|
err := c.RequestContext.Client.Visit(submissionURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
c.submission = append(c.submission, submission)
|
|
|
|
}
|
|
return c.submission, nil
|
|
|
|
}
|