Compare commits
4 Commits
6244396954
...
98558dcfbf
Author | SHA1 | Date | |
---|---|---|---|
98558dcfbf | |||
54eb37ac3a | |||
dfca20a3ec | |||
3390a05c70 |
49
main.go
49
main.go
@ -1,14 +1,13 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty"
|
"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty"
|
||||||
"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty/models"
|
"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty/models"
|
||||||
"github.com/davecgh/go-spew/spew"
|
|
||||||
"github.com/gocolly/colly/v2"
|
"github.com/gocolly/colly/v2"
|
||||||
"github.com/gocolly/colly/v2/debug"
|
|
||||||
_ "github.com/joho/godotenv/autoload"
|
_ "github.com/joho/godotenv/autoload"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -20,20 +19,50 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
collyOptions := []colly.CollectorOption{
|
collyOptions := []colly.CollectorOption{
|
||||||
colly.Debugger(&debug.LogDebugger{}),
|
//colly.Debugger(&debug.LogDebugger{}),
|
||||||
}
|
}
|
||||||
|
|
||||||
faClient := furaffinaty.NewClient("SoXX", cookies, collyOptions)
|
faClient := furaffinaty.NewClient("SoXX", cookies, collyOptions)
|
||||||
|
|
||||||
SubmissionIDs, err := faClient.GetSubmissionIDs()
|
/* SubmissionIDs, err := faClient.GetSubmissionIDs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
spew.Dump(SubmissionIDs)
|
spew.Dump(SubmissionIDs)
|
||||||
|
|
||||||
submissions, err := faClient.GetSubmissions(1)
|
submissions, err := faClient.GetSubmissions(1)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
spew.Dump(submissions)*/
|
||||||
|
|
||||||
|
_, err := faClient.GetSubmissionIDsForFavorites("soxx-thefennec")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
spew.Dump(submissions)
|
|
||||||
|
favorites, err := faClient.GetSubmissions(5)
|
||||||
|
saveSubmissionsToJSON(favorites, "favorites.json")
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func saveSubmissionsToJSON(submissions []models.Submission, filename string) {
|
||||||
|
jsonData, err := json.MarshalIndent(submissions, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error marshalling submissions to JSON: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Create(filename)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error creating file: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
_, err = file.Write(jsonData)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error writing to file: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -90,9 +90,57 @@ func (c *Client) GetSubmissionIDs() ([]string, error) {
|
|||||||
return c.submissionIDs, nil
|
return c.submissionIDs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Client) GetSubmissionIDsForFavorites(username string) ([]string, error) {
|
||||||
|
submissionURL := fmt.Sprintf("%s/favorites/%s", c.RequestContext.Host, username)
|
||||||
|
nextSubmissionURL := ""
|
||||||
|
|
||||||
|
// Set up the OnHTML callback for the "Next" button
|
||||||
|
c.RequestContext.Client.OnHTML("a.standard", func(e *colly.HTMLElement) {
|
||||||
|
nextURL := e.Attr("href")
|
||||||
|
if nextURL != "" {
|
||||||
|
nextSubmissionURL = fmt.Sprintf("%s%s", c.RequestContext.Host, nextURL)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
for {
|
||||||
|
newSubmissionIDs := []string{}
|
||||||
|
|
||||||
|
// Parse the HTML and get everything inside the figure tags into a struct
|
||||||
|
c.RequestContext.Client.OnHTML("figure", func(e *colly.HTMLElement) {
|
||||||
|
id := strings.Split(e.Attr("id"), "sid-")
|
||||||
|
if len(id) > 1 {
|
||||||
|
newSubmissionIDs = append(newSubmissionIDs, id[1])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
err := c.RequestContext.Client.Visit(submissionURL)
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Next Submission URL: %s", nextSubmissionURL)
|
||||||
|
|
||||||
|
if len(newSubmissionIDs) == 0 {
|
||||||
|
break // Exit loop if no new IDs found
|
||||||
|
}
|
||||||
|
|
||||||
|
c.submissionIDs = append(c.submissionIDs, newSubmissionIDs...)
|
||||||
|
|
||||||
|
if nextSubmissionURL == "" {
|
||||||
|
break // Exit loop if no next page found
|
||||||
|
}
|
||||||
|
|
||||||
|
submissionURL = nextSubmissionURL // Update to next page
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.submissionIDs, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetSubmissions
|
// GetSubmissions
|
||||||
// Set limit to 0 if you want all submissions
|
// Set limit to 0 if you want all submissions
|
||||||
func (c *Client) GetSubmissions(limit int) ([]models.Submission, error) {
|
func (c *Client) GetSubmissions(limit int) ([]models.Submission, error) {
|
||||||
|
submissionLength := 0
|
||||||
|
|
||||||
if c.submissionIDs == nil {
|
if c.submissionIDs == nil {
|
||||||
log.Println("No IDs found, scraping IDs...")
|
log.Println("No IDs found, scraping IDs...")
|
||||||
@ -104,13 +152,18 @@ func (c *Client) GetSubmissions(limit int) ([]models.Submission, error) {
|
|||||||
|
|
||||||
if limit == 0 {
|
if limit == 0 {
|
||||||
limit = len(c.submissionIDs)
|
limit = len(c.submissionIDs)
|
||||||
|
submissionLength = len(c.submissionIDs)
|
||||||
|
} else {
|
||||||
|
submissionLength = limit
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, id := range c.submissionIDs[:limit] {
|
for i, id := range c.submissionIDs[:limit] {
|
||||||
|
log.Printf("Working on submission: %s | Number: %d, from: %d", id, i, submissionLength)
|
||||||
submissionURL := fmt.Sprintf("%s/view/%s", c.RequestContext.Host, id)
|
submissionURL := fmt.Sprintf("%s/view/%s", c.RequestContext.Host, id)
|
||||||
var submission models.Submission
|
var submission models.Submission
|
||||||
|
|
||||||
submission.ID = id
|
submission.ID = id
|
||||||
|
submission.Url = submissionURL
|
||||||
|
|
||||||
c.RequestContext.Client.OnHTML("#submissionImg", func(e *colly.HTMLElement) {
|
c.RequestContext.Client.OnHTML("#submissionImg", func(e *colly.HTMLElement) {
|
||||||
submission.Image.DataFullviewSrc = e.Attr("data-fullview-src")
|
submission.Image.DataFullviewSrc = e.Attr("data-fullview-src")
|
||||||
|
@ -1,9 +1,12 @@
|
|||||||
package models
|
package models
|
||||||
|
|
||||||
import "time"
|
import (
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
type Submission struct {
|
type Submission struct {
|
||||||
ID string `json:"id,omitempty"`
|
ID string `json:"id,omitempty"`
|
||||||
|
Url string `json:"url,omitempty"`
|
||||||
Title string `json:"title,omitempty"`
|
Title string `json:"title,omitempty"`
|
||||||
Tags []string `json:"tags,omitempty"`
|
Tags []string `json:"tags,omitempty"`
|
||||||
Artist Artist `json:"artist"`
|
Artist Artist `json:"artist"`
|
||||||
|
Loading…
Reference in New Issue
Block a user