Compare commits

..

4 Commits

Author SHA1 Message Date
98558dcfbf feat: save to json
added the ability to save the output as json
2024-08-09 12:05:18 +02:00
54eb37ac3a feat: submission url
Add URL field to Submission struct in Furaffinaty package
2024-08-09 12:04:57 +02:00
dfca20a3ec fix: crash on visited page & logging
Update error handling and logging in Furaffinaty client
2024-08-09 12:04:37 +02:00
3390a05c70 feat: user favorite
Add function to get all favorite submission IDs for a user
2024-08-09 11:35:04 +02:00
3 changed files with 97 additions and 12 deletions

39
main.go
View File

@ -1,14 +1,13 @@
package main
import (
"encoding/json"
"log"
"os"
"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty"
"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty/models"
"github.com/davecgh/go-spew/spew"
"github.com/gocolly/colly/v2"
"github.com/gocolly/colly/v2/debug"
_ "github.com/joho/godotenv/autoload"
)
@ -20,12 +19,12 @@ func main() {
}
collyOptions := []colly.CollectorOption{
colly.Debugger(&debug.LogDebugger{}),
//colly.Debugger(&debug.LogDebugger{}),
}
faClient := furaffinaty.NewClient("SoXX", cookies, collyOptions)
SubmissionIDs, err := faClient.GetSubmissionIDs()
/* SubmissionIDs, err := faClient.GetSubmissionIDs()
if err != nil {
log.Fatal(err)
}
@ -35,5 +34,35 @@ func main() {
if err != nil {
log.Fatal(err)
}
spew.Dump(submissions)
spew.Dump(submissions)*/
_, err := faClient.GetSubmissionIDsForFavorites("soxx-thefennec")
if err != nil {
log.Fatal(err)
}
favorites, err := faClient.GetSubmissions(5)
saveSubmissionsToJSON(favorites, "favorites.json")
}
func saveSubmissionsToJSON(submissions []models.Submission, filename string) {
jsonData, err := json.MarshalIndent(submissions, "", " ")
if err != nil {
log.Fatalf("Error marshalling submissions to JSON: %v", err)
return
}
file, err := os.Create(filename)
if err != nil {
log.Fatalf("Error creating file: %v", err)
return
}
defer file.Close()
_, err = file.Write(jsonData)
if err != nil {
log.Fatalf("Error writing to file: %v", err)
return
}
}

View File

@ -90,9 +90,57 @@ func (c *Client) GetSubmissionIDs() ([]string, error) {
return c.submissionIDs, nil
}
func (c *Client) GetSubmissionIDsForFavorites(username string) ([]string, error) {
submissionURL := fmt.Sprintf("%s/favorites/%s", c.RequestContext.Host, username)
nextSubmissionURL := ""
// Set up the OnHTML callback for the "Next" button
c.RequestContext.Client.OnHTML("a.standard", func(e *colly.HTMLElement) {
nextURL := e.Attr("href")
if nextURL != "" {
nextSubmissionURL = fmt.Sprintf("%s%s", c.RequestContext.Host, nextURL)
}
})
for {
newSubmissionIDs := []string{}
// Parse the HTML and get everything inside the figure tags into a struct
c.RequestContext.Client.OnHTML("figure", func(e *colly.HTMLElement) {
id := strings.Split(e.Attr("id"), "sid-")
if len(id) > 1 {
newSubmissionIDs = append(newSubmissionIDs, id[1])
}
})
err := c.RequestContext.Client.Visit(submissionURL)
if err != nil {
log.Println(err)
break
}
log.Printf("Next Submission URL: %s", nextSubmissionURL)
if len(newSubmissionIDs) == 0 {
break // Exit loop if no new IDs found
}
c.submissionIDs = append(c.submissionIDs, newSubmissionIDs...)
if nextSubmissionURL == "" {
break // Exit loop if no next page found
}
submissionURL = nextSubmissionURL // Update to next page
}
return c.submissionIDs, nil
}
// GetSubmissions
// Set limit to 0 if you want all submissions
func (c *Client) GetSubmissions(limit int) ([]models.Submission, error) {
submissionLength := 0
if c.submissionIDs == nil {
log.Println("No IDs found, scraping IDs...")
@ -104,13 +152,18 @@ func (c *Client) GetSubmissions(limit int) ([]models.Submission, error) {
if limit == 0 {
limit = len(c.submissionIDs)
submissionLength = len(c.submissionIDs)
} else {
submissionLength = limit
}
for _, id := range c.submissionIDs[:limit] {
for i, id := range c.submissionIDs[:limit] {
log.Printf("Working on submission: %s | Number: %d, from: %d", id, i, submissionLength)
submissionURL := fmt.Sprintf("%s/view/%s", c.RequestContext.Host, id)
var submission models.Submission
submission.ID = id
submission.Url = submissionURL
c.RequestContext.Client.OnHTML("#submissionImg", func(e *colly.HTMLElement) {
submission.Image.DataFullviewSrc = e.Attr("data-fullview-src")

View File

@ -1,9 +1,12 @@
package models
import "time"
import (
"time"
)
type Submission struct {
ID string `json:"id,omitempty"`
Url string `json:"url,omitempty"`
Title string `json:"title,omitempty"`
Tags []string `json:"tags,omitempty"`
Artist Artist `json:"artist"`