feat: save to json

added the ability to save the output as json

main.go

@@ -1,14 +1,13 @@
 package main
 
 import (
+	"encoding/json"
 	"log"
 	"os"
 
 	"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty"
 	"git.anthrove.art/Anthrove/fa-sdk-go/pkg/furaffinaty/models"
-	"github.com/davecgh/go-spew/spew"
 	"github.com/gocolly/colly/v2"
-	"github.com/gocolly/colly/v2/debug"
 	_ "github.com/joho/godotenv/autoload"
 )
 
@@ -20,12 +19,12 @@ func main() {
 	}
 
 	collyOptions := []colly.CollectorOption{
-		colly.Debugger(&debug.LogDebugger{}),
+		//colly.Debugger(&debug.LogDebugger{}),
 	}
 
 	faClient := furaffinaty.NewClient("SoXX", cookies, collyOptions)
 
-	SubmissionIDs, err := faClient.GetSubmissionIDs()
+	/*	SubmissionIDs, err := faClient.GetSubmissionIDs()
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -35,5 +34,35 @@ func main() {
 		if err != nil {
 			log.Fatal(err)
 		}
-	spew.Dump(submissions)
+		spew.Dump(submissions)*/
+
+	_, err := faClient.GetSubmissionIDsForFavorites("soxx-thefennec")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	favorites, err := faClient.GetSubmissions(5)
+	saveSubmissionsToJSON(favorites, "favorites.json")
+
+}
+
+func saveSubmissionsToJSON(submissions []models.Submission, filename string) {
+	jsonData, err := json.MarshalIndent(submissions, "", "  ")
+	if err != nil {
+		log.Fatalf("Error marshalling submissions to JSON: %v", err)
+		return
+	}
+
+	file, err := os.Create(filename)
+	if err != nil {
+		log.Fatalf("Error creating file: %v", err)
+		return
+	}
+	defer file.Close()
+
+	_, err = file.Write(jsonData)
+	if err != nil {
+		log.Fatalf("Error writing to file: %v", err)
+		return
+	}
 }

pkg/furaffinaty/client.go

@@ -90,9 +90,57 @@ func (c *Client) GetSubmissionIDs() ([]string, error) {
 	return c.submissionIDs, nil
 }
 
+func (c *Client) GetSubmissionIDsForFavorites(username string) ([]string, error) {
+	submissionURL := fmt.Sprintf("%s/favorites/%s", c.RequestContext.Host, username)
+	nextSubmissionURL := ""
+
+	// Set up the OnHTML callback for the "Next" button
+	c.RequestContext.Client.OnHTML("a.standard", func(e *colly.HTMLElement) {
+		nextURL := e.Attr("href")
+		if nextURL != "" {
+			nextSubmissionURL = fmt.Sprintf("%s%s", c.RequestContext.Host, nextURL)
+		}
+	})
+
+	for {
+		newSubmissionIDs := []string{}
+
+		// Parse the HTML and get everything inside the figure tags into a struct
+		c.RequestContext.Client.OnHTML("figure", func(e *colly.HTMLElement) {
+			id := strings.Split(e.Attr("id"), "sid-")
+			if len(id) > 1 {
+				newSubmissionIDs = append(newSubmissionIDs, id[1])
+			}
+		})
+
+		err := c.RequestContext.Client.Visit(submissionURL)
+		if err != nil {
+			log.Println(err)
+			break
+		}
+
+		log.Printf("Next Submission URL: %s", nextSubmissionURL)
+
+		if len(newSubmissionIDs) == 0 {
+			break // Exit loop if no new IDs found
+		}
+
+		c.submissionIDs = append(c.submissionIDs, newSubmissionIDs...)
+
+		if nextSubmissionURL == "" {
+			break // Exit loop if no next page found
+		}
+
+		submissionURL = nextSubmissionURL // Update to next page
+	}
+
+	return c.submissionIDs, nil
+}
+
 // GetSubmissions
 // Set limit to 0 if you want all submissions
 func (c *Client) GetSubmissions(limit int) ([]models.Submission, error) {
+	submissionLength := 0
 
 	if c.submissionIDs == nil {
 		log.Println("No IDs found, scraping IDs...")
@@ -104,13 +152,18 @@ func (c *Client) GetSubmissions(limit int) ([]models.Submission, error) {
 
 	if limit == 0 {
 		limit = len(c.submissionIDs)
+		submissionLength = len(c.submissionIDs)
+	} else {
+		submissionLength = limit
 	}
 
-	for _, id := range c.submissionIDs[:limit] {
+	for i, id := range c.submissionIDs[:limit] {
+		log.Printf("Working on submission: %s | Number: %d, from: %d", id, i, submissionLength)
 		submissionURL := fmt.Sprintf("%s/view/%s", c.RequestContext.Host, id)
 		var submission models.Submission
 
 		submission.ID = id
+		submission.Url = submissionURL
 
 		c.RequestContext.Client.OnHTML("#submissionImg", func(e *colly.HTMLElement) {
 			submission.Image.DataFullviewSrc = e.Attr("data-fullview-src")

pkg/furaffinaty/models/submission.go

@@ -1,9 +1,12 @@
 package models
 
-import "time"
+import (
+	"time"
+)
 
 type Submission struct {
 	ID          string          `json:"id,omitempty"`
+	Url         string          `json:"url,omitempty"`
 	Title       string          `json:"title,omitempty"`
 	Tags        []string        `json:"tags,omitempty"`
 	Artist      Artist          `json:"artist"`