feat: implement basic structure with dockerfile

build/package/Dockerfile

@@ -0,0 +1,30 @@
+FROM golang:alpine as builder
+
+WORKDIR /go
+
+# Install dependencies
+RUN apk add -U --no-cache ca-certificates && update-ca-certificates && go install github.com/swaggo/swag/cmd/swag@latest
+
+# Cache dependencies
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy source code
+COPY . ./
+
+# Build the application
+RUN CGO_ENABLED=0 GOOS=linux go build  -a -installsuffix cgo -ldflags "-w -s" -o /app ./cmd/playground/
+
+FROM scratch
+
+ARG VERSION
+ENV VERSION=$VERSION
+
+WORKDIR /
+
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /app ./
+COPY web ./web
+
+EXPOSE 8080
+CMD ["/app"]

cmd/playground/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"git.anthrove.art/Anthrove/gorse-playground/internal/logic"
 	"git.anthrove.art/Anthrove/gorse-playground/pkg/models"
 	"git.anthrove.art/Anthrove/gorse-playground/pkg/utils"
@@ -9,9 +10,18 @@ import (
 	"github.com/gin-gonic/gin"
 	"net/http"
 	"strconv"
+	"time"
 )
 
 func main() {
+	/*err := logic.SubmitItems(context.Background())
+
+	if err != nil {
+		panic(err)
+	}*/
+
+	go Routine(context.Background())
+
 	router := gin.Default()
 	store := cookie.NewStore([]byte("secret"))
 	router.Use(sessions.Sessions("mysession", store))
@@ -126,7 +136,48 @@ func main() {
 
 		c.HTML(http.StatusOK, "post.gohtml", gin.H{"recs": recs, "next_page": pageInt + 1, "last_page": pageInt - 1})
 	})
+	router.POST("/like/:id", func(c *gin.Context) {
+		session := sessions.Default(c)
+		userid := session.Get("userid")
+		id := c.Param("id")
+
+		err := logic.UpsertFavorites(c, []models.GorseFavorite{{
+			Comment:      "",
+			FeedbackType: "like",
+			ItemId:       id,
+			Timestamp:    time.Now().String(),
+			UserId:       userid.(string),
+		}})
+
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"upsert favorite error": err.Error()})
+		}
+
+		c.JSON(http.StatusOK, gin.H{"item": id})
+	})
 
 	router.Run(":8080")
 
 }
+
+func Routine(c context.Context) {
+	now := time.Now()
+	next := time.Date(now.Year(), now.Month(), now.Day(), 4, 0, 0, 0, now.Location())
+	if now.After(next) {
+		next = next.Add(24 * time.Hour)
+	}
+	duration := next.Sub(now)
+	time.Sleep(duration)
+	ticker := time.NewTicker(24 * time.Hour)
+
+	for {
+		select {
+		case <-c.Done():
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			logic.SubmitItems(c)
+		}
+	}
+
+}

internal/logic/e621.go

@@ -5,11 +5,18 @@ import (
 	"encoding/base64"
 	"fmt"
 	"git.anthrove.art/Anthrove/gorse-playground/internal/config"
+	"git.anthrove.art/Anthrove/gorse-playground/pkg/e621"
+	"git.anthrove.art/Anthrove/gorse-playground/pkg/models"
+	"git.anthrove.art/Anthrove/gorse-playground/pkg/utils"
 	"github.com/anthrove/openapi-e621-go"
 	"github.com/caarlos0/env/v11"
 	"golang.org/x/time/rate"
 	"log"
 	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
 
 	_ "github.com/joho/godotenv/autoload"
 )
@@ -80,6 +87,83 @@ func GetFavoritePage(ctx context.Context, userId int, pageIdentifier int) ([]ope
 	return favorites.Posts, nil
 }
 
+func SubmitItems(ctx context.Context) error {
+	currentDate := time.Now().Format("2006-01-02")
+	err := utils.DownloadE6Data(ctx, "posts-"+currentDate+".csv.gz", "post-file.csv")
+
+	if err != nil {
+		return err
+	}
+
+	fileReader, err := os.Open("post-file.csv")
+
+	if err != nil {
+		return err
+	}
+
+	inputE621PostChannel := make(chan e621.Post)
+	outputAnthrovePostChannel := make(chan models.GorseItem)
+	postChan := utils.GetStreamingData[e621.Post](ctx, fileReader)
+
+	go func() {
+		defer close(inputE621PostChannel)
+		for post := range postChan {
+			inputE621PostChannel <- post
+		}
+		log.Println("Loading ended")
+	}()
+
+	go func() {
+		defer close(outputAnthrovePostChannel)
+		err := postToItem(inputE621PostChannel, outputAnthrovePostChannel)
+		if err != nil { //TODO: DEADLOCK
+			log.Println(err)
+		}
+		log.Println("Convert ended")
+	}()
+
+	log.Println("Start with comparison check")
+
+	items := make([]models.GorseItem, 0)
+	length := 0
+
+	for item := range outputAnthrovePostChannel {
+		timeDate, err := time.Parse(time.DateTime, item.Timestamp)
+
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
+		if !timeDate.After(time.Date(2024, 1, 1, 1, 1, 1, 0, time.UTC)) {
+			continue
+		}
+
+		items = append(items, item)
+
+		if length%20_000 == 0 && length > 0 {
+			log.Println("Worked ", length, " items")
+			err := UpsertItems(ctx, items)
+
+			if err != nil {
+				return err
+			}
+
+			items = make([]models.GorseItem, 0)
+		}
+
+		length++
+	}
+
+	err = UpsertItems(ctx, items)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func newRateMiddleware(transport *http.Transport) http.RoundTripper {
 	return &rateMiddleware{
 		transport: transport,
@@ -99,3 +183,19 @@ func (r rateMiddleware) RoundTrip(request *http.Request) (*http.Response, error)
 
 	return r.transport.RoundTrip(request)
 }
+
+func postToItem(input chan e621.Post, output chan models.GorseItem) error {
+	for e6Post := range input {
+
+		tagParts := strings.Split(e6Post.TagString, " ")
+
+		output <- models.GorseItem{
+			Comment:   e6Post.Description,
+			IsHidden:  e6Post.IsDeleted,
+			ItemId:    strconv.Itoa(e6Post.ID),
+			Labels:    tagParts,
+			Timestamp: e6Post.CreatedAt,
+		}
+	}
+	return nil
+}

internal/logic/gorse.go

@@ -44,12 +44,14 @@ func GetUserFavorites(ctx context.Context, userid string, page int) ([]string, e
 	}
 
 	q := req.URL.Query()
-	q.Set("name", "50")
+	q.Set("n", "20")
-	q.Set("offset", strconv.Itoa((page-1)*50))
+	q.Set("offset", strconv.Itoa((page-1)*20))
+	req.URL.RawQuery = q.Encode()
 
 	req = req.WithContext(ctx)
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("X-API-Key", gorseConfig.ApiKey)
+
 	resp, err := client.Do(req)
 	if err != nil {
 		return nil, err

pkg/e621/model.go

@@ -0,0 +1,33 @@
+package e621
+
+type Post struct {
+	ID             int    `csv:"id"`
+	UploaderID     int    `csv:"uploader_id"`
+	CreatedAt      string `csv:"created_at"`
+	MD5            string `csv:"md5"`
+	Source         string `csv:"source"`
+	Rating         string `csv:"rating"`
+	ImageWidth     int    `csv:"image_width"`
+	ImageHeight    int    `csv:"image_height"`
+	TagString      string `csv:"tag_string"`
+	LockedTags     string `csv:"locked_tags"`
+	FavCount       int    `csv:"fav_count"`
+	FileExt        string `csv:"file_ext"`
+	ParentID       int    `csv:"parent_id"`
+	ChangeSeq      int    `csv:"change_seq"`
+	ApproverID     int    `csv:"approver_id"`
+	FileSize       int    `csv:"file_size"`
+	CommentCount   int    `csv:"comment_count"`
+	Description    string `csv:"description"`
+	Duration       int    `csv:"duration"`
+	UpdatedAt      string `csv:"updated_at"`
+	IsDeleted      bool   `csv:"is_deleted"`
+	IsPending      bool   `csv:"is_pending"`
+	IsFlagged      bool   `csv:"is_flagged"`
+	Score          int    `csv:"score"`
+	UpScore        int    `csv:"up_score"`
+	DownScore      int    `csv:"down_score"`
+	IsRatingLocked bool   `csv:"is_rating_locked"`
+	IsStatusLocked bool   `csv:"is_status_locked"`
+	IsNoteLocked   bool   `csv:"is_note_locked"`
+}

pkg/utils/e621.go

@@ -0,0 +1,57 @@
+package utils
+
+import (
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+)
+
+var httpClient http.Client
+
+func DownloadE6Data(ctx context.Context, filename string, targetPath string) error {
+	req, err := buildE6Request(fmt.Sprintf("/db_export/%s", filename))
+
+	if err != nil {
+		return err
+	}
+
+	req = req.WithContext(ctx)
+	resp, err := httpClient.Do(req)
+
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	uncompressedStream, err := gzip.NewReader(resp.Body)
+
+	if err != nil {
+		return err
+	}
+	defer uncompressedStream.Close()
+
+	out, err := os.Create(targetPath)
+
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+	_, err = io.Copy(out, uncompressedStream)
+
+	return err
+}
+
+func buildE6Request(url string) (*http.Request, error) {
+	request, err := http.NewRequest("GET", fmt.Sprintf("%s%s", "https://e621.net", url), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Add("User-Agent", "Anthrove downloader (by alphyron)")
+
+	return request, nil
+}

pkg/utils/streaming.go

@@ -0,0 +1,117 @@
+package utils
+
+import (
+	"context"
+	"encoding/csv"
+	"io"
+	"log"
+	"os"
+	"reflect"
+	"strconv"
+	"time"
+)
+
+func GetStreamingFileData[T any](ctx context.Context, filePath string) chan T {
+	csvIn, err := os.Open(filePath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return GetStreamingData[T](ctx, csvIn)
+}
+
+func GetStreamingData[T any](ctx context.Context, rc io.Reader) chan T {
+	ch := make(chan T)
+	go func() {
+		inputChan := make(chan []string)
+		r := csv.NewReader(rc)
+		var header []string
+		var err error
+		if header, err = r.Read(); err != nil {
+			log.Fatal(err)
+		}
+		defer close(inputChan)
+		go func() {
+			defer close(ch)
+			returnChannel := parseRecord[T](header, inputChan)
+			for data := range returnChannel {
+				ch <- data
+			}
+		}()
+
+		for {
+			rec, err := r.Read()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				log.Fatal(err)
+			}
+
+			if len(rec) == 0 {
+				continue
+			}
+
+			inputChan <- rec
+		}
+		log.Println("Input finished")
+	}()
+	return ch
+}
+
+func parseRecord[T any](header []string, input chan []string) chan T {
+	channel := make(chan T)
+	go func() {
+		defer close(channel)
+		var e T
+		et := reflect.TypeOf(e)
+		var headers = make(map[string]int, et.NumField())
+		for i := 0; i < et.NumField(); i++ {
+			headers[et.Field(i).Name] = func(element string, array []string) int {
+				for k, v := range array {
+					if v == element {
+						return k
+					}
+				}
+				return -1
+			}(et.Field(i).Tag.Get("csv"), header)
+		}
+		for record := range input {
+			if len(record) == 0 {
+				continue
+			}
+
+			for h, i := range headers {
+				if i == -1 {
+					continue
+				}
+				elem := reflect.ValueOf(&e).Elem()
+				field := elem.FieldByName(h)
+				if field.CanSet() {
+					switch field.Type().Name() {
+					case "bool":
+						a, _ := strconv.ParseBool(record[i])
+						field.Set(reflect.ValueOf(a))
+					case "int":
+						a, _ := strconv.Atoi(record[i])
+						field.Set(reflect.ValueOf(a))
+					case "float64":
+						a, _ := strconv.ParseFloat(record[i], 64)
+						field.Set(reflect.ValueOf(a))
+					case "Time":
+						a, _ := time.Parse("2006-01-02T00:00:00Z", record[i])
+						field.Set(reflect.ValueOf(a))
+					case "string":
+						field.Set(reflect.ValueOf(record[i]))
+					default:
+						log.Printf("Unknown Fieldtype: %s\n", field.Type().Name())
+						field.Set(reflect.ValueOf(record[i]))
+					}
+				}
+			}
+			channel <- e
+		}
+		log.Println("parsing ended")
+	}()
+	return channel
+}

web/template/page/post.gohtml

@@ -6,7 +6,7 @@
 <h1>Available Recommondations</h1>
 <div>
     {{range .recs}}
-        <a href="https://e621.net/posts/{{.}}" target="_blank">{{.}}</a><br>
+        <a href="https://e621.net/posts/{{.}}" target="_blank">{{.}}</a><button onclick="rec({{.}})">Like</button><br>
     {{end}}
     <form method="get">
         <input value="{{ .last_page }}" style="display: none" name="page">
@@ -17,5 +17,13 @@
         <button type="submit">Next Page</button>
     </form>
 </div>
+<script>
+    function rec(id){
+        const xmlHttp = new XMLHttpRequest();
+        xmlHttp.open( "POST",window.location.origin + "/like/" + id, false ); // false for synchronous request
+        xmlHttp.send( null );
+        return xmlHttp.responseText;
+    }
+</script>
 </body>
 </html>