pkg/database/migrations/004_scrape_history.sql

@@ -0,0 +1,11 @@
+-- +migrate Up
+CREATE TABLE "ScrapeHistory"
+(
+    scrape_task_id CHAR(25) PRIMARY KEY,
+    user_source_id CHAR(25)  NOT NULL REFERENCES "UserSource" (id),
+    created_at     TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    finished_at    TIMESTAMP NULL,
+    error          TEXT      NULL,
+    added_posts    INT       NOT NULL,
+    deleted_posts  INT       NOT NULL
+);

pkg/database/scrape_history.go

@@ -0,0 +1,182 @@
+package database
+
+import (
+	"context"
+	"errors"
+
+	"git.anthrove.art/Anthrove/otter-space-sdk/v4/internal/utils"
+	otterError "git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/error"
+	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/attribute"
+	"gorm.io/gorm"
+)
+
+func CreateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory) (models.ScrapeHistory, error) {
+	ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "CreateScrapeHistory")
+	defer span.End()
+
+	localLogger = localLogger.WithFields(log.Fields{
+		"scrape_task_id": scrapeHistory.ScrapeTaskID,
+		"user_source_id": scrapeHistory.UserSourceID,
+	})
+
+	span.SetAttributes(
+		attribute.String("scrape_task_id", string(scrapeHistory.ScrapeTaskID)),
+		attribute.String("user_source_id", string(scrapeHistory.UserSourceID)),
+	)
+
+	utils.HandleEvent(span, localLogger, "Starting scrapeHistory creation")
+
+	if client == nil {
+		return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected})
+	}
+
+	if len(scrapeHistory.ScrapeTaskID) == 0 {
+		return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty})
+	}
+
+	if len(scrapeHistory.ScrapeTaskID) != 25 {
+		return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength})
+	}
+
+	if len(scrapeHistory.UserSourceID) == 0 {
+		return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.UserSourceIDIsEmpty})
+	}
+
+	if len(scrapeHistory.UserSourceID) != 25 {
+		return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.UserSourceIsWrongLength})
+	}
+
+	result := client.WithContext(ctx).Create(&scrapeHistory)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DuplicateKey})
+		}
+		return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, result.Error)
+	}
+
+	utils.HandleEvent(span, localLogger, "scrapeHistory created successfully")
+	return scrapeHistory, nil
+}
+
+func GetScrapeHistoryByID(ctx context.Context, id models.ScrapeTaskID) (models.ScrapeHistory, error) {
+	ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "GetScrapeHistoryByID")
+	defer span.End()
+
+	localLogger = localLogger.WithFields(log.Fields{
+		"scrape_task_id": id,
+	})
+
+	span.SetAttributes(
+		attribute.String("scrape_task_id", string(id)),
+	)
+
+	utils.HandleEvent(span, localLogger, "Starting get scrapeHistory by ID")
+
+	var post models.ScrapeHistory
+
+	if len(id) == 0 {
+		return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty})
+	}
+
+	if len(id) != 25 {
+		return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength})
+	}
+
+	result := client.WithContext(ctx).First(&post, "scrape_task_id = ?", id)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound})
+		}
+		return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, result.Error)
+	}
+
+	utils.HandleEvent(span, localLogger, "scrapeHistory retrieved successfully")
+	return post, nil
+}
+
+// UpdateScrapeHistory updates the scrape history information in the database.
+// Only a few parameter can be updated:
+//   - FinishedAt
+//   - Error
+//   - AddedPosts
+//   - DeletedPosts
+func UpdateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory) error {
+	ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "UpdateScrapeHistory")
+	defer span.End()
+
+	localLogger = localLogger.WithFields(log.Fields{
+		"scrape_task_id": scrapeHistory.ScrapeTaskID,
+	})
+
+	span.SetAttributes(
+		attribute.String("scrape_task_id", string(scrapeHistory.ScrapeTaskID)),
+	)
+
+	utils.HandleEvent(span, localLogger, "Starting scrapeHistory update")
+
+	if len(scrapeHistory.ScrapeTaskID) == 0 {
+		return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty})
+	}
+
+	if len(scrapeHistory.ScrapeTaskID) != 25 {
+		return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength})
+	}
+
+	updateScrapeHistory := models.ScrapeHistory{
+		ScrapeTaskID: scrapeHistory.ScrapeTaskID,
+		UserSourceID: scrapeHistory.UserSourceID,
+		FinishedAt:   scrapeHistory.FinishedAt,
+		Error:        scrapeHistory.Error,
+		AddedPosts:   scrapeHistory.AddedPosts,
+		DeletedPosts: scrapeHistory.DeletedPosts,
+	}
+
+	result := client.WithContext(ctx).Model(&updateScrapeHistory).Updates(updateScrapeHistory)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound})
+		}
+		return utils.HandleError(ctx, span, localLogger, result.Error)
+	}
+
+	utils.HandleEvent(span, localLogger, "scrapeHistory updated successfully")
+	return nil
+}
+
+func DeleteScrapeHistory(ctx context.Context, id models.ScrapeTaskID) error {
+	ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "DeleteScrapeHistory")
+	defer span.End()
+
+	localLogger = localLogger.WithFields(log.Fields{
+		"scrape_task_id": id,
+	})
+
+	span.SetAttributes(
+		attribute.String("scrape_task_id", string(id)),
+	)
+
+	utils.HandleEvent(span, localLogger, "Starting delete scrapeHistory")
+
+	var scrapeHistory models.ScrapeHistory
+
+	if len(id) == 0 {
+		return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty})
+	}
+
+	if len(id) != 25 {
+		return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength})
+	}
+
+	result := client.WithContext(ctx).Delete(&scrapeHistory, "scrape_task_id = ?", id)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound})
+		}
+		return utils.HandleError(ctx, span, localLogger, result.Error)
+	}
+
+	utils.HandleEvent(span, localLogger, "scrapeHistory deleted successfully")
+	return nil
+}

pkg/database/scrape_history_test.go

@@ -0,0 +1,411 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
+	"git.anthrove.art/Anthrove/otter-space-sdk/v4/test"
+	"go.opentelemetry.io/contrib/bridges/otellogrus"
+	"go.opentelemetry.io/otel"
+	"gorm.io/gorm"
+)
+
+func TestCreateScrapeHistory(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		logger.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+
+	client = gormDB
+
+	// Setup open telemetry
+	tracer = otel.Tracer(tracingName)
+
+	hook := otellogrus.NewHook(tracingName)
+	logger.AddHook(hook)
+
+	defer container.Terminate(ctx)
+
+	// -- -- Setup Tests
+
+	// -- Create User ot test with
+	validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}}
+
+	validUser, err = CreateUser(ctx, validUser)
+	if err != nil {
+		t.Fatalf("CreateUser err: %v", err)
+	}
+	// --
+
+	// -- Create Source to test with
+	validSource := models.Source{
+		DisplayName: "e621",
+		Domain:      "e621.net",
+		Icon:        "e621.net/icon.png",
+	}
+
+	validSource, err = CreateSource(ctx, validSource)
+	if err != nil {
+		t.Fatalf("CreateSource err: %v", err)
+	}
+	// --
+
+	//  -- Create UserSource model
+	validUSerSource := models.UserSource{
+		BaseModel: models.BaseModel[models.UserSourceID]{
+			ID:        models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")),
+			CreatedAt: time.Now(),
+			UpdatedAt: time.Now(),
+			DeletedAt: gorm.DeletedAt{},
+		},
+		User:                 models.User{},
+		UserID:               validUser.ID,
+		Source:               models.Source{},
+		SourceID:             validSource.ID,
+		ScrapeTimeInterval:   "P1D",
+		AccountUsername:      "marry",
+		AccountID:            "poppens",
+		LastScrapeTime:       time.Now(),
+		AccountValidate:      false,
+		AccountValidationKey: "im-a-key",
+	}
+
+	validUSerSource, err = CreateUserSource(ctx, validUSerSource)
+	if err != nil {
+		t.Fatalf("CreateUserSource err: %v", err)
+	}
+
+	validScrapeHistory := models.ScrapeHistory{
+		ScrapeTaskID: "000000000000valid_task_id",
+		UserSourceID: validUSerSource.ID,
+	}
+
+	//  --
+	tests := []struct {
+		name    string
+		args    models.ScrapeHistory
+		wantErr bool
+	}{
+		{
+			name:    "Valid ScrapeHistory",
+			args:    validScrapeHistory,
+			wantErr: false,
+		},
+		{
+			name:    "Duplicate ScrapeHistory",
+			args:    validScrapeHistory,
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := CreateScrapeHistory(ctx, tt.args)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("CreateScrapeHistory() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestGetScrapeHistoryByID(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		logger.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+
+	client = gormDB
+
+	// Setup open telemetry
+	tracer = otel.Tracer(tracingName)
+
+	hook := otellogrus.NewHook(tracingName)
+	logger.AddHook(hook)
+
+	defer container.Terminate(ctx)
+
+	// -- -- Setup Tests
+
+	// -- Create User ot test with
+	validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}}
+
+	validUser, err = CreateUser(ctx, validUser)
+	if err != nil {
+		t.Fatalf("CreateUser err: %v", err)
+	}
+	// --
+
+	// -- Create Source to test with
+	validSource := models.Source{
+		DisplayName: "e621",
+		Domain:      "e621.net",
+		Icon:        "e621.net/icon.png",
+	}
+
+	validSource, err = CreateSource(ctx, validSource)
+	if err != nil {
+		t.Fatalf("CreateSource err: %v", err)
+	}
+	// --
+
+	//  -- Create UserSource model
+	validUSerSource := models.UserSource{
+		BaseModel: models.BaseModel[models.UserSourceID]{
+			ID:        models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")),
+			CreatedAt: time.Now(),
+			UpdatedAt: time.Now(),
+			DeletedAt: gorm.DeletedAt{},
+		},
+		User:                 models.User{},
+		UserID:               validUser.ID,
+		Source:               models.Source{},
+		SourceID:             validSource.ID,
+		ScrapeTimeInterval:   "P1D",
+		AccountUsername:      "marry",
+		AccountID:            "poppens",
+		LastScrapeTime:       time.Now(),
+		AccountValidate:      false,
+		AccountValidationKey: "im-a-key",
+	}
+
+	validUSerSource, err = CreateUserSource(ctx, validUSerSource)
+	if err != nil {
+		t.Fatalf("CreateUserSource err: %v", err)
+	}
+
+	validScrapeHistory := models.ScrapeHistory{
+		ScrapeTaskID: "000000000000valid_task_id",
+		UserSourceID: validUSerSource.ID,
+	}
+
+	validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory)
+	if err != nil {
+		t.Fatalf("CreateScrapeHistory err: %v", err)
+	}
+
+	//  --
+	tests := []struct {
+		name    string
+		id      models.ScrapeTaskID
+		wantErr bool
+	}{
+		{"Valid ID", "000000000000valid_task_id", false},
+		{"Empty ID", "", true},
+		{"Non-existent ID", "non_existent_id", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := GetScrapeHistoryByID(ctx, tt.id)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetScrapeHistoryByID() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestUpdateScrapeHistory(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		logger.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+
+	client = gormDB
+
+	// Setup open telemetry
+	tracer = otel.Tracer(tracingName)
+
+	hook := otellogrus.NewHook(tracingName)
+	logger.AddHook(hook)
+
+	defer container.Terminate(ctx)
+
+	// -- -- Setup Tests
+
+	// -- Create User ot test with
+	validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}}
+
+	validUser, err = CreateUser(ctx, validUser)
+	if err != nil {
+		t.Fatalf("CreateUser err: %v", err)
+	}
+	// --
+
+	// -- Create Source to test with
+	validSource := models.Source{
+		DisplayName: "e621",
+		Domain:      "e621.net",
+		Icon:        "e621.net/icon.png",
+	}
+
+	validSource, err = CreateSource(ctx, validSource)
+	if err != nil {
+		t.Fatalf("CreateSource err: %v", err)
+	}
+	// --
+
+	//  -- Create UserSource model
+	validUSerSource := models.UserSource{
+		BaseModel: models.BaseModel[models.UserSourceID]{
+			ID:        models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")),
+			CreatedAt: time.Now(),
+			UpdatedAt: time.Now(),
+			DeletedAt: gorm.DeletedAt{},
+		},
+		User:                 models.User{},
+		UserID:               validUser.ID,
+		Source:               models.Source{},
+		SourceID:             validSource.ID,
+		ScrapeTimeInterval:   "P1D",
+		AccountUsername:      "marry",
+		AccountID:            "poppens",
+		LastScrapeTime:       time.Now(),
+		AccountValidate:      false,
+		AccountValidationKey: "im-a-key",
+	}
+
+	validUSerSource, err = CreateUserSource(ctx, validUSerSource)
+	if err != nil {
+		t.Fatalf("CreateUserSource err: %v", err)
+	}
+
+	validScrapeHistory := models.ScrapeHistory{
+		ScrapeTaskID: "000000000000valid_task_id",
+		UserSourceID: validUSerSource.ID,
+	}
+
+	validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory)
+	if err != nil {
+		t.Fatalf("CreateScrapeHistory err: %v", err)
+	}
+
+	updatedScrapeHistory := validScrapeHistory
+	updatedScrapeHistory.FinishedAt = time.Now()
+
+	//  --
+
+	tests := []struct {
+		name    string
+		args    models.ScrapeHistory
+		wantErr bool
+	}{
+		{"Valid Update", updatedScrapeHistory, false},
+		{"Empty ID", models.ScrapeHistory{ScrapeTaskID: ""}, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := UpdateScrapeHistory(ctx, tt.args)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("UpdateScrapeHistory() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestDeleteScrapeHistory(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		logger.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+
+	client = gormDB
+
+	// Setup open telemetry
+	tracer = otel.Tracer(tracingName)
+
+	hook := otellogrus.NewHook(tracingName)
+	logger.AddHook(hook)
+
+	defer container.Terminate(ctx)
+
+	// -- -- Setup Tests
+
+	// -- Create User ot test with
+	validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}}
+
+	validUser, err = CreateUser(ctx, validUser)
+	if err != nil {
+		t.Fatalf("CreateUser err: %v", err)
+	}
+	// --
+
+	// -- Create Source to test with
+	validSource := models.Source{
+		DisplayName: "e621",
+		Domain:      "e621.net",
+		Icon:        "e621.net/icon.png",
+	}
+
+	validSource, err = CreateSource(ctx, validSource)
+	if err != nil {
+		t.Fatalf("CreateSource err: %v", err)
+	}
+	// --
+
+	//  -- Create UserSource model
+	validUSerSource := models.UserSource{
+		BaseModel: models.BaseModel[models.UserSourceID]{
+			ID:        models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")),
+			CreatedAt: time.Now(),
+			UpdatedAt: time.Now(),
+			DeletedAt: gorm.DeletedAt{},
+		},
+		User:                 models.User{},
+		UserID:               validUser.ID,
+		Source:               models.Source{},
+		SourceID:             validSource.ID,
+		ScrapeTimeInterval:   "P1D",
+		AccountUsername:      "marry",
+		AccountID:            "poppens",
+		LastScrapeTime:       time.Now(),
+		AccountValidate:      false,
+		AccountValidationKey: "im-a-key",
+	}
+
+	validUSerSource, err = CreateUserSource(ctx, validUSerSource)
+	if err != nil {
+		t.Fatalf("CreateUserSource err: %v", err)
+	}
+
+	validScrapeHistory := models.ScrapeHistory{
+		ScrapeTaskID: "000000000000valid_task_id",
+		UserSourceID: validUSerSource.ID,
+	}
+
+	validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory)
+	if err != nil {
+		t.Fatalf("CreateScrapeHistory err: %v", err)
+	}
+
+	//  --
+	tests := []struct {
+		name    string
+		id      models.ScrapeTaskID
+		wantErr bool
+	}{
+		{"Valid ID", "000000000000valid_task_id", false},
+		{"Empty ID", "", true},
+		{"Short ID", "short_id", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := DeleteScrapeHistory(ctx, tt.id)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("DeleteScrapeHistory() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

pkg/error/validation.go

@@ -35,6 +35,9 @@ const (
 	PoolIDIsWrongLength = "PoolID has the wrong length"
 
 	PoolURLIsEmpty = "PoolURL cannot be empty"
+
+	ScrapeTaskIDIsEmpty       = "ScrapeTaskID cannot be empty"
+	ScrapeTaskIDIsWrongLength = "ScrapeTaskID has the wrong length, needs to be 25 characters long"
 )
 
 type EntityValidationFailed struct {

pkg/models/const.go

@@ -23,6 +23,8 @@ type (
 
 	UserSourceID   string
 	UserFavoriteID string
+
+	ScrapeTaskID string
 )
 
 const (

pkg/models/scrape_history.go

@@ -0,0 +1,19 @@
+package models
+
+import (
+	"time"
+)
+
+type ScrapeHistory struct {
+	ScrapeTaskID ScrapeTaskID `json:"scrape_task_id" gorm:"primaryKey"`
+	UserSourceID UserSourceID `json:"user_source_id" gorm:""`
+	CreatedAt    time.Time    `json:"created_at" gorm:""`
+	FinishedAt   time.Time    `json:"finished_at" gorm:""`
+	Error        string       `json:"error" gorm:"null"`
+	AddedPosts   int          `json:"added_posts" gorm:"not null"`
+	DeletedPosts int          `json:"deleted_posts" gorm:"not null"`
+}
+
+func (ScrapeHistory) TableName() string {
+	return "ScrapeHistory"
+}

pkg/models/scrape_history_test.go

@@ -0,0 +1,11 @@
+package models
+
+import "testing"
+
+func TestScrapeHistory_TableName(t *testing.T) {
+	scrapeHistory := ScrapeHistory{}
+	expectedTableName := "ScrapeHistory"
+	if tableName := scrapeHistory.TableName(); tableName != expectedTableName {
+		t.Fatalf("expected %s, but got %s", expectedTableName, tableName)
+	}
+}