diff --git a/pkg/database/migrations/004_scrape_history.sql b/pkg/database/migrations/004_scrape_history.sql new file mode 100644 index 0000000..5ae010b --- /dev/null +++ b/pkg/database/migrations/004_scrape_history.sql @@ -0,0 +1,11 @@ +-- +migrate Up +CREATE TABLE "ScrapeHistory" +( + scrape_task_id CHAR(25) PRIMARY KEY, + user_source_id CHAR(25) NOT NULL REFERENCES "UserSource" (id), + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + finished_at TIMESTAMP NULL, + error TEXT NULL, + added_posts INT NOT NULL, + deleted_posts INT NOT NULL +); \ No newline at end of file diff --git a/pkg/database/scrape_history.go b/pkg/database/scrape_history.go new file mode 100644 index 0000000..60dcae1 --- /dev/null +++ b/pkg/database/scrape_history.go @@ -0,0 +1,182 @@ +package database + +import ( + "context" + "errors" + + "git.anthrove.art/Anthrove/otter-space-sdk/v4/internal/utils" + otterError "git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/error" + "git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models" + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" + "gorm.io/gorm" +) + +func CreateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory) (models.ScrapeHistory, error) { + ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "CreateScrapeHistory") + defer span.End() + + localLogger = localLogger.WithFields(log.Fields{ + "scrape_task_id": scrapeHistory.ScrapeTaskID, + "user_source_id": scrapeHistory.UserSourceID, + }) + + span.SetAttributes( + attribute.String("scrape_task_id", string(scrapeHistory.ScrapeTaskID)), + attribute.String("user_source_id", string(scrapeHistory.UserSourceID)), + ) + + utils.HandleEvent(span, localLogger, "Starting scrapeHistory creation") + + if client == nil { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected}) + } + + if len(scrapeHistory.ScrapeTaskID) == 0 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) + } + + if len(scrapeHistory.ScrapeTaskID) != 25 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength}) + } + + if len(scrapeHistory.UserSourceID) == 0 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.UserSourceIDIsEmpty}) + } + + if len(scrapeHistory.UserSourceID) != 25 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.UserSourceIsWrongLength}) + } + + result := client.WithContext(ctx).Create(&scrapeHistory) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrDuplicatedKey) { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DuplicateKey}) + } + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, result.Error) + } + + utils.HandleEvent(span, localLogger, "scrapeHistory created successfully") + return scrapeHistory, nil +} + +func GetScrapeHistoryByID(ctx context.Context, id models.ScrapeTaskID) (models.ScrapeHistory, error) { + ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "GetScrapeHistoryByID") + defer span.End() + + localLogger = localLogger.WithFields(log.Fields{ + "scrape_task_id": id, + }) + + span.SetAttributes( + attribute.String("scrape_task_id", string(id)), + ) + + utils.HandleEvent(span, localLogger, "Starting get scrapeHistory by ID") + + var post models.ScrapeHistory + + if len(id) == 0 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) + } + + if len(id) != 25 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength}) + } + + result := client.WithContext(ctx).First(&post, "scrape_task_id = ?", id) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound}) + } + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, result.Error) + } + + utils.HandleEvent(span, localLogger, "scrapeHistory retrieved successfully") + return post, nil +} + +// UpdateScrapeHistory updates the scrape history information in the database. +// Only a few parameter can be updated: +// - FinishedAt +// - Error +// - AddedPosts +// - DeletedPosts +func UpdateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory) error { + ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "UpdateScrapeHistory") + defer span.End() + + localLogger = localLogger.WithFields(log.Fields{ + "scrape_task_id": scrapeHistory.ScrapeTaskID, + }) + + span.SetAttributes( + attribute.String("scrape_task_id", string(scrapeHistory.ScrapeTaskID)), + ) + + utils.HandleEvent(span, localLogger, "Starting scrapeHistory update") + + if len(scrapeHistory.ScrapeTaskID) == 0 { + return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) + } + + if len(scrapeHistory.ScrapeTaskID) != 25 { + return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength}) + } + + updateScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: scrapeHistory.ScrapeTaskID, + UserSourceID: scrapeHistory.UserSourceID, + FinishedAt: scrapeHistory.FinishedAt, + Error: scrapeHistory.Error, + AddedPosts: scrapeHistory.AddedPosts, + DeletedPosts: scrapeHistory.DeletedPosts, + } + + result := client.WithContext(ctx).Model(&updateScrapeHistory).Updates(updateScrapeHistory) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound}) + } + return utils.HandleError(ctx, span, localLogger, result.Error) + } + + utils.HandleEvent(span, localLogger, "scrapeHistory updated successfully") + return nil +} + +func DeleteScrapeHistory(ctx context.Context, id models.ScrapeTaskID) error { + ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "DeleteScrapeHistory") + defer span.End() + + localLogger = localLogger.WithFields(log.Fields{ + "scrape_task_id": id, + }) + + span.SetAttributes( + attribute.String("scrape_task_id", string(id)), + ) + + utils.HandleEvent(span, localLogger, "Starting delete scrapeHistory") + + var scrapeHistory models.ScrapeHistory + + if len(id) == 0 { + return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) + } + + if len(id) != 25 { + return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength}) + } + + result := client.WithContext(ctx).Delete(&scrapeHistory, "scrape_task_id = ?", id) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound}) + } + return utils.HandleError(ctx, span, localLogger, result.Error) + } + + utils.HandleEvent(span, localLogger, "scrapeHistory deleted successfully") + return nil +} diff --git a/pkg/database/scrape_history_test.go b/pkg/database/scrape_history_test.go new file mode 100644 index 0000000..e96795e --- /dev/null +++ b/pkg/database/scrape_history_test.go @@ -0,0 +1,411 @@ +package database + +import ( + "context" + "fmt" + "testing" + "time" + + "git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models" + "git.anthrove.art/Anthrove/otter-space-sdk/v4/test" + "go.opentelemetry.io/contrib/bridges/otellogrus" + "go.opentelemetry.io/otel" + "gorm.io/gorm" +) + +func TestCreateScrapeHistory(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + logger.Fatalf("Could not start PostgreSQL container: %v", err) + } + + client = gormDB + + // Setup open telemetry + tracer = otel.Tracer(tracingName) + + hook := otellogrus.NewHook(tracingName) + logger.AddHook(hook) + + defer container.Terminate(ctx) + + // -- -- Setup Tests + + // -- Create User ot test with + validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}} + + validUser, err = CreateUser(ctx, validUser) + if err != nil { + t.Fatalf("CreateUser err: %v", err) + } + // -- + + // -- Create Source to test with + validSource := models.Source{ + DisplayName: "e621", + Domain: "e621.net", + Icon: "e621.net/icon.png", + } + + validSource, err = CreateSource(ctx, validSource) + if err != nil { + t.Fatalf("CreateSource err: %v", err) + } + // -- + + // -- Create UserSource model + validUSerSource := models.UserSource{ + BaseModel: models.BaseModel[models.UserSourceID]{ + ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DeletedAt: gorm.DeletedAt{}, + }, + User: models.User{}, + UserID: validUser.ID, + Source: models.Source{}, + SourceID: validSource.ID, + ScrapeTimeInterval: "P1D", + AccountUsername: "marry", + AccountID: "poppens", + LastScrapeTime: time.Now(), + AccountValidate: false, + AccountValidationKey: "im-a-key", + } + + validUSerSource, err = CreateUserSource(ctx, validUSerSource) + if err != nil { + t.Fatalf("CreateUserSource err: %v", err) + } + + validScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: "000000000000valid_task_id", + UserSourceID: validUSerSource.ID, + } + + // -- + tests := []struct { + name string + args models.ScrapeHistory + wantErr bool + }{ + { + name: "Valid ScrapeHistory", + args: validScrapeHistory, + wantErr: false, + }, + { + name: "Duplicate ScrapeHistory", + args: validScrapeHistory, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := CreateScrapeHistory(ctx, tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("CreateScrapeHistory() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGetScrapeHistoryByID(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + logger.Fatalf("Could not start PostgreSQL container: %v", err) + } + + client = gormDB + + // Setup open telemetry + tracer = otel.Tracer(tracingName) + + hook := otellogrus.NewHook(tracingName) + logger.AddHook(hook) + + defer container.Terminate(ctx) + + // -- -- Setup Tests + + // -- Create User ot test with + validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}} + + validUser, err = CreateUser(ctx, validUser) + if err != nil { + t.Fatalf("CreateUser err: %v", err) + } + // -- + + // -- Create Source to test with + validSource := models.Source{ + DisplayName: "e621", + Domain: "e621.net", + Icon: "e621.net/icon.png", + } + + validSource, err = CreateSource(ctx, validSource) + if err != nil { + t.Fatalf("CreateSource err: %v", err) + } + // -- + + // -- Create UserSource model + validUSerSource := models.UserSource{ + BaseModel: models.BaseModel[models.UserSourceID]{ + ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DeletedAt: gorm.DeletedAt{}, + }, + User: models.User{}, + UserID: validUser.ID, + Source: models.Source{}, + SourceID: validSource.ID, + ScrapeTimeInterval: "P1D", + AccountUsername: "marry", + AccountID: "poppens", + LastScrapeTime: time.Now(), + AccountValidate: false, + AccountValidationKey: "im-a-key", + } + + validUSerSource, err = CreateUserSource(ctx, validUSerSource) + if err != nil { + t.Fatalf("CreateUserSource err: %v", err) + } + + validScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: "000000000000valid_task_id", + UserSourceID: validUSerSource.ID, + } + + validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory) + if err != nil { + t.Fatalf("CreateScrapeHistory err: %v", err) + } + + // -- + tests := []struct { + name string + id models.ScrapeTaskID + wantErr bool + }{ + {"Valid ID", "000000000000valid_task_id", false}, + {"Empty ID", "", true}, + {"Non-existent ID", "non_existent_id", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := GetScrapeHistoryByID(ctx, tt.id) + if (err != nil) != tt.wantErr { + t.Errorf("GetScrapeHistoryByID() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestUpdateScrapeHistory(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + logger.Fatalf("Could not start PostgreSQL container: %v", err) + } + + client = gormDB + + // Setup open telemetry + tracer = otel.Tracer(tracingName) + + hook := otellogrus.NewHook(tracingName) + logger.AddHook(hook) + + defer container.Terminate(ctx) + + // -- -- Setup Tests + + // -- Create User ot test with + validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}} + + validUser, err = CreateUser(ctx, validUser) + if err != nil { + t.Fatalf("CreateUser err: %v", err) + } + // -- + + // -- Create Source to test with + validSource := models.Source{ + DisplayName: "e621", + Domain: "e621.net", + Icon: "e621.net/icon.png", + } + + validSource, err = CreateSource(ctx, validSource) + if err != nil { + t.Fatalf("CreateSource err: %v", err) + } + // -- + + // -- Create UserSource model + validUSerSource := models.UserSource{ + BaseModel: models.BaseModel[models.UserSourceID]{ + ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DeletedAt: gorm.DeletedAt{}, + }, + User: models.User{}, + UserID: validUser.ID, + Source: models.Source{}, + SourceID: validSource.ID, + ScrapeTimeInterval: "P1D", + AccountUsername: "marry", + AccountID: "poppens", + LastScrapeTime: time.Now(), + AccountValidate: false, + AccountValidationKey: "im-a-key", + } + + validUSerSource, err = CreateUserSource(ctx, validUSerSource) + if err != nil { + t.Fatalf("CreateUserSource err: %v", err) + } + + validScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: "000000000000valid_task_id", + UserSourceID: validUSerSource.ID, + } + + validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory) + if err != nil { + t.Fatalf("CreateScrapeHistory err: %v", err) + } + + updatedScrapeHistory := validScrapeHistory + updatedScrapeHistory.FinishedAt = time.Now() + + // -- + + tests := []struct { + name string + args models.ScrapeHistory + wantErr bool + }{ + {"Valid Update", updatedScrapeHistory, false}, + {"Empty ID", models.ScrapeHistory{ScrapeTaskID: ""}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := UpdateScrapeHistory(ctx, tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("UpdateScrapeHistory() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestDeleteScrapeHistory(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + logger.Fatalf("Could not start PostgreSQL container: %v", err) + } + + client = gormDB + + // Setup open telemetry + tracer = otel.Tracer(tracingName) + + hook := otellogrus.NewHook(tracingName) + logger.AddHook(hook) + + defer container.Terminate(ctx) + + // -- -- Setup Tests + + // -- Create User ot test with + validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}} + + validUser, err = CreateUser(ctx, validUser) + if err != nil { + t.Fatalf("CreateUser err: %v", err) + } + // -- + + // -- Create Source to test with + validSource := models.Source{ + DisplayName: "e621", + Domain: "e621.net", + Icon: "e621.net/icon.png", + } + + validSource, err = CreateSource(ctx, validSource) + if err != nil { + t.Fatalf("CreateSource err: %v", err) + } + // -- + + // -- Create UserSource model + validUSerSource := models.UserSource{ + BaseModel: models.BaseModel[models.UserSourceID]{ + ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DeletedAt: gorm.DeletedAt{}, + }, + User: models.User{}, + UserID: validUser.ID, + Source: models.Source{}, + SourceID: validSource.ID, + ScrapeTimeInterval: "P1D", + AccountUsername: "marry", + AccountID: "poppens", + LastScrapeTime: time.Now(), + AccountValidate: false, + AccountValidationKey: "im-a-key", + } + + validUSerSource, err = CreateUserSource(ctx, validUSerSource) + if err != nil { + t.Fatalf("CreateUserSource err: %v", err) + } + + validScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: "000000000000valid_task_id", + UserSourceID: validUSerSource.ID, + } + + validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory) + if err != nil { + t.Fatalf("CreateScrapeHistory err: %v", err) + } + + // -- + tests := []struct { + name string + id models.ScrapeTaskID + wantErr bool + }{ + {"Valid ID", "000000000000valid_task_id", false}, + {"Empty ID", "", true}, + {"Short ID", "short_id", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := DeleteScrapeHistory(ctx, tt.id) + if (err != nil) != tt.wantErr { + t.Errorf("DeleteScrapeHistory() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/error/validation.go b/pkg/error/validation.go index f91a447..ab53258 100644 --- a/pkg/error/validation.go +++ b/pkg/error/validation.go @@ -35,6 +35,9 @@ const ( PoolIDIsWrongLength = "PoolID has the wrong length" PoolURLIsEmpty = "PoolURL cannot be empty" + + ScrapeTaskIDIsEmpty = "ScrapeTaskID cannot be empty" + ScrapeTaskIDIsWrongLength = "ScrapeTaskID has the wrong length, needs to be 25 characters long" ) type EntityValidationFailed struct { diff --git a/pkg/models/const.go b/pkg/models/const.go index 9755bfd..fb7a04f 100644 --- a/pkg/models/const.go +++ b/pkg/models/const.go @@ -23,6 +23,8 @@ type ( UserSourceID string UserFavoriteID string + + ScrapeTaskID string ) const ( diff --git a/pkg/models/scrape_history.go b/pkg/models/scrape_history.go new file mode 100644 index 0000000..f5c6ccd --- /dev/null +++ b/pkg/models/scrape_history.go @@ -0,0 +1,19 @@ +package models + +import ( + "time" +) + +type ScrapeHistory struct { + ScrapeTaskID ScrapeTaskID `json:"scrape_task_id" gorm:"primaryKey"` + UserSourceID UserSourceID `json:"user_source_id" gorm:""` + CreatedAt time.Time `json:"created_at" gorm:""` + FinishedAt time.Time `json:"finished_at" gorm:""` + Error string `json:"error" gorm:"null"` + AddedPosts int `json:"added_posts" gorm:"not null"` + DeletedPosts int `json:"deleted_posts" gorm:"not null"` +} + +func (ScrapeHistory) TableName() string { + return "ScrapeHistory" +} diff --git a/pkg/models/scrape_history_test.go b/pkg/models/scrape_history_test.go new file mode 100644 index 0000000..d479a26 --- /dev/null +++ b/pkg/models/scrape_history_test.go @@ -0,0 +1,11 @@ +package models + +import "testing" + +func TestScrapeHistory_TableName(t *testing.T) { + scrapeHistory := ScrapeHistory{} + expectedTableName := "ScrapeHistory" + if tableName := scrapeHistory.TableName(); tableName != expectedTableName { + t.Fatalf("expected %s, but got %s", expectedTableName, tableName) + } +}