From b1e97f414251785ed31ccc87544283ba9dc66ff1 Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 15 Oct 2024 11:15:49 +0200 Subject: [PATCH 1/8] feat(database): scrape history - added migration --- pkg/database/migrations/004_scrape_history.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 pkg/database/migrations/004_scrape_history.sql diff --git a/pkg/database/migrations/004_scrape_history.sql b/pkg/database/migrations/004_scrape_history.sql new file mode 100644 index 0000000..5d23e44 --- /dev/null +++ b/pkg/database/migrations/004_scrape_history.sql @@ -0,0 +1,11 @@ +-- +migrate Up +CREATE TABLE "ScrapeHistory" +( + user_source_id CHAR(25) NOT NULL REFERENCES "UserSource" (id), + scrape_task_id CHAR(25) PRIMARY KEY, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + finished_at TIMESTAMP NULL, + error TEXT NULL, + added_posts INT NOT NULL, + deleted_posts INT NOT NULL +); \ No newline at end of file -- 2.47.1 From 4312cbdff5cee8c73e24ad9dc2b2166699101c78 Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 15 Oct 2024 11:23:33 +0200 Subject: [PATCH 2/8] feat(database): added model - added model - added test for model --- pkg/models/scrape_history.go | 19 +++++++++++++++++++ pkg/models/scrape_history_test.go | 11 +++++++++++ 2 files changed, 30 insertions(+) create mode 100644 pkg/models/scrape_history.go create mode 100644 pkg/models/scrape_history_test.go diff --git a/pkg/models/scrape_history.go b/pkg/models/scrape_history.go new file mode 100644 index 0000000..e06f6c5 --- /dev/null +++ b/pkg/models/scrape_history.go @@ -0,0 +1,19 @@ +package models + +import ( + "time" +) + +type ScrapeHistory struct { + ScrapeTaskID string `json:"scrape_task_id" gorm:"primaryKey"` + UserSourceID string `json:"user_source_id" gorm:""` + CreatedAt time.Time `json:"created_at" gorm:""` + FinishedAt time.Time `json:"finished_at" gorm:"null"` + Error string `json:"error" gorm:"null"` + AddedPosts int `json:"added_posts" gorm:"not null"` + DeletedPosts int `json:"deleted_posts" gorm:"not null"` +} + +func (ScrapeHistory) TableName() string { + return "ScrapeHistory" +} diff --git a/pkg/models/scrape_history_test.go b/pkg/models/scrape_history_test.go new file mode 100644 index 0000000..d479a26 --- /dev/null +++ b/pkg/models/scrape_history_test.go @@ -0,0 +1,11 @@ +package models + +import "testing" + +func TestScrapeHistory_TableName(t *testing.T) { + scrapeHistory := ScrapeHistory{} + expectedTableName := "ScrapeHistory" + if tableName := scrapeHistory.TableName(); tableName != expectedTableName { + t.Fatalf("expected %s, but got %s", expectedTableName, tableName) + } +} -- 2.47.1 From 27bc719a02875c45c31b845d5f0eb71abd2faf21 Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 15 Oct 2024 12:25:32 +0200 Subject: [PATCH 3/8] feat(database): added CRUD - added Create - added Get - added Update - added Delete --- pkg/database/scrape_history.go | 166 +++++++++++++++++++++++++++++++++ pkg/error/validation.go | 3 + pkg/models/const.go | 2 + pkg/models/scrape_history.go | 14 +-- 4 files changed, 178 insertions(+), 7 deletions(-) create mode 100644 pkg/database/scrape_history.go diff --git a/pkg/database/scrape_history.go b/pkg/database/scrape_history.go new file mode 100644 index 0000000..8cf424f --- /dev/null +++ b/pkg/database/scrape_history.go @@ -0,0 +1,166 @@ +package database + +import ( + "context" + "errors" + + "git.anthrove.art/Anthrove/otter-space-sdk/v4/internal/utils" + otterError "git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/error" + "git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models" + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" + "gorm.io/gorm" +) + +func CreateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory) (models.ScrapeHistory, error) { + ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "CreateScrapeHistory") + defer span.End() + + localLogger = localLogger.WithFields(log.Fields{ + "scrape_task_id": scrapeHistory.ScrapeTaskID, + "user_source_id": scrapeHistory.UserSourceID, + }) + + span.SetAttributes( + attribute.String("scrape_task_id", string(scrapeHistory.ScrapeTaskID)), + attribute.String("user_source_id", string(scrapeHistory.UserSourceID)), + ) + + utils.HandleEvent(span, localLogger, "Starting scrapeHistory creation") + + if client == nil { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected}) + } + + result := client.WithContext(ctx).Create(&scrapeHistory) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrDuplicatedKey) { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DuplicateKey}) + } + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, result.Error) + } + + utils.HandleEvent(span, localLogger, "scrapeHistory created successfully") + return scrapeHistory, nil +} + +func GetScrapeHistoryByID(ctx context.Context, id models.ScrapeTaskID) (models.ScrapeHistory, error) { + ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "GetScrapeHistoryByID") + defer span.End() + + localLogger = localLogger.WithFields(log.Fields{ + "scrape_task_id": id, + }) + + span.SetAttributes( + attribute.String("scrape_task_id", string(id)), + ) + + utils.HandleEvent(span, localLogger, "Starting get scrapeHistory by ID") + + var post models.ScrapeHistory + + if client == nil { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected}) + } + + if len(id) == 0 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) + } + + result := client.WithContext(ctx).First(&post, "scrape_task_id = ?", id) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound}) + } + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, result.Error) + } + + utils.HandleEvent(span, localLogger, "scrapeHistory retrieved successfully") + return post, nil +} + +// UpdateScrapeHistory updates the scrape history information in the database. +// Only a few parameter can be updated: +// - FinishedAt +// - Error +// - AddedPosts +// - DeletedPosts +func UpdateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory) error { + ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "UpdateScrapeHistory") + defer span.End() + + localLogger = localLogger.WithFields(log.Fields{ + "scrape_task_id": scrapeHistory.ScrapeTaskID, + }) + + span.SetAttributes( + attribute.String("scrape_task_id", string(scrapeHistory.ScrapeTaskID)), + ) + + utils.HandleEvent(span, localLogger, "Starting scrapeHistory update") + + if client == nil { + return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected}) + } + + if len(scrapeHistory.ScrapeTaskID) == 0 { + return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) + } + + updateScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: scrapeHistory.ScrapeTaskID, + UserSourceID: scrapeHistory.UserSourceID, + FinishedAt: scrapeHistory.FinishedAt, + Error: scrapeHistory.Error, + AddedPosts: scrapeHistory.AddedPosts, + DeletedPosts: scrapeHistory.DeletedPosts, + } + + result := client.WithContext(ctx).Model(&updateScrapeHistory).Updates(updateScrapeHistory) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound}) + } + return utils.HandleError(ctx, span, localLogger, result.Error) + } + + utils.HandleEvent(span, localLogger, "scrapeHistory updated successfully") + return nil +} + +func DeleteScrapeHistory(ctx context.Context, id models.ScrapeTaskID) error { + ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "DeleteScrapeHistory") + defer span.End() + + localLogger = localLogger.WithFields(log.Fields{ + "scrape_task_id": id, + }) + + span.SetAttributes( + attribute.String("scrape_task_id", string(id)), + ) + + utils.HandleEvent(span, localLogger, "Starting delete scrapeHistory") + + var scrapeHistory models.ScrapeHistory + + if client == nil { + return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected}) + } + + if len(id) == 0 { + return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) + } + + result := client.WithContext(ctx).Delete(&scrapeHistory, "scrape_task_id = ?", id) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound}) + } + return utils.HandleError(ctx, span, localLogger, result.Error) + } + + utils.HandleEvent(span, localLogger, "scrapeHistory deleted successfully") + return nil +} diff --git a/pkg/error/validation.go b/pkg/error/validation.go index f91a447..2fe7289 100644 --- a/pkg/error/validation.go +++ b/pkg/error/validation.go @@ -35,6 +35,9 @@ const ( PoolIDIsWrongLength = "PoolID has the wrong length" PoolURLIsEmpty = "PoolURL cannot be empty" + + ScrapeTaskIDIsEmpty = "ScrapeTaskID cannot be empty" + ScrapeTaskIDIsWrongLength = "ScrapeTaskID has the wrong length" ) type EntityValidationFailed struct { diff --git a/pkg/models/const.go b/pkg/models/const.go index 9755bfd..fb7a04f 100644 --- a/pkg/models/const.go +++ b/pkg/models/const.go @@ -23,6 +23,8 @@ type ( UserSourceID string UserFavoriteID string + + ScrapeTaskID string ) const ( diff --git a/pkg/models/scrape_history.go b/pkg/models/scrape_history.go index e06f6c5..f5c6ccd 100644 --- a/pkg/models/scrape_history.go +++ b/pkg/models/scrape_history.go @@ -5,13 +5,13 @@ import ( ) type ScrapeHistory struct { - ScrapeTaskID string `json:"scrape_task_id" gorm:"primaryKey"` - UserSourceID string `json:"user_source_id" gorm:""` - CreatedAt time.Time `json:"created_at" gorm:""` - FinishedAt time.Time `json:"finished_at" gorm:"null"` - Error string `json:"error" gorm:"null"` - AddedPosts int `json:"added_posts" gorm:"not null"` - DeletedPosts int `json:"deleted_posts" gorm:"not null"` + ScrapeTaskID ScrapeTaskID `json:"scrape_task_id" gorm:"primaryKey"` + UserSourceID UserSourceID `json:"user_source_id" gorm:""` + CreatedAt time.Time `json:"created_at" gorm:""` + FinishedAt time.Time `json:"finished_at" gorm:""` + Error string `json:"error" gorm:"null"` + AddedPosts int `json:"added_posts" gorm:"not null"` + DeletedPosts int `json:"deleted_posts" gorm:"not null"` } func (ScrapeHistory) TableName() string { -- 2.47.1 From a6d95067feb574ceea596d5ea8874b08d756d88f Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 15 Oct 2024 12:25:42 +0200 Subject: [PATCH 4/8] feat(database): added tests - added Create - added Get - added Update - added Delete --- pkg/database/scrape_history_test.go | 411 ++++++++++++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 pkg/database/scrape_history_test.go diff --git a/pkg/database/scrape_history_test.go b/pkg/database/scrape_history_test.go new file mode 100644 index 0000000..867a731 --- /dev/null +++ b/pkg/database/scrape_history_test.go @@ -0,0 +1,411 @@ +package database + +import ( + "context" + "fmt" + "testing" + "time" + + "git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models" + "git.anthrove.art/Anthrove/otter-space-sdk/v4/test" + "go.opentelemetry.io/contrib/bridges/otellogrus" + "go.opentelemetry.io/otel" + "gorm.io/gorm" +) + +func TestCreateScrapeHistory(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + logger.Fatalf("Could not start PostgreSQL container: %v", err) + } + + client = gormDB + + // Setup open telemetry + tracer = otel.Tracer(tracingName) + + hook := otellogrus.NewHook(tracingName) + logger.AddHook(hook) + + defer container.Terminate(ctx) + + // -- -- Setup Tests + + // -- Create User ot test with + validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}} + + validUser, err = CreateUser(ctx, validUser) + if err != nil { + t.Fatalf("CreateUser err: %v", err) + } + // -- + + // -- Create Source to test with + validSource := models.Source{ + DisplayName: "e621", + Domain: "e621.net", + Icon: "e621.net/icon.png", + } + + validSource, err = CreateSource(ctx, validSource) + if err != nil { + t.Fatalf("CreateSource err: %v", err) + } + // -- + + // -- Create UserSource model + validUSerSource := models.UserSource{ + BaseModel: models.BaseModel[models.UserSourceID]{ + ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DeletedAt: gorm.DeletedAt{}, + }, + User: models.User{}, + UserID: validUser.ID, + Source: models.Source{}, + SourceID: validSource.ID, + ScrapeTimeInterval: "P1D", + AccountUsername: "marry", + AccountID: "poppens", + LastScrapeTime: time.Now(), + AccountValidate: false, + AccountValidationKey: "im-a-key", + } + + validUSerSource, err = CreateUserSource(ctx, validUSerSource) + if err != nil { + t.Fatalf("CreateUserSource err: %v", err) + } + + validScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: "valid_task_id", + UserSourceID: validUSerSource.ID, + } + + // -- + tests := []struct { + name string + args models.ScrapeHistory + wantErr bool + }{ + { + name: "Valid ScrapeHistory", + args: validScrapeHistory, + wantErr: false, + }, + { + name: "Duplicate ScrapeHistory", + args: validScrapeHistory, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := CreateScrapeHistory(ctx, tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("CreateScrapeHistory() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGetScrapeHistoryByID(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + logger.Fatalf("Could not start PostgreSQL container: %v", err) + } + + client = gormDB + + // Setup open telemetry + tracer = otel.Tracer(tracingName) + + hook := otellogrus.NewHook(tracingName) + logger.AddHook(hook) + + defer container.Terminate(ctx) + + // -- -- Setup Tests + + // -- Create User ot test with + validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}} + + validUser, err = CreateUser(ctx, validUser) + if err != nil { + t.Fatalf("CreateUser err: %v", err) + } + // -- + + // -- Create Source to test with + validSource := models.Source{ + DisplayName: "e621", + Domain: "e621.net", + Icon: "e621.net/icon.png", + } + + validSource, err = CreateSource(ctx, validSource) + if err != nil { + t.Fatalf("CreateSource err: %v", err) + } + // -- + + // -- Create UserSource model + validUSerSource := models.UserSource{ + BaseModel: models.BaseModel[models.UserSourceID]{ + ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DeletedAt: gorm.DeletedAt{}, + }, + User: models.User{}, + UserID: validUser.ID, + Source: models.Source{}, + SourceID: validSource.ID, + ScrapeTimeInterval: "P1D", + AccountUsername: "marry", + AccountID: "poppens", + LastScrapeTime: time.Now(), + AccountValidate: false, + AccountValidationKey: "im-a-key", + } + + validUSerSource, err = CreateUserSource(ctx, validUSerSource) + if err != nil { + t.Fatalf("CreateUserSource err: %v", err) + } + + validScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: "valid_task_id", + UserSourceID: validUSerSource.ID, + } + + validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory) + if err != nil { + t.Fatalf("CreateScrapeHistory err: %v", err) + } + + // -- + tests := []struct { + name string + id models.ScrapeTaskID + wantErr bool + }{ + {"Valid ID", "valid_task_id", false}, + {"Empty ID", "", true}, + {"Non-existent ID", "non_existent_id", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := GetScrapeHistoryByID(ctx, tt.id) + if (err != nil) != tt.wantErr { + t.Errorf("GetScrapeHistoryByID() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestUpdateScrapeHistory(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + logger.Fatalf("Could not start PostgreSQL container: %v", err) + } + + client = gormDB + + // Setup open telemetry + tracer = otel.Tracer(tracingName) + + hook := otellogrus.NewHook(tracingName) + logger.AddHook(hook) + + defer container.Terminate(ctx) + + // -- -- Setup Tests + + // -- Create User ot test with + validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}} + + validUser, err = CreateUser(ctx, validUser) + if err != nil { + t.Fatalf("CreateUser err: %v", err) + } + // -- + + // -- Create Source to test with + validSource := models.Source{ + DisplayName: "e621", + Domain: "e621.net", + Icon: "e621.net/icon.png", + } + + validSource, err = CreateSource(ctx, validSource) + if err != nil { + t.Fatalf("CreateSource err: %v", err) + } + // -- + + // -- Create UserSource model + validUSerSource := models.UserSource{ + BaseModel: models.BaseModel[models.UserSourceID]{ + ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DeletedAt: gorm.DeletedAt{}, + }, + User: models.User{}, + UserID: validUser.ID, + Source: models.Source{}, + SourceID: validSource.ID, + ScrapeTimeInterval: "P1D", + AccountUsername: "marry", + AccountID: "poppens", + LastScrapeTime: time.Now(), + AccountValidate: false, + AccountValidationKey: "im-a-key", + } + + validUSerSource, err = CreateUserSource(ctx, validUSerSource) + if err != nil { + t.Fatalf("CreateUserSource err: %v", err) + } + + validScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: "valid_task_id", + UserSourceID: validUSerSource.ID, + } + + validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory) + if err != nil { + t.Fatalf("CreateScrapeHistory err: %v", err) + } + + updatedScrapeHistory := validScrapeHistory + updatedScrapeHistory.FinishedAt = time.Now() + + // -- + + tests := []struct { + name string + args models.ScrapeHistory + wantErr bool + }{ + {"Valid Update", updatedScrapeHistory, false}, + {"Empty ID", models.ScrapeHistory{ScrapeTaskID: ""}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := UpdateScrapeHistory(ctx, tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("UpdateScrapeHistory() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestDeleteScrapeHistory(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + logger.Fatalf("Could not start PostgreSQL container: %v", err) + } + + client = gormDB + + // Setup open telemetry + tracer = otel.Tracer(tracingName) + + hook := otellogrus.NewHook(tracingName) + logger.AddHook(hook) + + defer container.Terminate(ctx) + + // -- -- Setup Tests + + // -- Create User ot test with + validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}} + + validUser, err = CreateUser(ctx, validUser) + if err != nil { + t.Fatalf("CreateUser err: %v", err) + } + // -- + + // -- Create Source to test with + validSource := models.Source{ + DisplayName: "e621", + Domain: "e621.net", + Icon: "e621.net/icon.png", + } + + validSource, err = CreateSource(ctx, validSource) + if err != nil { + t.Fatalf("CreateSource err: %v", err) + } + // -- + + // -- Create UserSource model + validUSerSource := models.UserSource{ + BaseModel: models.BaseModel[models.UserSourceID]{ + ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + DeletedAt: gorm.DeletedAt{}, + }, + User: models.User{}, + UserID: validUser.ID, + Source: models.Source{}, + SourceID: validSource.ID, + ScrapeTimeInterval: "P1D", + AccountUsername: "marry", + AccountID: "poppens", + LastScrapeTime: time.Now(), + AccountValidate: false, + AccountValidationKey: "im-a-key", + } + + validUSerSource, err = CreateUserSource(ctx, validUSerSource) + if err != nil { + t.Fatalf("CreateUserSource err: %v", err) + } + + validScrapeHistory := models.ScrapeHistory{ + ScrapeTaskID: "valid_task_id", + UserSourceID: validUSerSource.ID, + } + + validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory) + if err != nil { + t.Fatalf("CreateScrapeHistory err: %v", err) + } + + // -- + tests := []struct { + name string + id models.ScrapeTaskID + wantErr bool + }{ + {"Valid ID", "valid_task_id", false}, + {"Empty ID", "", true}, + {"Short ID", "short_id", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := DeleteScrapeHistory(ctx, tt.id) + if (err != nil) != tt.wantErr { + t.Errorf("DeleteScrapeHistory() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} -- 2.47.1 From 59f404883e373be696f70bd3655fe14efbea2fa6 Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 15 Oct 2024 12:28:52 +0200 Subject: [PATCH 5/8] fix(database): migration - primary key is now first in position --- pkg/database/migrations/004_scrape_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/database/migrations/004_scrape_history.sql b/pkg/database/migrations/004_scrape_history.sql index 5d23e44..5ae010b 100644 --- a/pkg/database/migrations/004_scrape_history.sql +++ b/pkg/database/migrations/004_scrape_history.sql @@ -1,8 +1,8 @@ -- +migrate Up CREATE TABLE "ScrapeHistory" ( - user_source_id CHAR(25) NOT NULL REFERENCES "UserSource" (id), scrape_task_id CHAR(25) PRIMARY KEY, + user_source_id CHAR(25) NOT NULL REFERENCES "UserSource" (id), created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TIMESTAMP NULL, error TEXT NULL, -- 2.47.1 From f70879cf8dbd624e7ec75e5a64a84313ef1b7b32 Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 15 Oct 2024 12:36:56 +0200 Subject: [PATCH 6/8] feat(database): validation - added more validation for IDsi --- pkg/database/scrape_history.go | 32 ++++++++++++++++++++------------ pkg/error/validation.go | 2 +- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pkg/database/scrape_history.go b/pkg/database/scrape_history.go index 8cf424f..c57412f 100644 --- a/pkg/database/scrape_history.go +++ b/pkg/database/scrape_history.go @@ -32,6 +32,14 @@ func CreateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected}) } + if len(scrapeHistory.ScrapeTaskID) == 0 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) + } + + if len(scrapeHistory.ScrapeTaskID) != 25 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength}) + } + result := client.WithContext(ctx).Create(&scrapeHistory) if result.Error != nil { if errors.Is(result.Error, gorm.ErrDuplicatedKey) { @@ -60,14 +68,14 @@ func GetScrapeHistoryByID(ctx context.Context, id models.ScrapeTaskID) (models.S var post models.ScrapeHistory - if client == nil { - return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected}) - } - if len(id) == 0 { return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) } + if len(id) != 25 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength}) + } + result := client.WithContext(ctx).First(&post, "scrape_task_id = ?", id) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { @@ -100,14 +108,14 @@ func UpdateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory utils.HandleEvent(span, localLogger, "Starting scrapeHistory update") - if client == nil { - return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected}) - } - if len(scrapeHistory.ScrapeTaskID) == 0 { return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) } + if len(scrapeHistory.ScrapeTaskID) != 25 { + return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength}) + } + updateScrapeHistory := models.ScrapeHistory{ ScrapeTaskID: scrapeHistory.ScrapeTaskID, UserSourceID: scrapeHistory.UserSourceID, @@ -145,14 +153,14 @@ func DeleteScrapeHistory(ctx context.Context, id models.ScrapeTaskID) error { var scrapeHistory models.ScrapeHistory - if client == nil { - return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected}) - } - if len(id) == 0 { return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty}) } + if len(id) != 25 { + return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength}) + } + result := client.WithContext(ctx).Delete(&scrapeHistory, "scrape_task_id = ?", id) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { diff --git a/pkg/error/validation.go b/pkg/error/validation.go index 2fe7289..ab53258 100644 --- a/pkg/error/validation.go +++ b/pkg/error/validation.go @@ -37,7 +37,7 @@ const ( PoolURLIsEmpty = "PoolURL cannot be empty" ScrapeTaskIDIsEmpty = "ScrapeTaskID cannot be empty" - ScrapeTaskIDIsWrongLength = "ScrapeTaskID has the wrong length" + ScrapeTaskIDIsWrongLength = "ScrapeTaskID has the wrong length, needs to be 25 characters long" ) type EntityValidationFailed struct { -- 2.47.1 From 0f1e6c6aaaa2263db803a147c40a602896f9df5b Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 15 Oct 2024 12:41:01 +0200 Subject: [PATCH 7/8] feat(database): tests - now the test against 25 char id length --- pkg/database/scrape_history_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/database/scrape_history_test.go b/pkg/database/scrape_history_test.go index 867a731..e96795e 100644 --- a/pkg/database/scrape_history_test.go +++ b/pkg/database/scrape_history_test.go @@ -81,7 +81,7 @@ func TestCreateScrapeHistory(t *testing.T) { } validScrapeHistory := models.ScrapeHistory{ - ScrapeTaskID: "valid_task_id", + ScrapeTaskID: "000000000000valid_task_id", UserSourceID: validUSerSource.ID, } @@ -181,7 +181,7 @@ func TestGetScrapeHistoryByID(t *testing.T) { } validScrapeHistory := models.ScrapeHistory{ - ScrapeTaskID: "valid_task_id", + ScrapeTaskID: "000000000000valid_task_id", UserSourceID: validUSerSource.ID, } @@ -196,7 +196,7 @@ func TestGetScrapeHistoryByID(t *testing.T) { id models.ScrapeTaskID wantErr bool }{ - {"Valid ID", "valid_task_id", false}, + {"Valid ID", "000000000000valid_task_id", false}, {"Empty ID", "", true}, {"Non-existent ID", "non_existent_id", true}, } @@ -279,7 +279,7 @@ func TestUpdateScrapeHistory(t *testing.T) { } validScrapeHistory := models.ScrapeHistory{ - ScrapeTaskID: "valid_task_id", + ScrapeTaskID: "000000000000valid_task_id", UserSourceID: validUSerSource.ID, } @@ -380,7 +380,7 @@ func TestDeleteScrapeHistory(t *testing.T) { } validScrapeHistory := models.ScrapeHistory{ - ScrapeTaskID: "valid_task_id", + ScrapeTaskID: "000000000000valid_task_id", UserSourceID: validUSerSource.ID, } @@ -395,7 +395,7 @@ func TestDeleteScrapeHistory(t *testing.T) { id models.ScrapeTaskID wantErr bool }{ - {"Valid ID", "valid_task_id", false}, + {"Valid ID", "000000000000valid_task_id", false}, {"Empty ID", "", true}, {"Short ID", "short_id", true}, } -- 2.47.1 From ef31399ab61cd9fea36232e30c08ee0e659484b7 Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 15 Oct 2024 12:45:46 +0200 Subject: [PATCH 8/8] feat(database): validation - check for user source --- pkg/database/scrape_history.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/database/scrape_history.go b/pkg/database/scrape_history.go index c57412f..60dcae1 100644 --- a/pkg/database/scrape_history.go +++ b/pkg/database/scrape_history.go @@ -40,6 +40,14 @@ func CreateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength}) } + if len(scrapeHistory.UserSourceID) == 0 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.UserSourceIDIsEmpty}) + } + + if len(scrapeHistory.UserSourceID) != 25 { + return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.UserSourceIsWrongLength}) + } + result := client.WithContext(ctx).Create(&scrapeHistory) if result.Error != nil { if errors.Is(result.Error, gorm.ErrDuplicatedKey) { -- 2.47.1