Anthrove/otter-space-sdk
5
0
Fork 0
Code Issues Pull Requests Actions Projects Releases 10 Activity

Add Scrape History #15

Merged
SoXX merged 8 commits from dev/scrape-history into main 2024-10-15 10:46:41 +00:00
Conversation 0 Commits 8 Files Changed 7 +639
7 changed files with 639 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/database/migrations/004_scrape_history.sql Normal file
View File

@ -0,0 +1,11 @@
-- +migrate Up
CREATE TABLE "ScrapeHistory"
(
scrape_task_id CHAR(25) PRIMARY KEY,
user_source_id CHAR(25) NOT NULL REFERENCES "UserSource" (id),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
finished_at TIMESTAMP NULL,
error TEXT NULL,
added_posts INT NOT NULL,
deleted_posts INT NOT NULL
);

182
pkg/database/scrape_history.go Normal file
View File

@ -0,0 +1,182 @@
package database
import (
"context"
"errors"
"git.anthrove.art/Anthrove/otter-space-sdk/v4/internal/utils"
otterError "git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/error"
"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
log "github.com/sirupsen/logrus"
"go.opentelemetry.io/otel/attribute"
"gorm.io/gorm"
)
func CreateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory) (models.ScrapeHistory, error) {
ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "CreateScrapeHistory")
defer span.End()
localLogger = localLogger.WithFields(log.Fields{
"scrape_task_id": scrapeHistory.ScrapeTaskID,
"user_source_id": scrapeHistory.UserSourceID,
})
span.SetAttributes(
attribute.String("scrape_task_id", string(scrapeHistory.ScrapeTaskID)),
attribute.String("user_source_id", string(scrapeHistory.UserSourceID)),
)
utils.HandleEvent(span, localLogger, "Starting scrapeHistory creation")
if client == nil {
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DatabaseIsNotConnected})
}
if len(scrapeHistory.ScrapeTaskID) == 0 {
Alphyron marked this conversation as resolved Outdated
Alphyron commented 2024-10-15 10:29:30 +00:00
Outdated
Review
Copy Link

keine ID validierung / verifizierung?

keine ID validierung / verifizierung?
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty})
}
if len(scrapeHistory.ScrapeTaskID) != 25 {
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength})
}
Alphyron marked this conversation as resolved
Alphyron commented 2024-10-15 10:42:29 +00:00
Review
Copy Link

UserSourceID validation is missing

UserSourceID validation is missing
if len(scrapeHistory.UserSourceID) == 0 {
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.UserSourceIDIsEmpty})
}
if len(scrapeHistory.UserSourceID) != 25 {
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.UserSourceIsWrongLength})
}
result := client.WithContext(ctx).Create(&scrapeHistory)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.DuplicateKey})
}
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, result.Error)
}
utils.HandleEvent(span, localLogger, "scrapeHistory created successfully")
return scrapeHistory, nil
}
func GetScrapeHistoryByID(ctx context.Context, id models.ScrapeTaskID) (models.ScrapeHistory, error) {
ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "GetScrapeHistoryByID")
defer span.End()
localLogger = localLogger.WithFields(log.Fields{
"scrape_task_id": id,
})
span.SetAttributes(
Alphyron marked this conversation as resolved
Alphyron commented 2024-10-15 10:29:55 +00:00
Review
Copy Link

ID wrong length check is missing

ID wrong length check is missing
attribute.String("scrape_task_id", string(id)),
)
utils.HandleEvent(span, localLogger, "Starting get scrapeHistory by ID")
var post models.ScrapeHistory
if len(id) == 0 {
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty})
}
if len(id) != 25 {
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength})
}
result := client.WithContext(ctx).First(&post, "scrape_task_id = ?", id)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound})
}
return models.ScrapeHistory{}, utils.HandleError(ctx, span, localLogger, result.Error)
}
utils.HandleEvent(span, localLogger, "scrapeHistory retrieved successfully")
return post, nil
}
// UpdateScrapeHistory updates the scrape history information in the database.
// Only a few parameter can be updated:
// - FinishedAt
// - Error
// - AddedPosts
// - DeletedPosts
func UpdateScrapeHistory(ctx context.Context, scrapeHistory models.ScrapeHistory) error {
ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "UpdateScrapeHistory")
defer span.End()
localLogger = localLogger.WithFields(log.Fields{
"scrape_task_id": scrapeHistory.ScrapeTaskID,
Alphyron marked this conversation as resolved
Alphyron commented 2024-10-15 10:31:44 +00:00
Review
Copy Link

ID wrong length check is missing

ID wrong length check is missing
})
span.SetAttributes(
attribute.String("scrape_task_id", string(scrapeHistory.ScrapeTaskID)),
)
utils.HandleEvent(span, localLogger, "Starting scrapeHistory update")
if len(scrapeHistory.ScrapeTaskID) == 0 {
return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty})
}
if len(scrapeHistory.ScrapeTaskID) != 25 {
return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength})
}
updateScrapeHistory := models.ScrapeHistory{
ScrapeTaskID: scrapeHistory.ScrapeTaskID,
UserSourceID: scrapeHistory.UserSourceID,
FinishedAt: scrapeHistory.FinishedAt,
Error: scrapeHistory.Error,
AddedPosts: scrapeHistory.AddedPosts,
DeletedPosts: scrapeHistory.DeletedPosts,
}
result := client.WithContext(ctx).Model(&updateScrapeHistory).Updates(updateScrapeHistory)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound})
}
return utils.HandleError(ctx, span, localLogger, result.Error)
}
utils.HandleEvent(span, localLogger, "scrapeHistory updated successfully")
return nil
}
func DeleteScrapeHistory(ctx context.Context, id models.ScrapeTaskID) error {
ctx, span, localLogger := utils.SetupTracing(ctx, tracer, "DeleteScrapeHistory")
defer span.End()
localLogger = localLogger.WithFields(log.Fields{
"scrape_task_id": id,
})
Alphyron marked this conversation as resolved
Alphyron commented 2024-10-15 10:31:53 +00:00
Review
Copy Link

ID wrong length check is missing

ID wrong length check is missing
span.SetAttributes(
attribute.String("scrape_task_id", string(id)),
)
utils.HandleEvent(span, localLogger, "Starting delete scrapeHistory")
var scrapeHistory models.ScrapeHistory
if len(id) == 0 {
return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsEmpty})
}
if len(id) != 25 {
return utils.HandleError(ctx, span, localLogger, &otterError.EntityValidationFailed{Reason: otterError.ScrapeTaskIDIsWrongLength})
}
result := client.WithContext(ctx).Delete(&scrapeHistory, "scrape_task_id = ?", id)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
return utils.HandleError(ctx, span, localLogger, &otterError.Database{Reason: otterError.NoDataFound})
}
return utils.HandleError(ctx, span, localLogger, result.Error)
}
utils.HandleEvent(span, localLogger, "scrapeHistory deleted successfully")
return nil
}

411
pkg/database/scrape_history_test.go Normal file
View File

@ -0,0 +1,411 @@
package database
import (
"context"
"fmt"
"testing"
"time"
"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
"git.anthrove.art/Anthrove/otter-space-sdk/v4/test"
"go.opentelemetry.io/contrib/bridges/otellogrus"
"go.opentelemetry.io/otel"
"gorm.io/gorm"
)
func TestCreateScrapeHistory(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
logger.Fatalf("Could not start PostgreSQL container: %v", err)
}
client = gormDB
// Setup open telemetry
tracer = otel.Tracer(tracingName)
hook := otellogrus.NewHook(tracingName)
logger.AddHook(hook)
defer container.Terminate(ctx)
// -- -- Setup Tests
// -- Create User ot test with
validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}}
validUser, err = CreateUser(ctx, validUser)
if err != nil {
t.Fatalf("CreateUser err: %v", err)
}
// --
// -- Create Source to test with
validSource := models.Source{
DisplayName: "e621",
Domain: "e621.net",
Icon: "e621.net/icon.png",
}
validSource, err = CreateSource(ctx, validSource)
if err != nil {
t.Fatalf("CreateSource err: %v", err)
}
// --
// -- Create UserSource model
validUSerSource := models.UserSource{
BaseModel: models.BaseModel[models.UserSourceID]{
ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
DeletedAt: gorm.DeletedAt{},
},
User: models.User{},
UserID: validUser.ID,
Source: models.Source{},
SourceID: validSource.ID,
ScrapeTimeInterval: "P1D",
AccountUsername: "marry",
AccountID: "poppens",
LastScrapeTime: time.Now(),
AccountValidate: false,
AccountValidationKey: "im-a-key",
}
validUSerSource, err = CreateUserSource(ctx, validUSerSource)
if err != nil {
t.Fatalf("CreateUserSource err: %v", err)
}
validScrapeHistory := models.ScrapeHistory{
ScrapeTaskID: "000000000000valid_task_id",
UserSourceID: validUSerSource.ID,
}
// --
tests := []struct {
name string
args models.ScrapeHistory
wantErr bool
}{
{
name: "Valid ScrapeHistory",
args: validScrapeHistory,
wantErr: false,
},
{
name: "Duplicate ScrapeHistory",
args: validScrapeHistory,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := CreateScrapeHistory(ctx, tt.args)
if (err != nil) != tt.wantErr {
t.Errorf("CreateScrapeHistory() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestGetScrapeHistoryByID(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
logger.Fatalf("Could not start PostgreSQL container: %v", err)
}
client = gormDB
// Setup open telemetry
tracer = otel.Tracer(tracingName)
hook := otellogrus.NewHook(tracingName)
logger.AddHook(hook)
defer container.Terminate(ctx)
// -- -- Setup Tests
// -- Create User ot test with
validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}}
validUser, err = CreateUser(ctx, validUser)
if err != nil {
t.Fatalf("CreateUser err: %v", err)
}
// --
// -- Create Source to test with
validSource := models.Source{
DisplayName: "e621",
Domain: "e621.net",
Icon: "e621.net/icon.png",
}
validSource, err = CreateSource(ctx, validSource)
if err != nil {
t.Fatalf("CreateSource err: %v", err)
}
// --
// -- Create UserSource model
validUSerSource := models.UserSource{
BaseModel: models.BaseModel[models.UserSourceID]{
ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
DeletedAt: gorm.DeletedAt{},
},
User: models.User{},
UserID: validUser.ID,
Source: models.Source{},
SourceID: validSource.ID,
ScrapeTimeInterval: "P1D",
AccountUsername: "marry",
AccountID: "poppens",
LastScrapeTime: time.Now(),
AccountValidate: false,
AccountValidationKey: "im-a-key",
}
validUSerSource, err = CreateUserSource(ctx, validUSerSource)
if err != nil {
t.Fatalf("CreateUserSource err: %v", err)
}
validScrapeHistory := models.ScrapeHistory{
ScrapeTaskID: "000000000000valid_task_id",
UserSourceID: validUSerSource.ID,
}
validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory)
if err != nil {
t.Fatalf("CreateScrapeHistory err: %v", err)
}
// --
tests := []struct {
name string
id models.ScrapeTaskID
wantErr bool
}{
{"Valid ID", "000000000000valid_task_id", false},
{"Empty ID", "", true},
{"Non-existent ID", "non_existent_id", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := GetScrapeHistoryByID(ctx, tt.id)
if (err != nil) != tt.wantErr {
t.Errorf("GetScrapeHistoryByID() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestUpdateScrapeHistory(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
logger.Fatalf("Could not start PostgreSQL container: %v", err)
}
client = gormDB
// Setup open telemetry
tracer = otel.Tracer(tracingName)
hook := otellogrus.NewHook(tracingName)
logger.AddHook(hook)
defer container.Terminate(ctx)
// -- -- Setup Tests
// -- Create User ot test with
validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}}
validUser, err = CreateUser(ctx, validUser)
if err != nil {
t.Fatalf("CreateUser err: %v", err)
}
// --
// -- Create Source to test with
validSource := models.Source{
DisplayName: "e621",
Domain: "e621.net",
Icon: "e621.net/icon.png",
}
validSource, err = CreateSource(ctx, validSource)
if err != nil {
t.Fatalf("CreateSource err: %v", err)
}
// --
// -- Create UserSource model
validUSerSource := models.UserSource{
BaseModel: models.BaseModel[models.UserSourceID]{
ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
DeletedAt: gorm.DeletedAt{},
},
User: models.User{},
UserID: validUser.ID,
Source: models.Source{},
SourceID: validSource.ID,
ScrapeTimeInterval: "P1D",
AccountUsername: "marry",
AccountID: "poppens",
LastScrapeTime: time.Now(),
AccountValidate: false,
AccountValidationKey: "im-a-key",
}
validUSerSource, err = CreateUserSource(ctx, validUSerSource)
if err != nil {
t.Fatalf("CreateUserSource err: %v", err)
}
validScrapeHistory := models.ScrapeHistory{
ScrapeTaskID: "000000000000valid_task_id",
UserSourceID: validUSerSource.ID,
}
validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory)
if err != nil {
t.Fatalf("CreateScrapeHistory err: %v", err)
}
updatedScrapeHistory := validScrapeHistory
updatedScrapeHistory.FinishedAt = time.Now()
// --
tests := []struct {
name string
args models.ScrapeHistory
wantErr bool
}{
{"Valid Update", updatedScrapeHistory, false},
{"Empty ID", models.ScrapeHistory{ScrapeTaskID: ""}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := UpdateScrapeHistory(ctx, tt.args)
if (err != nil) != tt.wantErr {
t.Errorf("UpdateScrapeHistory() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestDeleteScrapeHistory(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
logger.Fatalf("Could not start PostgreSQL container: %v", err)
}
client = gormDB
// Setup open telemetry
tracer = otel.Tracer(tracingName)
hook := otellogrus.NewHook(tracingName)
logger.AddHook(hook)
defer container.Terminate(ctx)
// -- -- Setup Tests
// -- Create User ot test with
validUser := models.User{BaseModel: models.BaseModel[models.UserID]{ID: models.UserID(fmt.Sprintf("%025s", "User1"))}}
validUser, err = CreateUser(ctx, validUser)
if err != nil {
t.Fatalf("CreateUser err: %v", err)
}
// --
// -- Create Source to test with
validSource := models.Source{
DisplayName: "e621",
Domain: "e621.net",
Icon: "e621.net/icon.png",
}
validSource, err = CreateSource(ctx, validSource)
if err != nil {
t.Fatalf("CreateSource err: %v", err)
}
// --
// -- Create UserSource model
validUSerSource := models.UserSource{
BaseModel: models.BaseModel[models.UserSourceID]{
ID: models.UserSourceID(fmt.Sprintf("%025s", "UserSourceId1")),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
DeletedAt: gorm.DeletedAt{},
},
User: models.User{},
UserID: validUser.ID,
Source: models.Source{},
SourceID: validSource.ID,
ScrapeTimeInterval: "P1D",
AccountUsername: "marry",
AccountID: "poppens",
LastScrapeTime: time.Now(),
AccountValidate: false,
AccountValidationKey: "im-a-key",
}
validUSerSource, err = CreateUserSource(ctx, validUSerSource)
if err != nil {
t.Fatalf("CreateUserSource err: %v", err)
}
validScrapeHistory := models.ScrapeHistory{
ScrapeTaskID: "000000000000valid_task_id",
UserSourceID: validUSerSource.ID,
}
validScrapeHistory, err = CreateScrapeHistory(ctx, validScrapeHistory)
if err != nil {
t.Fatalf("CreateScrapeHistory err: %v", err)
}
// --
tests := []struct {
name string
id models.ScrapeTaskID
wantErr bool
}{
{"Valid ID", "000000000000valid_task_id", false},
{"Empty ID", "", true},
{"Short ID", "short_id", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := DeleteScrapeHistory(ctx, tt.id)
if (err != nil) != tt.wantErr {
t.Errorf("DeleteScrapeHistory() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

3
pkg/error/validation.go
View File

@ -35,6 +35,9 @@ const (
PoolIDIsWrongLength = "PoolID has the wrong length"
PoolURLIsEmpty = "PoolURL cannot be empty"
ScrapeTaskIDIsEmpty = "ScrapeTaskID cannot be empty"
ScrapeTaskIDIsWrongLength = "ScrapeTaskID has the wrong length, needs to be 25 characters long"
)
type EntityValidationFailed struct {

2
pkg/models/const.go
View File

@ -23,6 +23,8 @@ type (
UserSourceID string
UserFavoriteID string
ScrapeTaskID string
)
const (

19
pkg/models/scrape_history.go Normal file
View File

@ -0,0 +1,19 @@
package models
import (
"time"
)
type ScrapeHistory struct {
ScrapeTaskID ScrapeTaskID `json:"scrape_task_id" gorm:"primaryKey"`
UserSourceID UserSourceID `json:"user_source_id" gorm:""`
CreatedAt time.Time `json:"created_at" gorm:""`
FinishedAt time.Time `json:"finished_at" gorm:""`
Error string `json:"error" gorm:"null"`
AddedPosts int `json:"added_posts" gorm:"not null"`
DeletedPosts int `json:"deleted_posts" gorm:"not null"`
}
func (ScrapeHistory) TableName() string {
return "ScrapeHistory"
}

11
pkg/models/scrape_history_test.go Normal file
View File

@ -0,0 +1,11 @@
package models
import "testing"
func TestScrapeHistory_TableName(t *testing.T) {
scrapeHistory := ScrapeHistory{}
expectedTableName := "ScrapeHistory"
if tableName := scrapeHistory.TableName(); tableName != expectedTableName {
t.Fatalf("expected %s, but got %s", expectedTableName, tableName)
}
}