New and generic scrape algorithm #11

Merged
SoXX merged 20 commits from dev/algorithm into main 2024-11-01 21:39:21 +00:00
2 changed files with 7 additions and 6 deletions
Showing only changes of commit 4d209f2a90 - Show all commits

View File

@ -9,6 +9,7 @@ import (
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"slices"
"time"
)
type User struct {
@ -132,7 +133,12 @@ outer:
if len(anthroveFaves) > 0 {
span.AddEvent("Executing CreateUserFavoriteInBatch")
for i, fav := range anthroveFaves {
fav.CreatedAt = time.Now().Add(time.Millisecond * time.Duration(i) * -1)
}
slices.Reverse(anthroveFaves)
err = database.CreateUserFavoriteInBatch(ctx, anthroveFaves, BatchSize)
if err != nil {
span.RecordError(err)

View File

@ -2,9 +2,6 @@ package plug
import (
"context"
"slices"
"time"
"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/database"
"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
log "github.com/sirupsen/logrus"
@ -12,6 +9,7 @@ import (
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gorm.io/gorm"
"slices"
)
var BatchSize = 50
@ -98,9 +96,6 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS
return found
}) {
anthroveFaves = append(anthroveFaves, models.UserFavorite{
BaseModel: models.BaseModel[models.UserFavoriteID]{
CreatedAt: time.Now().Add(time.Millisecond * time.Duration(i) * -1),
},
UserID: userSource.UserID,
PostID: post.ID,
UserSourceID: userSource.ID,