Anthrove/plug-sdk
4
0
Fork 0
Code Issues Pull Requests Actions Projects Releases 6 Activity

feat(telemetry): Integrate OpenTelemetry tracing into BatchPostProcessing function

Browse Source
This commit is contained in:
SoXX 2024-08-27 09:49:20 +02:00
parent 0070352623
commit 08e4364069
1 changed files with 37 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
pkg/plug/scrape.go
View File

@ -6,15 +6,22 @@ import (
"git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/database"
"git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/models"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gorm.io/gorm"
)
var BatchSize = 50
func BatchPostProcessing(ctx context.Context, userSource models.UserSource, posts []models.Post) error {
ctx, span := tracer.Start(ctx, "BatchPostProcessing")
defer span.End()
db, err := database.GetGorm(ctx)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
@ -22,23 +29,29 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post
for _, post := range posts {
postIDs = append(postIDs, post.References[0].SourcePostID)
}
span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs))))
existingPosts, err := getAnthrovePost(ctx, db, userSource.SourceID, postIDs) // Third Query
existingPosts, err := getAnthrovePost(ctx, db, userSource.SourceID, postIDs)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
span.AddEvent("Fetched existing posts", trace.WithAttributes(attribute.Int("existing_post_count", len(existingPosts))))
// for loop to convert PostReference to Post IDs
var existingPostIDs []models.PostID
for _, post := range existingPosts {
existingPostIDs = append(existingPostIDs, models.PostID(post.PostID))
}
var existingFavPostIDs []models.PostID
existingFavPostIDs, err = getAlreadyFavoritesPostIDs(ctx, db, existingPostIDs, userSource.ID) // Fourth Query
existingFavPostIDs, err = getAlreadyFavoritesPostIDs(ctx, db, existingPostIDs, userSource.ID)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
span.AddEvent("Fetched existing favorite posts", trace.WithAttributes(attribute.Int("existing_fav_post_count", len(existingFavPostIDs))))
anthroveFaves := make([]models.UserFavorite, 0, len(existingPosts))
newPosts := make([]models.Post, 0, len(existingPosts))
@ -46,7 +59,6 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post
if !slices.ContainsFunc(existingPosts, func(reference models.PostReference) bool {
found := reference.SourcePostID == post.References[0].SourcePostID
if found {
// If Favoure is already existing skip this step TODO
if !slices.Contains(existingFavPostIDs, models.PostID(reference.PostID)) {
anthroveFaves = append(anthroveFaves, models.UserFavorite{
UserID: userSource.UserID,
@ -55,56 +67,69 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post
})
}
}
return found
}) {
// Here we always need to create a fav because new post, so new fav too!
anthroveFaves = append(anthroveFaves, models.UserFavorite{
UserID: userSource.UserID,
PostID: post.ID,
UserSourceID: userSource.ID,
})
newPosts = append(newPosts, post)
}
}
span.AddEvent("Processed posts for favorites and new posts", trace.WithAttributes(attribute.Int("new_post_count", len(newPosts)), attribute.Int("new_fav_count", len(anthroveFaves))))
if len(newPosts) > 0 {
err = database.CreatePostInBatch(ctx, newPosts, BatchSize) // Fifth Query
err = database.CreatePostInBatch(ctx, newPosts, BatchSize)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
span.AddEvent("Created new posts in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize)))
}
if len(anthroveFaves) > 0 {
err = database.CreateUserFavoriteInBatch(ctx, anthroveFaves, BatchSize) // Sixth Query
err = database.CreateUserFavoriteInBatch(ctx, anthroveFaves, BatchSize)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return err
}
span.AddEvent("Created user favorites in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize)))
}
return nil
}
func getAnthrovePost(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string) ([]models.PostReference, error) {
var existingPosts []models.PostReference
ctx, span := tracer.Start(ctx, "getAnthrovePost")
defer span.End()
var existingPosts []models.PostReference
err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND source_post_id IN ?", id, postIDs).Error
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return existingPosts, err
}
span.AddEvent("Fetched Anthrove posts", trace.WithAttributes(attribute.Int("post_count", len(existingPosts))))
return existingPosts, nil
}
func getAlreadyFavoritesPostIDs(ctx context.Context, gorm *gorm.DB, existingPostIDs []models.PostID, userSourceID models.UserSourceID) ([]models.PostID, error) {
var existingFavPostIDS []models.PostID
ctx, span := tracer.Start(ctx, "getAlreadyFavoritesPostIDs")
defer span.End()
// SELECT * FROM UserFavourite WHERE post_id IN (Post IDS) AND user_source_id = {your id} // Second Query
var existingFavPostIDS []models.PostID
err := gorm.WithContext(ctx).Model(&models.UserFavorite{}).Select("post_id").Find(&existingFavPostIDS, "user_source_id = ? AND post_id IN ?", userSourceID, existingPostIDs).Error
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return existingFavPostIDS, err
}
span.AddEvent("Fetched already favorite post IDs", trace.WithAttributes(attribute.Int("fav_post_count", len(existingFavPostIDS))))
return existingFavPostIDS, nil
}