From 08e436406922dbb5e85dacac182f065be907b36f Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 27 Aug 2024 09:49:20 +0200 Subject: [PATCH] feat(telemetry): Integrate OpenTelemetry tracing into BatchPostProcessing function --- pkg/plug/scrape.go | 49 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/pkg/plug/scrape.go b/pkg/plug/scrape.go index 78e0224..a78ab30 100644 --- a/pkg/plug/scrape.go +++ b/pkg/plug/scrape.go @@ -6,15 +6,22 @@ import ( "git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/database" "git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/models" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "gorm.io/gorm" ) var BatchSize = 50 func BatchPostProcessing(ctx context.Context, userSource models.UserSource, posts []models.Post) error { + ctx, span := tracer.Start(ctx, "BatchPostProcessing") + defer span.End() db, err := database.GetGorm(ctx) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) return err } @@ -22,23 +29,29 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post for _, post := range posts { postIDs = append(postIDs, post.References[0].SourcePostID) } + span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs)))) - existingPosts, err := getAnthrovePost(ctx, db, userSource.SourceID, postIDs) // Third Query + existingPosts, err := getAnthrovePost(ctx, db, userSource.SourceID, postIDs) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) return err } + span.AddEvent("Fetched existing posts", trace.WithAttributes(attribute.Int("existing_post_count", len(existingPosts)))) - // for loop to convert PostReference to Post IDs var existingPostIDs []models.PostID for _, post := range existingPosts { existingPostIDs = append(existingPostIDs, models.PostID(post.PostID)) } var existingFavPostIDs []models.PostID - existingFavPostIDs, err = getAlreadyFavoritesPostIDs(ctx, db, existingPostIDs, userSource.ID) // Fourth Query + existingFavPostIDs, err = getAlreadyFavoritesPostIDs(ctx, db, existingPostIDs, userSource.ID) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) return err } + span.AddEvent("Fetched existing favorite posts", trace.WithAttributes(attribute.Int("existing_fav_post_count", len(existingFavPostIDs)))) anthroveFaves := make([]models.UserFavorite, 0, len(existingPosts)) newPosts := make([]models.Post, 0, len(existingPosts)) @@ -46,7 +59,6 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post if !slices.ContainsFunc(existingPosts, func(reference models.PostReference) bool { found := reference.SourcePostID == post.References[0].SourcePostID if found { - // If Favoure is already existing skip this step TODO if !slices.Contains(existingFavPostIDs, models.PostID(reference.PostID)) { anthroveFaves = append(anthroveFaves, models.UserFavorite{ UserID: userSource.UserID, @@ -55,56 +67,69 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post }) } } - return found }) { - // Here we always need to create a fav because new post, so new fav too! anthroveFaves = append(anthroveFaves, models.UserFavorite{ UserID: userSource.UserID, PostID: post.ID, UserSourceID: userSource.ID, }) - newPosts = append(newPosts, post) } } + span.AddEvent("Processed posts for favorites and new posts", trace.WithAttributes(attribute.Int("new_post_count", len(newPosts)), attribute.Int("new_fav_count", len(anthroveFaves)))) if len(newPosts) > 0 { - err = database.CreatePostInBatch(ctx, newPosts, BatchSize) // Fifth Query + err = database.CreatePostInBatch(ctx, newPosts, BatchSize) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) return err } + span.AddEvent("Created new posts in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize))) } if len(anthroveFaves) > 0 { - err = database.CreateUserFavoriteInBatch(ctx, anthroveFaves, BatchSize) // Sixth Query + err = database.CreateUserFavoriteInBatch(ctx, anthroveFaves, BatchSize) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) return err } + span.AddEvent("Created user favorites in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize))) } return nil } func getAnthrovePost(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string) ([]models.PostReference, error) { - var existingPosts []models.PostReference + ctx, span := tracer.Start(ctx, "getAnthrovePost") + defer span.End() + var existingPosts []models.PostReference err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND source_post_id IN ?", id, postIDs).Error if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) return existingPosts, err } + span.AddEvent("Fetched Anthrove posts", trace.WithAttributes(attribute.Int("post_count", len(existingPosts)))) return existingPosts, nil } func getAlreadyFavoritesPostIDs(ctx context.Context, gorm *gorm.DB, existingPostIDs []models.PostID, userSourceID models.UserSourceID) ([]models.PostID, error) { - var existingFavPostIDS []models.PostID + ctx, span := tracer.Start(ctx, "getAlreadyFavoritesPostIDs") + defer span.End() - // SELECT * FROM UserFavourite WHERE post_id IN (Post IDS) AND user_source_id = {your id} // Second Query + var existingFavPostIDS []models.PostID err := gorm.WithContext(ctx).Model(&models.UserFavorite{}).Select("post_id").Find(&existingFavPostIDS, "user_source_id = ? AND post_id IN ?", userSourceID, existingPostIDs).Error if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) return existingFavPostIDS, err } + span.AddEvent("Fetched already favorite post IDs", trace.WithAttributes(attribute.Int("fav_post_count", len(existingFavPostIDS)))) return existingFavPostIDS, nil }