From 969bc177377aafae5168d8d731d7f07bab39cee3 Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 27 Aug 2024 10:04:55 +0200 Subject: [PATCH] feat(telemetry): added more info --- pkg/plug/scrape.go | 51 ++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/pkg/plug/scrape.go b/pkg/plug/scrape.go index 7358770..1b0652e 100644 --- a/pkg/plug/scrape.go +++ b/pkg/plug/scrape.go @@ -14,17 +14,31 @@ import ( ) var BatchSize = 50 +var BasicLoggingFields log.Fields func BatchPostProcessing(ctx context.Context, userSource models.UserSource, posts []models.Post) error { ctx, span := tracer.Start(ctx, "BatchPostProcessing") defer span.End() - log.WithContext(ctx).Info("Starting BatchPostProcessing") + + span.SetAttributes( + attribute.String("user_source_id", string(userSource.ID)), + attribute.String("user_source_user_id", string(userSource.UserID)), + attribute.String("user_source_source_id", string(userSource.SourceID)), + ) + + BasicLoggingFields = log.Fields{ + "user_source_id": userSource.ID, + "user_source_user_id": userSource.UserID, + "user_source_source_id": userSource.SourceID, + } + + log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting BatchPostProcessing") db, err := database.GetGorm(ctx) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) - log.WithContext(ctx).WithError(err).Error("Failed to get Gorm DB") + log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to get Gorm DB") return err } @@ -33,17 +47,17 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post postIDs = append(postIDs, post.References[0].SourcePostID) } span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs)))) - log.WithContext(ctx).WithField("post_count", len(postIDs)).Info("Collected post IDs") + log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("post_count", len(postIDs)).Info("Collected post IDs") existingPosts, err := getAnthrovePost(ctx, db, userSource.SourceID, postIDs) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) - log.WithContext(ctx).WithError(err).Error("Failed to fetch existing posts") + log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch existing posts") return err } span.AddEvent("Fetched existing posts", trace.WithAttributes(attribute.Int("existing_post_count", len(existingPosts)))) - log.WithContext(ctx).WithField("existing_post_count", len(existingPosts)).Info("Fetched existing posts") + log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("existing_post_count", len(existingPosts)).Info("Fetched existing posts") var existingPostIDs []models.PostID for _, post := range existingPosts { @@ -55,11 +69,11 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) - log.WithContext(ctx).WithError(err).Error("Failed to fetch existing favorite posts") + log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch existing favorite posts") return err } span.AddEvent("Fetched existing favorite posts", trace.WithAttributes(attribute.Int("existing_fav_post_count", len(existingFavPostIDs)))) - log.WithContext(ctx).WithField("existing_fav_post_count", len(existingFavPostIDs)).Info("Fetched existing favorite posts") + log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("existing_fav_post_count", len(existingFavPostIDs)).Info("Fetched existing favorite posts") anthroveFaves := make([]models.UserFavorite, 0, len(existingPosts)) newPosts := make([]models.Post, 0, len(existingPosts)) @@ -86,10 +100,7 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post } } span.AddEvent("Processed posts for favorites and new posts", trace.WithAttributes(attribute.Int("new_post_count", len(newPosts)), attribute.Int("new_fav_count", len(anthroveFaves)))) - log.WithContext(ctx).WithFields(log.Fields{ - "new_post_count": len(newPosts), - "new_fav_count": len(anthroveFaves), - }).Info("Processed posts for favorites and new posts") + log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Processed posts for favorites and new posts") if len(newPosts) > 0 { err = database.CreatePostInBatch(ctx, newPosts, BatchSize) @@ -100,7 +111,7 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post return err } span.AddEvent("Created new posts in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize))) - log.WithContext(ctx).WithField("batch_size", BatchSize).Info("Created new posts in batch") + log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created new posts in batch") } if len(anthroveFaves) > 0 { @@ -108,11 +119,11 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) - log.WithContext(ctx).WithError(err).Error("Failed to create user favorites in batch") + log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to create user favorites in batch") return err } span.AddEvent("Created user favorites in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize))) - log.WithContext(ctx).WithField("batch_size", BatchSize).Info("Created user favorites in batch") + log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created user favorites in batch") } return nil @@ -122,18 +133,18 @@ func getAnthrovePost(ctx context.Context, gorm *gorm.DB, id models.SourceID, pos ctx, span := tracer.Start(ctx, "getAnthrovePost") defer span.End() - log.WithContext(ctx).Info("Starting getAnthrovePost") + log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAnthrovePost") var existingPosts []models.PostReference err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND source_post_id IN ?", id, postIDs).Error if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) - log.WithContext(ctx).WithError(err).Error("Failed to fetch Anthrove posts") + log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch Anthrove posts") return existingPosts, err } span.AddEvent("Fetched Anthrove posts", trace.WithAttributes(attribute.Int("post_count", len(existingPosts)))) - log.WithContext(ctx).WithField("post_count", len(existingPosts)).Info("Fetched Anthrove posts") + log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("post_count", len(existingPosts)).Info("Fetched Anthrove posts") return existingPosts, nil } @@ -142,18 +153,18 @@ func getAlreadyFavoritesPostIDs(ctx context.Context, gorm *gorm.DB, existingPost ctx, span := tracer.Start(ctx, "getAlreadyFavoritesPostIDs") defer span.End() - log.WithContext(ctx).Info("Starting getAlreadyFavoritesPostIDs") + log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAlreadyFavoritesPostIDs") var existingFavPostIDS []models.PostID err := gorm.WithContext(ctx).Model(&models.UserFavorite{}).Select("post_id").Find(&existingFavPostIDS, "user_source_id = ? AND post_id IN ?", userSourceID, existingPostIDs).Error if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) - log.WithContext(ctx).WithError(err).Error("Failed to fetch already favorite post IDs") + log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch already favorite post IDs") return existingFavPostIDS, err } span.AddEvent("Fetched already favorite post IDs", trace.WithAttributes(attribute.Int("fav_post_count", len(existingFavPostIDS)))) - log.WithContext(ctx).WithField("fav_post_count", len(existingFavPostIDS)).Info("Fetched already favorite post IDs") + log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("fav_post_count", len(existingFavPostIDS)).Info("Fetched already favorite post IDs") return existingFavPostIDS, nil }