From 19b96e64e14beea8683a65be10cf491448ea8bc9 Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 27 Aug 2024 12:25:00 +0200 Subject: [PATCH] feat: added summery - Introduce `BatchSummery` struct for post and favorite counts - Add `BatchPostProcessingWithSummery` function to include summary - Update error returns to include empty `BatchSummery` struct --- pkg/plug/scrape.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/pkg/plug/scrape.go b/pkg/plug/scrape.go index 1b0652e..4d7854d 100644 --- a/pkg/plug/scrape.go +++ b/pkg/plug/scrape.go @@ -16,7 +16,17 @@ import ( var BatchSize = 50 var BasicLoggingFields log.Fields +type BatchSummery struct { + AddedPosts int64 + AddedFavorites int64 +} + func BatchPostProcessing(ctx context.Context, userSource models.UserSource, posts []models.Post) error { + _, err := BatchPostProcessingWithSummery(ctx, userSource, posts) + return err +} + +func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserSource, posts []models.Post) (BatchSummery, error) { ctx, span := tracer.Start(ctx, "BatchPostProcessing") defer span.End() @@ -39,7 +49,7 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post span.RecordError(err) span.SetStatus(codes.Error, err.Error()) log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to get Gorm DB") - return err + return BatchSummery{}, err } postIDs := make([]string, 0, len(posts)) @@ -54,7 +64,7 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post span.RecordError(err) span.SetStatus(codes.Error, err.Error()) log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch existing posts") - return err + return BatchSummery{}, err } span.AddEvent("Fetched existing posts", trace.WithAttributes(attribute.Int("existing_post_count", len(existingPosts)))) log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("existing_post_count", len(existingPosts)).Info("Fetched existing posts") @@ -70,7 +80,7 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post span.RecordError(err) span.SetStatus(codes.Error, err.Error()) log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch existing favorite posts") - return err + return BatchSummery{}, err } span.AddEvent("Fetched existing favorite posts", trace.WithAttributes(attribute.Int("existing_fav_post_count", len(existingFavPostIDs)))) log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("existing_fav_post_count", len(existingFavPostIDs)).Info("Fetched existing favorite posts") @@ -108,7 +118,7 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post span.RecordError(err) span.SetStatus(codes.Error, err.Error()) log.WithContext(ctx).WithError(err).Error("Failed to create new posts in batch") - return err + return BatchSummery{}, err } span.AddEvent("Created new posts in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize))) log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created new posts in batch") @@ -120,13 +130,16 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post span.RecordError(err) span.SetStatus(codes.Error, err.Error()) log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to create user favorites in batch") - return err + return BatchSummery{}, err } span.AddEvent("Created user favorites in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize))) log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created user favorites in batch") } - return nil + return BatchSummery{ + AddedPosts: int64(len(newPosts)), + AddedFavorites: int64(len(anthroveFaves)), + }, nil } func getAnthrovePost(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string) ([]models.PostReference, error) { -- 2.45.2