Anthrove/plug-sdk
5
0
Fork 0
Code Issues Pull Requests Actions Projects Releases 10 Activity

feat(telemetry): added more info

Browse Source
This commit is contained in:
SoXX 2024-08-27 10:04:55 +02:00
parent 942bbede1b
commit 969bc17737
1 changed files with 31 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
pkg/plug/scrape.go
View File

@ -14,17 +14,31 @@ import (
) )
var BatchSize = 50 var BatchSize = 50
var BasicLoggingFields log.Fields
func BatchPostProcessing(ctx context.Context, userSource models.UserSource, posts []models.Post) error { func BatchPostProcessing(ctx context.Context, userSource models.UserSource, posts []models.Post) error {
ctx, span := tracer.Start(ctx, "BatchPostProcessing") ctx, span := tracer.Start(ctx, "BatchPostProcessing")
defer span.End() defer span.End()
log.WithContext(ctx).Info("Starting BatchPostProcessing")
span.SetAttributes(
attribute.String("user_source_id", string(userSource.ID)),
attribute.String("user_source_user_id", string(userSource.UserID)),
attribute.String("user_source_source_id", string(userSource.SourceID)),
)
BasicLoggingFields = log.Fields{
"user_source_id": userSource.ID,
"user_source_user_id": userSource.UserID,
"user_source_source_id": userSource.SourceID,
}
log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting BatchPostProcessing")
db, err := database.GetGorm(ctx) db, err := database.GetGorm(ctx)
if err != nil { if err != nil {
span.RecordError(err) span.RecordError(err)
span.SetStatus(codes.Error, err.Error()) span.SetStatus(codes.Error, err.Error())
log.WithContext(ctx).WithError(err).Error("Failed to get Gorm DB") log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to get Gorm DB")
return err return err
} }
@ -33,17 +47,17 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post
postIDs = append(postIDs, post.References[0].SourcePostID) postIDs = append(postIDs, post.References[0].SourcePostID)
} }
span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs)))) span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs))))
log.WithContext(ctx).WithField("post_count", len(postIDs)).Info("Collected post IDs") log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("post_count", len(postIDs)).Info("Collected post IDs")
existingPosts, err := getAnthrovePost(ctx, db, userSource.SourceID, postIDs) existingPosts, err := getAnthrovePost(ctx, db, userSource.SourceID, postIDs)
if err != nil { if err != nil {
span.RecordError(err) span.RecordError(err)
span.SetStatus(codes.Error, err.Error()) span.SetStatus(codes.Error, err.Error())
log.WithContext(ctx).WithError(err).Error("Failed to fetch existing posts") log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch existing posts")
return err return err
} }
span.AddEvent("Fetched existing posts", trace.WithAttributes(attribute.Int("existing_post_count", len(existingPosts)))) span.AddEvent("Fetched existing posts", trace.WithAttributes(attribute.Int("existing_post_count", len(existingPosts))))
log.WithContext(ctx).WithField("existing_post_count", len(existingPosts)).Info("Fetched existing posts") log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("existing_post_count", len(existingPosts)).Info("Fetched existing posts")
var existingPostIDs []models.PostID var existingPostIDs []models.PostID
for _, post := range existingPosts { for _, post := range existingPosts {
@ -55,11 +69,11 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post
if err != nil { if err != nil {
span.RecordError(err) span.RecordError(err)
span.SetStatus(codes.Error, err.Error()) span.SetStatus(codes.Error, err.Error())
log.WithContext(ctx).WithError(err).Error("Failed to fetch existing favorite posts") log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch existing favorite posts")
return err return err
} }
span.AddEvent("Fetched existing favorite posts", trace.WithAttributes(attribute.Int("existing_fav_post_count", len(existingFavPostIDs)))) span.AddEvent("Fetched existing favorite posts", trace.WithAttributes(attribute.Int("existing_fav_post_count", len(existingFavPostIDs))))
log.WithContext(ctx).WithField("existing_fav_post_count", len(existingFavPostIDs)).Info("Fetched existing favorite posts") log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("existing_fav_post_count", len(existingFavPostIDs)).Info("Fetched existing favorite posts")
anthroveFaves := make([]models.UserFavorite, 0, len(existingPosts)) anthroveFaves := make([]models.UserFavorite, 0, len(existingPosts))
newPosts := make([]models.Post, 0, len(existingPosts)) newPosts := make([]models.Post, 0, len(existingPosts))
@ -86,10 +100,7 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post
} }
} }
span.AddEvent("Processed posts for favorites and new posts", trace.WithAttributes(attribute.Int("new_post_count", len(newPosts)), attribute.Int("new_fav_count", len(anthroveFaves)))) span.AddEvent("Processed posts for favorites and new posts", trace.WithAttributes(attribute.Int("new_post_count", len(newPosts)), attribute.Int("new_fav_count", len(anthroveFaves))))
log.WithContext(ctx).WithFields(log.Fields{ log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Processed posts for favorites and new posts")
"new_post_count": len(newPosts),
"new_fav_count": len(anthroveFaves),
}).Info("Processed posts for favorites and new posts")
if len(newPosts) > 0 { if len(newPosts) > 0 {
err = database.CreatePostInBatch(ctx, newPosts, BatchSize) err = database.CreatePostInBatch(ctx, newPosts, BatchSize)
@ -100,7 +111,7 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post
return err return err
} }
span.AddEvent("Created new posts in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize))) span.AddEvent("Created new posts in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize)))
log.WithContext(ctx).WithField("batch_size", BatchSize).Info("Created new posts in batch") log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created new posts in batch")
} }
if len(anthroveFaves) > 0 { if len(anthroveFaves) > 0 {
@ -108,11 +119,11 @@ func BatchPostProcessing(ctx context.Context, userSource models.UserSource, post
if err != nil { if err != nil {
span.RecordError(err) span.RecordError(err)
span.SetStatus(codes.Error, err.Error()) span.SetStatus(codes.Error, err.Error())
log.WithContext(ctx).WithError(err).Error("Failed to create user favorites in batch") log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to create user favorites in batch")
return err return err
} }
span.AddEvent("Created user favorites in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize))) span.AddEvent("Created user favorites in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize)))
log.WithContext(ctx).WithField("batch_size", BatchSize).Info("Created user favorites in batch") log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created user favorites in batch")
} }
return nil return nil
@ -122,18 +133,18 @@ func getAnthrovePost(ctx context.Context, gorm *gorm.DB, id models.SourceID, pos
ctx, span := tracer.Start(ctx, "getAnthrovePost") ctx, span := tracer.Start(ctx, "getAnthrovePost")
defer span.End() defer span.End()
log.WithContext(ctx).Info("Starting getAnthrovePost") log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAnthrovePost")
var existingPosts []models.PostReference var existingPosts []models.PostReference
err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND source_post_id IN ?", id, postIDs).Error err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND source_post_id IN ?", id, postIDs).Error
if err != nil { if err != nil {
span.RecordError(err) span.RecordError(err)
span.SetStatus(codes.Error, err.Error()) span.SetStatus(codes.Error, err.Error())
log.WithContext(ctx).WithError(err).Error("Failed to fetch Anthrove posts") log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch Anthrove posts")
return existingPosts, err return existingPosts, err
} }
span.AddEvent("Fetched Anthrove posts", trace.WithAttributes(attribute.Int("post_count", len(existingPosts)))) span.AddEvent("Fetched Anthrove posts", trace.WithAttributes(attribute.Int("post_count", len(existingPosts))))
log.WithContext(ctx).WithField("post_count", len(existingPosts)).Info("Fetched Anthrove posts") log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("post_count", len(existingPosts)).Info("Fetched Anthrove posts")
return existingPosts, nil return existingPosts, nil
} }
@ -142,18 +153,18 @@ func getAlreadyFavoritesPostIDs(ctx context.Context, gorm *gorm.DB, existingPost
ctx, span := tracer.Start(ctx, "getAlreadyFavoritesPostIDs") ctx, span := tracer.Start(ctx, "getAlreadyFavoritesPostIDs")
defer span.End() defer span.End()
log.WithContext(ctx).Info("Starting getAlreadyFavoritesPostIDs") log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAlreadyFavoritesPostIDs")
var existingFavPostIDS []models.PostID var existingFavPostIDS []models.PostID
err := gorm.WithContext(ctx).Model(&models.UserFavorite{}).Select("post_id").Find(&existingFavPostIDS, "user_source_id = ? AND post_id IN ?", userSourceID, existingPostIDs).Error err := gorm.WithContext(ctx).Model(&models.UserFavorite{}).Select("post_id").Find(&existingFavPostIDS, "user_source_id = ? AND post_id IN ?", userSourceID, existingPostIDs).Error
if err != nil { if err != nil {
span.RecordError(err) span.RecordError(err)
span.SetStatus(codes.Error, err.Error()) span.SetStatus(codes.Error, err.Error())
log.WithContext(ctx).WithError(err).Error("Failed to fetch already favorite post IDs") log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch already favorite post IDs")
return existingFavPostIDS, err return existingFavPostIDS, err
} }
span.AddEvent("Fetched already favorite post IDs", trace.WithAttributes(attribute.Int("fav_post_count", len(existingFavPostIDS)))) span.AddEvent("Fetched already favorite post IDs", trace.WithAttributes(attribute.Int("fav_post_count", len(existingFavPostIDS))))
log.WithContext(ctx).WithField("fav_post_count", len(existingFavPostIDS)).Info("Fetched already favorite post IDs") log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("fav_post_count", len(existingFavPostIDS)).Info("Fetched already favorite post IDs")
return existingFavPostIDS, nil return existingFavPostIDS, nil
} }