Anthrove/plug-sdk
5
0
Fork 0
Code Issues Pull Requests Actions Projects Releases 11 Activity

feat: Add source URLs to batch post processing and update database query #14

Merged
SoXX merged 1 commits from dev/#13-enhance-sql into main 2024-11-08 20:12:40 +00:00
Conversation 0 Commits 1 Files Changed 1 +6 -3
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pkg/plug/scrape.go
View File

@ -48,13 +48,15 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS
}
postIDs := make([]string, 0, len(posts))
sourceUrls := make([]string, 0, len(posts))
for _, post := range posts {
postIDs = append(postIDs, post.References[0].SourcePostID)
sourceUrls = append(sourceUrls, post.References[0].URL)
}
span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs))))
log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("post_count", len(postIDs)).Info("Collected post IDs")
existingPostsReferences, err := getAnthrovePostReferences(ctx, db, userSource.SourceID, postIDs)
existingPostsReferences, err := getAnthrovePostReferences(ctx, db, userSource.SourceID, postIDs, sourceUrls)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@ -111,14 +113,15 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS
return newPosts, anthroveFaves, nil
}
func getAnthrovePostReferences(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string) ([]models.PostReference, error) {
func getAnthrovePostReferences(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string, postLinks []string) ([]models.PostReference, error) {
ctx, span := tracer.Start(ctx, "getAnthrovePostReferences")
defer span.End()
log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAnthrovePostReferences")
var existingPosts []models.PostReference
err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND source_post_id IN ?", id, postIDs).Error
err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND (source_post_id IN ? OR url IN ?)", id, postIDs, postLinks).Error
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())