diff --git a/pkg/plug/scrape.go b/pkg/plug/scrape.go index fa09f37..e743132 100644 --- a/pkg/plug/scrape.go +++ b/pkg/plug/scrape.go @@ -48,13 +48,15 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS } postIDs := make([]string, 0, len(posts)) + sourceUrls := make([]string, 0, len(posts)) for _, post := range posts { postIDs = append(postIDs, post.References[0].SourcePostID) + sourceUrls = append(sourceUrls, post.References[0].URL) } span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs)))) log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("post_count", len(postIDs)).Info("Collected post IDs") - existingPostsReferences, err := getAnthrovePostReferences(ctx, db, userSource.SourceID, postIDs) + existingPostsReferences, err := getAnthrovePostReferences(ctx, db, userSource.SourceID, postIDs, sourceUrls) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -111,14 +113,15 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS return newPosts, anthroveFaves, nil } -func getAnthrovePostReferences(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string) ([]models.PostReference, error) { +func getAnthrovePostReferences(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string, postLinks []string) ([]models.PostReference, error) { ctx, span := tracer.Start(ctx, "getAnthrovePostReferences") defer span.End() log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAnthrovePostReferences") var existingPosts []models.PostReference - err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND source_post_id IN ?", id, postIDs).Error + + err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND (source_post_id IN ? OR url IN ?)", id, postIDs, postLinks).Error if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error())