Anthrove/plug-sdk
5
0
Fork 0
Code Issues Pull Requests Actions Projects Releases 10 Activity

feat: Add source URLs to batch post processing and update database query
All checks were successful
Gitea Build Check / Build (pull_request) Successful in 45s
Details
Gitea Build Check / Build (push) Successful in 42s
Details

Browse Source
This commit is contained in:
SoXX 2024-11-08 15:50:21 +01:00
parent 0919b0bb15
commit a686b5956f
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pkg/plug/scrape.go
View File

@ -48,13 +48,15 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS
} }
postIDs := make([]string, 0, len(posts)) postIDs := make([]string, 0, len(posts))
sourceUrls := make([]string, 0, len(posts))
for _, post := range posts { for _, post := range posts {
postIDs = append(postIDs, post.References[0].SourcePostID) postIDs = append(postIDs, post.References[0].SourcePostID)
sourceUrls = append(sourceUrls, post.References[0].URL)
} }
span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs)))) span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs))))
log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("post_count", len(postIDs)).Info("Collected post IDs") log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("post_count", len(postIDs)).Info("Collected post IDs")
existingPostsReferences, err := getAnthrovePostReferences(ctx, db, userSource.SourceID, postIDs) existingPostsReferences, err := getAnthrovePostReferences(ctx, db, userSource.SourceID, postIDs, sourceUrls)
if err != nil { if err != nil {
span.RecordError(err) span.RecordError(err)
span.SetStatus(codes.Error, err.Error()) span.SetStatus(codes.Error, err.Error())
@ -111,14 +113,15 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS
return newPosts, anthroveFaves, nil return newPosts, anthroveFaves, nil
} }
func getAnthrovePostReferences(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string) ([]models.PostReference, error) { func getAnthrovePostReferences(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string, postLinks []string) ([]models.PostReference, error) {
ctx, span := tracer.Start(ctx, "getAnthrovePostReferences") ctx, span := tracer.Start(ctx, "getAnthrovePostReferences")
defer span.End() defer span.End()
log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAnthrovePostReferences") log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAnthrovePostReferences")
var existingPosts []models.PostReference var existingPosts []models.PostReference
err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND source_post_id IN ?", id, postIDs).Error
err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND (source_post_id IN ? OR url IN ?)", id, postIDs, postLinks).Error
if err != nil { if err != nil {
span.RecordError(err) span.RecordError(err)
span.SetStatus(codes.Error, err.Error()) span.SetStatus(codes.Error, err.Error())