Anthrove/plug-sdk
5
0
Fork 0
Code Issues Pull Requests Actions Projects Releases 10 Activity

New and generic scrape algorithm #11

Merged
SoXX merged 20 commits from dev/algorithm into main 2024-11-01 21:39:21 +00:00
Conversation 0 Commits 20 Files Changed 8 +11 -11
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 107a317095 - Show all commits

22
pkg/plug/algorithm.go
View File

@ -10,15 +10,15 @@ import (
)
type User struct {
userFavoriteCount int64
userName string
userID string
UserFavoriteCount int64
UserName string
UserID string
}
type Favorites struct {
posts []models.Post
nextPage string
lastPage string
Posts []models.Post
NextPage string
LastPage string
}
type Plug interface {
@ -70,7 +70,7 @@ func Algorithm(ctx context.Context, plugInterface Plug, db *gorm.DB, userSource
nextPage := ""
outer:
for anthroveUserFavCount < profile.userFavoriteCount {
for anthroveUserFavCount < profile.UserFavoriteCount {
select {
case <-ctx.Done():
break outer
@ -87,13 +87,13 @@ outer:
return taskSummery, err
}
if len(favorites.posts) <= 0 {
if len(favorites.Posts) <= 0 {
span.AddEvent("No more favorites found")
log.WithContext(ctx).WithFields(basicLoggingInfo).Info("No more favorites found")
break outer
}
summery, err := BatchPostProcessingWithSummery(ctx, userSource, favorites.posts)
summery, err := BatchPostProcessingWithSummery(ctx, userSource, favorites.Posts)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@ -101,12 +101,12 @@ outer:
return taskSummery, err
}
if summery.AddedFavorites != int64(len(favorites.posts)) {
if summery.AddedFavorites != int64(len(favorites.Posts)) {
span.AddEvent("user has no more favorites to add")
break outer
}
nextPage = favorites.nextPage
nextPage = favorites.NextPage
taskSummery.AddedPosts += int(summery.AddedFavorites)
}
}