130 lines
4.0 KiB
Go
130 lines
4.0 KiB
Go
package plug
|
|
|
|
import (
|
|
"context"
|
|
"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
|
|
log "github.com/sirupsen/logrus"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
type User struct {
|
|
UserFavoriteCount int64
|
|
UserName string
|
|
UserID string
|
|
}
|
|
|
|
type Favorites struct {
|
|
Posts []models.Post
|
|
NextPage string
|
|
LastPage string
|
|
}
|
|
|
|
type Plug interface {
|
|
// GetFavoritePage
|
|
// The API Key can be an empty string if it's not supplied by the user, in this case the default API Key should be used
|
|
GetFavoritePage(ctx context.Context, apiKey string, userSource models.UserSource, pageIdentifier string) (Favorites, error)
|
|
|
|
// GetUserProfile
|
|
// The API Key can be an empty string if it's not supplied by the user, in this case the default API Key should be used
|
|
GetUserProfile(ctx context.Context, apiKey string, userSource models.UserSource) (User, error)
|
|
}
|
|
|
|
func Algorithm(ctx context.Context, plugInterface Plug, db *gorm.DB, userSource models.UserSource, deepScrape bool, apiKey string) (TaskSummery, error) {
|
|
ctx, span := tracer.Start(ctx, "mainScrapeAlgorithm")
|
|
defer span.End()
|
|
|
|
span.SetAttributes(
|
|
attribute.String("user_source_id", string(userSource.ID)),
|
|
attribute.String("user_source_user_id", string(userSource.UserID)),
|
|
attribute.String("user_source_source_id", string(userSource.SourceID)),
|
|
)
|
|
|
|
basicLoggingInfo := log.Fields{
|
|
"user_source_id": userSource.ID,
|
|
"user_source_user_id": userSource.UserID,
|
|
"user_source_source_id": userSource.SourceID,
|
|
}
|
|
|
|
log.WithContext(ctx).WithFields(basicLoggingInfo).Info("Starting mainScrapeAlgorithm")
|
|
|
|
taskSummery := TaskSummery{
|
|
AddedPosts: 0,
|
|
DeletedPosts: 0,
|
|
}
|
|
|
|
anthroveUserFavCount, err := getUserFavoriteCountFromDatabase(ctx, db, userSource.UserID, userSource.ID)
|
|
if err != nil {
|
|
span.RecordError(err)
|
|
span.SetStatus(codes.Error, err.Error())
|
|
log.WithContext(ctx).WithFields(basicLoggingInfo).WithError(err).Error("Failed to get user favorite count from db")
|
|
return taskSummery, err
|
|
}
|
|
|
|
profile, err := plugInterface.GetUserProfile(ctx, apiKey, userSource)
|
|
if err != nil {
|
|
return taskSummery, err
|
|
}
|
|
|
|
nextPage := ""
|
|
|
|
outer:
|
|
for anthroveUserFavCount < profile.UserFavoriteCount {
|
|
select {
|
|
case <-ctx.Done():
|
|
break outer
|
|
default:
|
|
span.AddEvent("Executing getFavorites request")
|
|
|
|
favorites, err := plugInterface.GetFavoritePage(ctx, apiKey, userSource, nextPage)
|
|
span.AddEvent("Finished executing getFavorites request")
|
|
|
|
if err != nil {
|
|
span.RecordError(err)
|
|
span.SetStatus(codes.Error, err.Error())
|
|
log.WithContext(ctx).WithFields(basicLoggingInfo).WithError(err).Error("Failed to execute favorites page")
|
|
return taskSummery, err
|
|
}
|
|
|
|
if len(favorites.Posts) <= 0 {
|
|
span.AddEvent("No more favorites found")
|
|
log.WithContext(ctx).WithFields(basicLoggingInfo).Info("No more favorites found")
|
|
break outer
|
|
}
|
|
|
|
summery, err := BatchPostProcessingWithSummery(ctx, userSource, favorites.Posts)
|
|
if err != nil {
|
|
span.RecordError(err)
|
|
span.SetStatus(codes.Error, err.Error())
|
|
log.WithContext(ctx).WithFields(basicLoggingInfo).WithError(err).Error("Failed in BatchPostProcessing")
|
|
return taskSummery, err
|
|
}
|
|
|
|
if summery.AddedFavorites != int64(len(favorites.Posts)) {
|
|
span.AddEvent("user has no more favorites to add")
|
|
break outer
|
|
}
|
|
|
|
nextPage = favorites.NextPage
|
|
taskSummery.AddedPosts += int(summery.AddedFavorites)
|
|
}
|
|
}
|
|
|
|
span.AddEvent("Completed scraping algorithm")
|
|
log.WithContext(ctx).WithFields(basicLoggingInfo).Info("Completed scraping algorithm")
|
|
return taskSummery, nil
|
|
}
|
|
|
|
// getUserFavoriteCountFromDatabase
|
|
func getUserFavoriteCountFromDatabase(ctx context.Context, gorm *gorm.DB, userID models.UserID, userSourceID models.UserSourceID) (int64, error) {
|
|
var count int64
|
|
|
|
err := gorm.WithContext(ctx).Model(&models.UserFavorite{}).Where("user_id = ? AND user_source_id = ?", userID, userSourceID).Count(&count).Error
|
|
if err != nil {
|
|
return count, err
|
|
}
|
|
|
|
return count, nil
|
|
}
|