feat: Add source URLs to batch post processing and update database query

chore(dependencies): update to latest otter-space-sdk
fix: variable userSource.UserID to userSource.AccountID for clarity
2024-11-08 15:50:21 +01:00 · 2024-11-06 13:39:30 +01:00 · 2024-11-01 22:34:43 +01:00 · 2024-11-01 22:29:09 +01:00 · 2024-11-01 22:15:43 +01:00 · 2024-11-01 22:14:29 +01:00
10 changed files with 535 additions and 223 deletions
--- a/README.md
+++ b/README.md
@ -7,7 +7,7 @@ Anthrove Plug SDK is a Golang-based Software Development Kit (SDK) that provides
 To install the Anthrove Plug SDK, you will need to have Go installed on your system. You can then use the go get command to fetch the SDK:

 ```bash
-go get git.anthrove.art/Anthrove/plug-sdk/v3
+go get git.anthrove.art/Anthrove/plug-sdk/v5
 ```
 ## Usage

@ -141,7 +141,7 @@ func main() {
 		log.Fatal(err)
 	}

-	plug.SetTaskExecutionFunction(service.YourTaskFunction)
+	plug.SetTaskExecutionFunction(service.plugInterface)
 	plug.SetGetMessageExecutionFunction(service.YourMessageFunction)
 	err = plug.Listen(ctx, ":8080", source)
 	if err != nil {
--- a/go.mod
+++ b/go.mod
@ -1,21 +1,21 @@
-module git.anthrove.art/Anthrove/plug-sdk/v3
+module git.anthrove.art/Anthrove/plug-sdk/v5

-go 1.22.0
+go 1.23.0

 require (
-	git.anthrove.art/Anthrove/otter-space-sdk/v4 v4.0.0
+	git.anthrove.art/Anthrove/otter-space-sdk/v5 v5.0.0
 	github.com/matoous/go-nanoid/v2 v2.1.0
 	github.com/sirupsen/logrus v1.9.3
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0
-	go.opentelemetry.io/otel v1.29.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0
-	go.opentelemetry.io/otel/exporters/prometheus v0.51.0
-	go.opentelemetry.io/otel/sdk v1.29.0
-	go.opentelemetry.io/otel/sdk/metric v1.29.0
-	go.opentelemetry.io/otel/trace v1.29.0
-	google.golang.org/grpc v1.65.0
-	google.golang.org/protobuf v1.34.2
-	gorm.io/gorm v1.25.11
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0
+	go.opentelemetry.io/otel v1.31.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.53.0
+	go.opentelemetry.io/otel/sdk v1.31.0
+	go.opentelemetry.io/otel/sdk/metric v1.31.0
+	go.opentelemetry.io/otel/trace v1.31.0
+	google.golang.org/grpc v1.67.1
+	google.golang.org/protobuf v1.35.1
+	gorm.io/gorm v1.25.12
 )

 require (
@ -28,28 +28,28 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgx/v5 v5.5.5 // indirect
-	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/pgx/v5 v5.7.1 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/prometheus/client_golang v1.20.1 // indirect
+	github.com/prometheus/client_golang v1.20.5 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/common v0.60.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rubenv/sql-migrate v1.7.0 // indirect
-	go.opentelemetry.io/contrib/bridges/otellogrus v0.3.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 // indirect
-	go.opentelemetry.io/otel/log v0.4.0 // indirect
-	go.opentelemetry.io/otel/metric v1.29.0 // indirect
+	go.opentelemetry.io/contrib/bridges/otellogrus v0.5.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect
+	go.opentelemetry.io/otel/log v0.7.0 // indirect
+	go.opentelemetry.io/otel/metric v1.31.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
-	golang.org/x/crypto v0.26.0 // indirect
-	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.24.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect
 	gorm.io/driver/postgres v1.5.9 // indirect
 )
--- a/go.sum
+++ b/go.sum
@ -1,34 +1,30 @@
-dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
-dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
-git.anthrove.art/Anthrove/otter-space-sdk/v4 v4.0.0 h1:LYkdMoRhidEzZGwIupB5K6u6+9m20oJSszqsFsjvaeA=
-git.anthrove.art/Anthrove/otter-space-sdk/v4 v4.0.0/go.mod h1:TckER0W5rc8PwV3wF+LChcsP1fKy4SD/6AQOMt0W7H4=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
+dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+git.anthrove.art/Anthrove/otter-space-sdk/v5 v5.0.0 h1:3AjgosR7aC+vFGhVDa5LUnLTfvG07j2qniBV1meSE8I=
+git.anthrove.art/Anthrove/otter-space-sdk/v5 v5.0.0/go.mod h1:rb2bCKLAqYDB/l+yw/CIxAktJtd7FmWmKbhJ0v6HqEY=
+github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
+github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38=
-github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao=
-github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4=
-github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM=
-github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
-github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E=
-github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
+github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
+github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
+github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
+github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/docker v27.0.3+incompatible h1:aBGI9TeQ4MPlhquTQKq9XbK79rKFVwXNUAYz9aXyEBE=
-github.com/docker/docker v27.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI=
+github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@ -42,8 +38,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
-github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
+github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@ -56,18 +52,18 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjw
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
-github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
-github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
-github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
-github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
-github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
+github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@ -76,8 +72,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
-github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0=
+github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE=
@ -88,10 +84,12 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
 github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
-github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
-github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
-github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
-github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
+github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
+github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo=
+github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
+github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
+github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
@ -106,24 +104,24 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
-github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY=
 github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg=
-github.com/prometheus/client_golang v1.20.1 h1:IMJXHOD6eARkQpxo8KkhgEVFlBNm+nkrFUyGlIu7Na8=
-github.com/prometheus/client_golang v1.20.1/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
-github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/common v0.60.0 h1:+V9PAREWNvJMAuJ1x1BaWl9dewMW4YrHZQbx0sJNllA=
+github.com/prometheus/common v0.60.0/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
-github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI=
 github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE=
-github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=
-github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM=
+github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
+github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
@ -133,63 +131,63 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/testcontainers/testcontainers-go v0.32.0 h1:ug1aK08L3gCHdhknlTTwWjPHPS+/alvLJU/DRxTD/ME=
-github.com/testcontainers/testcontainers-go v0.32.0/go.mod h1:CRHrzHLQhlXUsa5gXjTOfqIEJcrK5+xMDmBr/WMI88E=
-github.com/testcontainers/testcontainers-go/modules/postgres v0.32.0 h1:ZE4dTdswj3P0j71nL+pL0m2e5HTXJwPoIFr+DDgdPaU=
-github.com/testcontainers/testcontainers-go/modules/postgres v0.32.0/go.mod h1:njrNuyuoF2fjhVk6TG/R3Oeu82YwfYkbf5WVTyBXhV4=
-github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
-github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
-github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
-github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
-github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
-github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.opentelemetry.io/contrib/bridges/otellogrus v0.3.0 h1:QHEj9AK6bEiEA9S5OdDUE9KAx4xp6pRkYMnybHDmjZU=
-go.opentelemetry.io/contrib/bridges/otellogrus v0.3.0/go.mod h1:HRlW/1YWrBrbzB6FvHU7jUuz33F74PEvQVBL+b+wUhM=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
-go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=
-go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 h1:dIIDULZJpgdiHz5tXrTgKIMLkus6jEFa7x5SOKcyR7E=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0/go.mod h1:jlRVBe7+Z1wyxFSUs48L6OBQZ5JwH2Hg/Vbl+t9rAgI=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0 h1:nSiV3s7wiCam610XcLbYOmMfJxB9gO4uK3Xgv5gmTgg=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0/go.mod h1:hKn/e/Nmd19/x1gvIHwtOwVWM+VhuITSWip3JUDghj0=
-go.opentelemetry.io/otel/exporters/prometheus v0.51.0 h1:G7uexXb/K3T+T9fNLCCKncweEtNEBMTO+46hKX5EdKw=
-go.opentelemetry.io/otel/exporters/prometheus v0.51.0/go.mod h1:v0mFe5Kk7woIh938mrZBJBmENYquyA0IICrlYm4Y0t4=
-go.opentelemetry.io/otel/log v0.4.0 h1:/vZ+3Utqh18e8TPjuc3ecg284078KWrR8BRz+PQAj3o=
-go.opentelemetry.io/otel/log v0.4.0/go.mod h1:DhGnQvky7pHy82MIRV43iXh3FlKN8UUKftn0KbLOq6I=
-go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=
-go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
-go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo=
-go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok=
-go.opentelemetry.io/otel/sdk/metric v1.29.0 h1:K2CfmJohnRgvZ9UAj2/FhIf/okdWcNdBwe1m8xFXiSY=
-go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ=
-go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
-go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
+github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw=
+github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8=
+github.com/testcontainers/testcontainers-go/modules/postgres v0.33.0 h1:c+Gt+XLJjqFAejgX4hSpnHIpC9eAhvgI/TFWL/PbrFI=
+github.com/testcontainers/testcontainers-go/modules/postgres v0.33.0/go.mod h1:I4DazHBoWDyf69ByOIyt3OdNjefiUx372459txOpQ3o=
+github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
+github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
+github.com/tklauser/numcpus v0.9.0 h1:lmyCHtANi8aRUgkckBgoDk1nHCux3n2cgkJLXdQGPDo=
+github.com/tklauser/numcpus v0.9.0/go.mod h1:SN6Nq1O3VychhC1npsWostA+oW+VOQTxZrS604NSRyI=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.opentelemetry.io/contrib/bridges/otellogrus v0.5.0 h1:T9cxTlfBz5go7dFWejO5E4JfeglWgWL1MRTPq9p8da0=
+go.opentelemetry.io/contrib/bridges/otellogrus v0.5.0/go.mod h1:f6xZHPJ3A+RFc3Lfz4SoGIpfmVr4PEF4XM1H3fl+6/4=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 h1:yMkBS9yViCc7U7yeLzJPM2XizlfdVvBRSmsQDWu6qc0=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0/go.mod h1:n8MR6/liuGB5EmTETUBeU5ZgqMOlqKRxUaqPQBOANZ8=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI=
+go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
+go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0 h1:FFeLy03iVTXP6ffeN2iXrxfGsZGCjVx0/4KlizjyBwU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0/go.mod h1:TMu73/k1CP8nBUpDLc71Wj/Kf7ZS9FK5b53VapRsP9o=
+go.opentelemetry.io/otel/exporters/prometheus v0.53.0 h1:QXobPHrwiGLM4ufrY3EOmDPJpo2P90UuFau4CDPJA/I=
+go.opentelemetry.io/otel/exporters/prometheus v0.53.0/go.mod h1:WOAXGr3D00CfzmFxtTV1eR0GpoHuPEu+HJT8UWW2SIU=
+go.opentelemetry.io/otel/log v0.7.0 h1:d1abJc0b1QQZADKvfe9JqqrfmPYQCz2tUSO+0XZmuV4=
+go.opentelemetry.io/otel/log v0.7.0/go.mod h1:2jf2z7uVfnzDNknKTO9G+ahcOAyWcp1fJmk/wJjULRo=
+go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
+go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
+go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
+go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
+go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
+go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
+go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
+go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
 go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
 go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
-golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
-golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
-golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
-golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
-golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd h1:BBOTEWLuuEGQy9n1y9MhVJ9Qt0BDu21X8qZs71/uPZo=
-google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd h1:6TEm2ZxXoQmFWFlt1vNxvVOa1Q0dXFQD1m/rYjXmS0E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
-google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
-google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
-google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
-google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg=
+google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
+google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
+google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@ -198,5 +196,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8=
 gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI=
-gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
-gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
+gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
+gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
--- a/pkg/otter/connect.go
+++ b/pkg/otter/connect.go
@ -1,12 +0,0 @@
-package otter
-
-import (
-	"context"
-
-	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/database"
-	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
-)
-
-func ConnectToDatabase(ctx context.Context, config models.DatabaseConfig) error {
-	return database.Connect(ctx, config)
-}
--- a/pkg/plug/algorithm.go
+++ b/pkg/plug/algorithm.go
@ -0,0 +1,160 @@
+package plug
+
+import (
+	"context"
+	"slices"
+	"time"
+
+	"git.anthrove.art/Anthrove/otter-space-sdk/v5/pkg/database"
+	"git.anthrove.art/Anthrove/otter-space-sdk/v5/pkg/models"
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+)
+
+type User struct {
+	UserFavoriteCount int64
+	UserName          string
+	UserID            string
+}
+
+type Favorites struct {
+	Posts    []models.Post
+	NextPage string
+	LastPage string
+}
+
+type Plug interface {
+	// GetFavoritePage
+	// The API Key can be an empty string if it's not supplied by the user, in this case the default API Key should be used
+	GetFavoritePage(ctx context.Context, apiKey string, userSource models.UserSource, pageIdentifier string) (Favorites, error)
+
+	// GetUserProfile
+	// The API Key can be an empty string if it's not supplied by the user, in this case the default API Key should be used
+	GetUserProfile(ctx context.Context, apiKey string, userSource models.UserSource) (User, error)
+}
+
+func algorithm(ctx context.Context, plugInterface Plug, userSource models.UserSource, anthroveUserFavCount int64, deepScrape bool, apiKey string) (TaskSummery, error) {
+	ctx, span := tracer.Start(ctx, "mainScrapeAlgorithm")
+	defer span.End()
+
+	span.SetAttributes(
+		attribute.String("user_source_id", string(userSource.ID)),
+		attribute.String("user_source_user_id", string(userSource.UserID)),
+		attribute.String("user_source_source_id", string(userSource.SourceID)),
+	)
+
+	basicLoggingInfo := log.Fields{
+		"user_source_id":        userSource.ID,
+		"user_source_user_id":   userSource.UserID,
+		"user_source_source_id": userSource.SourceID,
+	}
+
+	log.WithContext(ctx).WithFields(basicLoggingInfo).Info("Starting mainScrapeAlgorithm")
+
+	taskSummery := TaskSummery{
+		AddedPosts:   0,
+		DeletedPosts: 0,
+	}
+
+	profile, err := plugInterface.GetUserProfile(ctx, apiKey, userSource)
+	if err != nil {
+		return taskSummery, err
+	}
+
+	userSource.AccountID = profile.UserID
+	userSource.AccountUsername = profile.UserName
+
+	nextPage := ""
+	var newPosts []models.Post
+	var anthroveFaves []models.UserFavorite
+
+outer:
+	for {
+		if anthroveUserFavCount >= profile.UserFavoriteCount && profile.UserFavoriteCount > 0 {
+			break outer
+		}
+
+		select {
+		case <-ctx.Done():
+			break outer
+		default:
+			span.AddEvent("Executing getFavorites request")
+			favorites, err := plugInterface.GetFavoritePage(ctx, apiKey, userSource, nextPage)
+			span.AddEvent("Finished executing getFavorites request")
+			if err != nil {
+				span.RecordError(err)
+				span.SetStatus(codes.Error, err.Error())
+				log.WithContext(ctx).WithFields(basicLoggingInfo).WithError(err).Error("Failed to execute favorites page")
+				return taskSummery, err
+			}
+
+			if len(favorites.Posts) == 0 {
+				span.AddEvent("No more favorites found")
+				log.WithContext(ctx).WithFields(basicLoggingInfo).Info("No more favorites found")
+				break outer
+			}
+
+			span.AddEvent("Executing BatchPostProcessingWithSummery")
+			pageNewPosts, pageAnthroveFaves, err := BatchPostProcessingWithSummery(ctx, userSource, favorites.Posts)
+
+			anthroveFaves = append(anthroveFaves, pageAnthroveFaves...)
+			newPosts = append(newPosts, pageNewPosts...)
+
+			span.AddEvent("Finished executing BatchPostProcessingWithSummery")
+			if err != nil {
+				span.RecordError(err)
+				span.SetStatus(codes.Error, err.Error())
+				log.WithContext(ctx).WithFields(basicLoggingInfo).WithError(err).Error("Failed in BatchPostProcessing")
+				return taskSummery, err
+			}
+
+			if len(pageAnthroveFaves) <= 0 || len(favorites.Posts) != len(pageAnthroveFaves) {
+				span.AddEvent("No more favorites found to add")
+				log.WithContext(ctx).WithFields(basicLoggingInfo).Info("No more favorites found")
+				break outer
+			}
+
+			nextPage = favorites.NextPage
+			taskSummery.AddedPosts += len(pageAnthroveFaves)
+		}
+	}
+
+	if len(newPosts) > 0 {
+		span.AddEvent("Executing CreatePostInBatch")
+		err = database.CreatePostInBatch(ctx, newPosts, BatchSize)
+		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, err.Error())
+			log.WithContext(ctx).WithError(err).Error("Failed to create new posts in batch")
+			return taskSummery, err
+		}
+		span.AddEvent("Created new posts in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize)))
+		log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created new posts in batch")
+	}
+
+	if len(anthroveFaves) > 0 {
+		span.AddEvent("Executing CreateUserFavoriteInBatch")
+
+		for i, fav := range anthroveFaves {
+			fav.CreatedAt = time.Now().Add(time.Millisecond * time.Duration(i) * -1)
+		}
+
+		slices.Reverse(anthroveFaves)
+
+		err = database.CreateUserFavoriteInBatch(ctx, anthroveFaves, BatchSize)
+		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, err.Error())
+			log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to create user favorites in batch")
+			return taskSummery, err
+		}
+		span.AddEvent("Created user favorites in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize)))
+		log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created user favorites in batch")
+	}
+
+	span.AddEvent("Completed scraping algorithm")
+	log.WithContext(ctx).WithFields(basicLoggingInfo).Info("Completed scraping algorithm")
+	return taskSummery, nil
+}
--- a/pkg/plug/grpc.go
+++ b/pkg/plug/grpc.go
@ -3,10 +3,13 @@ package plug
 import (
 	"context"
 	"errors"
+	"time"

-	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/database"
-	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
-	gRPC "git.anthrove.art/Anthrove/plug-sdk/v3/pkg/grpc"
+	"gorm.io/gorm"
+
+	"git.anthrove.art/Anthrove/otter-space-sdk/v5/pkg/database"
+	"git.anthrove.art/Anthrove/otter-space-sdk/v5/pkg/models"
+	gRPC "git.anthrove.art/Anthrove/plug-sdk/v5/pkg/grpc"
 	gonanoid "github.com/matoous/go-nanoid/v2"
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/attribute"
@ -16,20 +19,20 @@ import (

 type server struct {
 	gRPC.UnimplementedPlugConnectorServer
-	ctx                   map[string]context.CancelFunc
-	taskExecutionFunction TaskExecution
-	sendMessageExecution  SendMessageExecution
-	getMessageExecution   GetMessageExecution
-	source                models.Source
+	ctx                  map[string]context.CancelFunc
+	plugInterface        Plug
+	sendMessageExecution SendMessageExecution
+	getMessageExecution  GetMessageExecution
+	source               models.Source
 }

-func NewGrpcServer(source models.Source, taskExecutionFunction TaskExecution, sendMessageExecution SendMessageExecution, getMessageExecution GetMessageExecution) gRPC.PlugConnectorServer {
+func NewGrpcServer(source models.Source, plugAPIInterface Plug, sendMessageExecution SendMessageExecution, getMessageExecution GetMessageExecution) gRPC.PlugConnectorServer {
 	return &server{
-		ctx:                   make(map[string]context.CancelFunc),
-		taskExecutionFunction: taskExecutionFunction,
-		sendMessageExecution:  sendMessageExecution,
-		getMessageExecution:   getMessageExecution,
-		source:                source,
+		ctx:                  make(map[string]context.CancelFunc),
+		plugInterface:        plugAPIInterface,
+		sendMessageExecution: sendMessageExecution,
+		getMessageExecution:  getMessageExecution,
+		source:               source,
 	}
 }

@ -39,7 +42,7 @@ func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation)

 	var plugTaskState gRPC.PlugTaskStatus

-	id, err := gonanoid.New()
+	id, err := gonanoid.New(25)
 	if err != nil {
 		span.RecordError(err)
 		span.SetStatus(codes.Error, err.Error())
@ -47,6 +50,21 @@ func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation)
 	}
 	span.AddEvent("Generated task ID", trace.WithAttributes(attribute.String("task_id", id)))

+	scrapeTaskHistory := models.ScrapeHistory{
+		ScrapeTaskID: models.ScrapeTaskID(id),
+		UserSourceID: models.UserSourceID(creation.UserSourceId),
+	}
+	scrapeTaskHistory, err = database.CreateScrapeHistory(ctx, scrapeTaskHistory)
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+		return nil, err
+	}
+	span.AddEvent("Creates ScrapeTaskHistory", trace.WithAttributes(
+		attribute.String("user_source_id", creation.UserSourceId),
+		attribute.String("scrape_task_id", id),
+	))
+
 	plugTaskState.TaskId = id
 	plugTaskState.TaskState = gRPC.PlugTaskState_RUNNING

@ -54,7 +72,15 @@ func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation)
 	if err != nil {
 		span.RecordError(err)
 		span.SetStatus(codes.Error, err.Error())
-		return nil, err
+
+		dberr := database.UpdateScrapeHistory(ctx, models.ScrapeHistory{
+			ScrapeTaskID: models.ScrapeTaskID(id),
+			UserSourceID: userSource.ID,
+			FinishedAt:   time.Now(),
+			Error:        err.Error(),
+		})
+
+		return nil, errors.Join(err, dberr)
 	}
 	span.AddEvent("Retrieved user source", trace.WithAttributes(attribute.String("user_source_id", creation.UserSourceId)))

@ -67,7 +93,14 @@ func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation)

 		plugTaskState.TaskState = gRPC.PlugTaskState_STOPPED

-		return &plugTaskState, err
+		dberr := database.UpdateScrapeHistory(ctx, models.ScrapeHistory{
+			ScrapeTaskID: models.ScrapeTaskID(id),
+			UserSourceID: userSource.ID,
+			FinishedAt:   time.Now(),
+			Error:        err.Error(),
+		})
+
+		return &plugTaskState, errors.Join(err, dberr)
 	}

 	// gRPC closes the context after the call ended. So the whole scrapping stopped without waiting
@ -81,20 +114,64 @@ func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation)
 	log.WithContext(taskCtx).WithFields(log.Fields{
 		"task_id":        id,
 		"user_source_id": creation.UserSourceId,
-	}).Info("Starting task")
+	}).Debug("Starting task")
+
+	db, err := database.GetGorm(taskCtx)
+	if err != nil {
+		log.WithContext(taskCtx).WithError(err).WithField("task_id", id).Error("Task execution failed")
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+		dberr := database.UpdateScrapeHistory(taskCtx, models.ScrapeHistory{
+			ScrapeTaskID: models.ScrapeTaskID(id),
+			UserSourceID: userSource.ID,
+			FinishedAt:   time.Now(),
+			Error:        errorString(err),
+		})
+		return &plugTaskState, errors.Join(err, dberr)
+	}
+
+	anthroveUserFavCount, err := getUserFavoriteCountFromDatabase(taskCtx, db, userSource.UserID, userSource.ID)
+	if err != nil {
+		log.WithContext(taskCtx).WithError(err).WithField("task_id", id).Error("Task execution failed")
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+		dberr := database.UpdateScrapeHistory(taskCtx, models.ScrapeHistory{
+			ScrapeTaskID: models.ScrapeTaskID(id),
+			UserSourceID: userSource.ID,
+			FinishedAt:   time.Now(),
+			Error:        errorString(err),
+		})
+		return &plugTaskState, errors.Join(err, dberr)
+	}

 	go func() {
-		err := s.taskExecutionFunction(taskCtx, userSource, creation.DeepScrape, creation.ApiKey, func() {
-			s.removeTask(id)
-		})
+		var err error
+
+		taskSummery, err := algorithm(taskCtx, s.plugInterface, userSource, anthroveUserFavCount, creation.DeepScrape, creation.ApiKey)
 		if err != nil {
 			log.WithContext(taskCtx).WithError(err).WithField("task_id", id).Error("Task execution failed")
 			span.RecordError(err)
 			span.SetStatus(codes.Error, err.Error())
 		} else {
-			log.WithContext(taskCtx).WithField("task_id", id).Info("Task completed successfully")
+			log.WithContext(taskCtx).WithField("task_id", id).Debug("Task completed successfully")
 			span.AddEvent("Task completed successfully", trace.WithAttributes(attribute.String("task_id", id)))
 		}
+
+		err = database.UpdateScrapeHistory(taskCtx, models.ScrapeHistory{
+			ScrapeTaskID: models.ScrapeTaskID(id),
+			UserSourceID: userSource.ID,
+			FinishedAt:   time.Now(),
+			Error:        errorString(err),
+			AddedPosts:   taskSummery.AddedPosts,
+			DeletedPosts: taskSummery.DeletedPosts,
+		})
+
+		if err != nil {
+			log.WithContext(taskCtx).WithError(err).WithField("task_id", id).Error("Task execution failed")
+			span.RecordError(err)
+			span.SetStatus(codes.Error, err.Error())
+		}
+		s.removeTask(id)
 	}()

 	span.SetAttributes(attribute.String("task_id", id))
@ -120,7 +197,7 @@ func (s *server) TaskStatus(ctx context.Context, task *gRPC.PlugTask) (*gRPC.Plu
 	log.WithContext(ctx).WithFields(log.Fields{
 		"task_id":    task.TaskId,
 		"task_state": plugTaskState.TaskState,
-	}).Info("Task status requested")
+	}).Debug("Task status requested")

 	span.SetAttributes(attribute.String("task_id", task.TaskId))
 	return &plugTaskState, nil
@ -141,12 +218,105 @@ func (s *server) TaskCancel(ctx context.Context, task *gRPC.PlugTask) (*gRPC.Plu
 	log.WithContext(ctx).WithFields(log.Fields{
 		"task_id":    task.TaskId,
 		"task_state": plugTaskState.TaskState,
-	}).Info("Task cancellation requested")
+	}).Debug("Task cancellation requested")

 	span.SetAttributes(attribute.String("task_id", task.TaskId))
 	return &plugTaskState, nil
 }

+func (s *server) GetUserMessages(ctx context.Context, message *gRPC.GetMessagesRequest) (*gRPC.GetMessagesResponse, error) {
+	ctx, span := tracer.Start(ctx, "GetUserMessages")
+	defer span.End()
+
+	userSourceID := models.UserSourceID(message.UserSourceId)
+
+	userSource, err := database.GetUserSourceByID(ctx, userSourceID)
+
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+		log.WithContext(ctx).WithError(err).Error("Getting userSource")
+		return nil, err
+	}
+
+	messages, err := s.getMessageExecution(ctx, userSource)
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+		log.WithContext(ctx).WithError(err).Error("Execution function")
+		return nil, err
+	}
+
+	var response gRPC.GetMessagesResponse
+	for _, message := range messages {
+		response.Messages = append(response.Messages, &gRPC.Message{
+			FromUserSourceId: string(userSource.ID),
+			CreatedAt:        message.CreatedAt,
+			Body:             message.Body,
+			Title:            message.Title,
+		})
+	}
+
+	span.SetAttributes(
+		attribute.String("user_source_id", string(userSource.ID)),
+		attribute.String("user_id", string(userSource.UserID)),
+		attribute.String("source_id", string(userSource.SourceID)),
+	)
+
+	fields := log.Fields{
+		"user_source_id": userSource.ID,
+		"user_id":        userSource.UserID,
+		"source_id":      userSource.SourceID,
+		"len_messages":   len(messages),
+	}
+
+	log.WithContext(ctx).WithFields(fields).Debug("Got User messages")
+
+	return &response, err
+
+}
+
+func (s *server) SendMessage(ctx context.Context, message *gRPC.SendMessageRequest) (*gRPC.SendMessageResponse, error) {
+	ctx, span := tracer.Start(ctx, "SendMessage")
+	defer span.End()
+
+	response := &gRPC.SendMessageResponse{
+		Success: false,
+	}
+
+	sourceID := models.UserSourceID(message.UserSourceId)
+	userSource := models.UserSource{BaseModel: models.BaseModel[models.UserSourceID]{ID: sourceID}}
+
+	err := s.sendMessageExecution(ctx, userSource, message.Message)
+	if err != nil {
+		log.WithContext(ctx).WithError(err).Error("Sending message execution")
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+
+		return response, err
+	}
+
+	response.Success = true
+	return response, err
+}
+
+func (s *server) Ping(ctx context.Context, ping *gRPC.PingRequest) (*gRPC.PongResponse, error) {
+	ctx, span := tracer.Start(ctx, "Ping")
+	defer span.End()
+
+	var pong gRPC.PongResponse
+	pong.Message = ping.Message
+	pong.Timestamp = ping.Timestamp
+
+	fields := log.Fields{
+		"messsage":  ping.Message,
+		"timestamp": ping.Timestamp,
+	}
+	log.WithContext(ctx).WithFields(fields).Trace("Got pinged")
+
+	return &pong, nil
+}
+
 func (s *server) removeTask(taskID string) {
 	fn, exists := s.ctx[taskID]
 	if !exists {
@ -155,3 +325,22 @@ func (s *server) removeTask(taskID string) {
 	fn()
 	delete(s.ctx, taskID)
 }
+
+func errorString(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}
+
+// getUserFavoriteCountFromDatabase
+func getUserFavoriteCountFromDatabase(ctx context.Context, gorm *gorm.DB, userID models.UserID, userSourceID models.UserSourceID) (int64, error) {
+	var count int64
+
+	err := gorm.WithContext(ctx).Model(&models.UserFavorite{}).Where("user_id = ? AND user_source_id = ?", userID, userSourceID).Count(&count).Error
+	if err != nil {
+		return count, err
+	}
+
+	return count, nil
+}
--- a/pkg/plug/otlp.go
+++ b/pkg/plug/otlp.go
@ -3,7 +3,7 @@ package plug
 import (
 	"context"

-	"git.anthrove.art/Anthrove/plug-sdk/v3/pkg/telemetry"
+	"git.anthrove.art/Anthrove/plug-sdk/v5/pkg/telemetry"
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/codes"
 )
--- a/pkg/plug/scrape.go
+++ b/pkg/plug/scrape.go
@ -4,8 +4,8 @@ import (
 	"context"
 	"slices"

-	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/database"
-	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
+	"git.anthrove.art/Anthrove/otter-space-sdk/v5/pkg/database"
+	"git.anthrove.art/Anthrove/otter-space-sdk/v5/pkg/models"
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
@ -21,12 +21,7 @@ type BatchSummery struct {
 	AddedFavorites int64
 }

-func BatchPostProcessing(ctx context.Context, userSource models.UserSource, posts []models.Post) error {
-	_, err := BatchPostProcessingWithSummery(ctx, userSource, posts)
-	return err
-}
-
-func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserSource, posts []models.Post) (BatchSummery, error) {
+func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserSource, posts []models.Post) ([]models.Post, []models.UserFavorite, error) {
 	ctx, span := tracer.Start(ctx, "BatchPostProcessing")
 	defer span.End()

@ -49,28 +44,30 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS
 		span.RecordError(err)
 		span.SetStatus(codes.Error, err.Error())
 		log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to get Gorm DB")
-		return BatchSummery{}, err
+		return nil, nil, err
 	}

 	postIDs := make([]string, 0, len(posts))
+	sourceUrls := make([]string, 0, len(posts))
 	for _, post := range posts {
 		postIDs = append(postIDs, post.References[0].SourcePostID)
+		sourceUrls = append(sourceUrls, post.References[0].URL)
 	}
 	span.AddEvent("Collected post IDs", trace.WithAttributes(attribute.Int("post_count", len(postIDs))))
 	log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("post_count", len(postIDs)).Info("Collected post IDs")

-	existingPosts, err := getAnthrovePost(ctx, db, userSource.SourceID, postIDs)
+	existingPostsReferences, err := getAnthrovePostReferences(ctx, db, userSource.SourceID, postIDs, sourceUrls)
 	if err != nil {
 		span.RecordError(err)
 		span.SetStatus(codes.Error, err.Error())
 		log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch existing posts")
-		return BatchSummery{}, err
+		return nil, nil, err
 	}
-	span.AddEvent("Fetched existing posts", trace.WithAttributes(attribute.Int("existing_post_count", len(existingPosts))))
-	log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("existing_post_count", len(existingPosts)).Info("Fetched existing posts")
+	span.AddEvent("Fetched existing posts", trace.WithAttributes(attribute.Int("existing_post_count", len(existingPostsReferences))))
+	log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("existing_post_count", len(existingPostsReferences)).Info("Fetched existing posts")

 	var existingPostIDs []models.PostID
-	for _, post := range existingPosts {
+	for _, post := range existingPostsReferences {
 		existingPostIDs = append(existingPostIDs, models.PostID(post.PostID))
 	}

@ -80,15 +77,15 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS
 		span.RecordError(err)
 		span.SetStatus(codes.Error, err.Error())
 		log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to fetch existing favorite posts")
-		return BatchSummery{}, err
+		return nil, nil, err
 	}
 	span.AddEvent("Fetched existing favorite posts", trace.WithAttributes(attribute.Int("existing_fav_post_count", len(existingFavPostIDs))))
 	log.WithContext(ctx).WithFields(BasicLoggingFields).WithField("existing_fav_post_count", len(existingFavPostIDs)).Info("Fetched existing favorite posts")

-	anthroveFaves := make([]models.UserFavorite, 0, len(existingPosts))
-	newPosts := make([]models.Post, 0, len(existingPosts))
+	anthroveFaves := make([]models.UserFavorite, 0, len(existingPostsReferences))
+	newPosts := make([]models.Post, 0, len(existingPostsReferences))
 	for _, post := range posts {
-		if !slices.ContainsFunc(existingPosts, func(reference models.PostReference) bool {
+		if !slices.ContainsFunc(existingPostsReferences, func(reference models.PostReference) bool {
 			found := reference.SourcePostID == post.References[0].SourcePostID
 			if found {
 				if !slices.Contains(existingFavPostIDs, models.PostID(reference.PostID)) {
@ -105,6 +102,7 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS
 				UserID:       userSource.UserID,
 				PostID:       post.ID,
 				UserSourceID: userSource.ID,
+				UserSource:   models.UserSource{},
 			})
 			newPosts = append(newPosts, post)
 		}
@ -112,44 +110,18 @@ func BatchPostProcessingWithSummery(ctx context.Context, userSource models.UserS
 	span.AddEvent("Processed posts for favorites and new posts", trace.WithAttributes(attribute.Int("new_post_count", len(newPosts)), attribute.Int("new_fav_count", len(anthroveFaves))))
 	log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Processed posts for favorites and new posts")

-	if len(newPosts) > 0 {
-		err = database.CreatePostInBatch(ctx, newPosts, BatchSize)
-		if err != nil {
-			span.RecordError(err)
-			span.SetStatus(codes.Error, err.Error())
-			log.WithContext(ctx).WithError(err).Error("Failed to create new posts in batch")
-			return BatchSummery{}, err
-		}
-		span.AddEvent("Created new posts in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize)))
-		log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created new posts in batch")
-	}
-
-	if len(anthroveFaves) > 0 {
-		err = database.CreateUserFavoriteInBatch(ctx, anthroveFaves, BatchSize)
-		if err != nil {
-			span.RecordError(err)
-			span.SetStatus(codes.Error, err.Error())
-			log.WithContext(ctx).WithError(err).WithFields(BasicLoggingFields).Error("Failed to create user favorites in batch")
-			return BatchSummery{}, err
-		}
-		span.AddEvent("Created user favorites in batch", trace.WithAttributes(attribute.Int("batch_size", BatchSize)))
-		log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Created user favorites in batch")
-	}
-
-	return BatchSummery{
-		AddedPosts:     int64(len(newPosts)),
-		AddedFavorites: int64(len(anthroveFaves)),
-	}, nil
+	return newPosts, anthroveFaves, nil
 }

-func getAnthrovePost(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string) ([]models.PostReference, error) {
-	ctx, span := tracer.Start(ctx, "getAnthrovePost")
+func getAnthrovePostReferences(ctx context.Context, gorm *gorm.DB, id models.SourceID, postIDs []string, postLinks []string) ([]models.PostReference, error) {
+	ctx, span := tracer.Start(ctx, "getAnthrovePostReferences")
 	defer span.End()

-	log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAnthrovePost")
+	log.WithContext(ctx).WithFields(BasicLoggingFields).Info("Starting getAnthrovePostReferences")

 	var existingPosts []models.PostReference
-	err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND source_post_id IN ?", id, postIDs).Error
+
+	err := gorm.WithContext(ctx).Model(models.PostReference{}).Find(&existingPosts, "source_id = ? AND (source_post_id IN ? OR url IN ?)", id, postIDs, postLinks).Error
 	if err != nil {
 		span.RecordError(err)
 		span.SetStatus(codes.Error, err.Error())
--- a/pkg/plug/server.go
+++ b/pkg/plug/server.go
@ -4,8 +4,8 @@ import (
 	"context"
 	"net"

-	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
-	pb "git.anthrove.art/Anthrove/plug-sdk/v3/pkg/grpc"
+	"git.anthrove.art/Anthrove/otter-space-sdk/v5/pkg/models"
+	pb "git.anthrove.art/Anthrove/plug-sdk/v5/pkg/grpc"
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"go.opentelemetry.io/otel"
@ -15,7 +15,7 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 )

-var tracer = otel.Tracer("git.anthrove.art/Anthrove/plug-sdk/v3/pkg/plug")
+var tracer = otel.Tracer("git.anthrove.art/Anthrove/plug-sdk/v5/pkg/plug")

 type Message struct {
 	Title     string
@ -23,14 +23,18 @@ type Message struct {
 	CreatedAt *timestamppb.Timestamp
 }

-type TaskExecution func(ctx context.Context, userSource models.UserSource, deepScrape bool, apiKey string, cancelFunction func()) error
+type TaskSummery struct {
+	AddedPosts   int
+	DeletedPosts int
+}
+
 type SendMessageExecution func(ctx context.Context, userSource models.UserSource, message string) error
 type GetMessageExecution func(ctx context.Context, userSource models.UserSource) ([]Message, error)

 var (
-	taskExecutionFunction TaskExecution
-	sendMessageExecution  SendMessageExecution
-	getMessageExecution   GetMessageExecution
+	sendMessageExecution SendMessageExecution
+	getMessageExecution  GetMessageExecution
+	plugAPIInterface     Plug
 )

 func Listen(ctx context.Context, listenAddr string, source models.Source) error {
@ -73,7 +77,7 @@ func Listen(ctx context.Context, listenAddr string, source models.Source) error
 		grpc.StatsHandler(otelgrpc.NewServerHandler()),
 	)

-	pb.RegisterPlugConnectorServer(grpcServer, NewGrpcServer(source, taskExecutionFunction, sendMessageExecution, getMessageExecution))
+	pb.RegisterPlugConnectorServer(grpcServer, NewGrpcServer(source, plugAPIInterface, sendMessageExecution, getMessageExecution))

 	go func() {
 		err = grpcServer.Serve(lis)
@ -93,8 +97,9 @@ func Listen(ctx context.Context, listenAddr string, source models.Source) error
 	return nil
 }

-func SetTaskExecutionFunction(function TaskExecution) {
-	taskExecutionFunction = function
+// RegisterPlugInterface sets the provided Plug interface implementation for task execution.
+func RegisterPlugInterface(plugInterface Plug) {
+	plugAPIInterface = plugInterface
 }

 func SetSendMessageExecutionFunction(function SendMessageExecution) {
--- a/pkg/plug/source.go
+++ b/pkg/plug/source.go
@ -3,8 +3,8 @@ package plug
 import (
 	"context"

-	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/database"
-	"git.anthrove.art/Anthrove/otter-space-sdk/v4/pkg/models"
+	"git.anthrove.art/Anthrove/otter-space-sdk/v5/pkg/database"
+	"git.anthrove.art/Anthrove/otter-space-sdk/v5/pkg/models"
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/codes"
 )
Author	SHA1	Message	Date
SoXX	a686b5956f	feat: Add source URLs to batch post processing and update database query All checks were successful Gitea Build Check / Build (pull_request) Successful in 45s Details Gitea Build Check / Build (push) Successful in 42s Details	2024-11-08 15:50:21 +01:00
SoXX	0919b0bb15	chore(dependencies): update to latest otter-space-sdk All checks were successful Gitea Build Check / Build (push) Successful in 46s Details Gitea Build Check / Build (pull_request) Successful in 44s Details	2024-11-06 13:39:30 +01:00
SoXX	af0dc2ab0d	fix: variable userSource.UserID to userSource.AccountID for clarity All checks were successful Gitea Build Check / Build (push) Successful in 39s Details Gitea Build Check / Build (pull_request) Successful in 38s Details	2024-11-01 22:34:43 +01:00
SoXX	ffd3767f32	fix: error when userID is empty All checks were successful Gitea Build Check / Build (push) Successful in 37s Details Gitea Build Check / Build (pull_request) Successful in 36s Details - a error could happen if the user dose not have a source UserID/UserName set	2024-11-01 22:29:09 +01:00
SoXX	282e769c13	fix: unused variable warning in post-processing loop All checks were successful Gitea Build Check / Build (push) Successful in 39s Details Gitea Build Check / Build (pull_request) Successful in 38s Details	2024-11-01 22:15:43 +01:00
SoXX	4d209f2a90	fix: favorites creation timestamps and remove duplicate imports Some checks failed Gitea Build Check / Build (push) Failing after 25s Details Gitea Build Check / Build (pull_request) Failing after 24s Details	2024-11-01 22:14:29 +01:00
SoXX	0224a43736	Merge branch 'dev/algorithm' of git.anthrove.art:Anthrove/plug-sdk into dev/algorithm All checks were successful Gitea Build Check / Build (push) Successful in 38s Details Gitea Build Check / Build (pull_request) Successful in 38s Details	2024-11-01 21:43:00 +01:00
SoXX	3da2501459	Fix: variable name typo from anthroveFaves to pageAnthroveFaves	2024-11-01 21:42:16 +01:00
SoXX	0c0d8451c6	fix: Remove redundant break statement in taskSummery block All checks were successful Gitea Build Check / Build (push) Successful in 40s Details Gitea Build Check / Build (pull_request) Successful in 40s Details	2024-11-01 21:29:28 +01:00
SoXX	9d950cf31d	fix: comparison logic in algorithm to correctly break the loop All checks were successful Gitea Build Check / Build (push) Successful in 41s Details Gitea Build Check / Build (pull_request) Successful in 41s Details	2024-11-01 21:15:29 +01:00
SoXX	996ec30de1	fix: Remove unused BatchSummery return values in error cases All checks were successful Gitea Build Check / Build (push) Successful in 44s Details Gitea Build Check / Build (pull_request) Successful in 41s Details	2024-11-01 21:10:13 +01:00
SoXX	799f943650	refactor: improve readability and efficiency Some checks failed Gitea Build Check / Build (push) Failing after 31s Details Gitea Build Check / Build (pull_request) Failing after 27s Details - Remove nested for loop and redundant checks - Streamline error handling and logging - Rename getAnthrovePost to getAnthrovePostReferences for clarity - Simplify BatchPostProcessingWithSummery to return only necessary values	2024-11-01 21:06:03 +01:00
SoXX	16846c59c4	fix: variable scope and append results in BatchPostProcessing All checks were successful Gitea Build Check / Build (push) Successful in 54s Details Gitea Build Check / Build (pull_request) Successful in 56s Details	2024-10-28 21:33:45 +01:00
SoXX	263d65adc1	fix: Fix conditional check for anthroveFaves length All checks were successful Gitea Build Check / Build (push) Successful in 59s Details Gitea Build Check / Build (pull_request) Successful in 56s Details	2024-10-28 21:29:25 +01:00
SoXX	44137a7251	feat: Add tracing events and logging for batch processing steps All checks were successful Gitea Build Check / Build (push) Successful in 58s Details Gitea Build Check / Build (pull_request) Successful in 57s Details	2024-10-28 21:08:54 +01:00
SoXX	b2db0664d6	feat: start implementing new algorithm All checks were successful Gitea Build Check / Build (push) Successful in 1m2s Details Gitea Build Check / Build (pull_request) Successful in 1m4s Details	2024-10-26 23:13:24 +02:00
SoXX	1435ae8ea4	refactor: Rename SetTaskExecutionFunction to RegisterPlugInterface for clarity and add docstring All checks were successful Gitea Build Check / Build (push) Successful in 53s Details Gitea Build Check / Build (pull_request) Successful in 54s Details	2024-10-26 21:31:27 +02:00
SoXX	376d7ba75d	feat: implemented generic algorithm All checks were successful Gitea Build Check / Build (push) Successful in 57s Details Gitea Build Check / Build (pull_request) Successful in 55s Details	2024-10-26 21:15:02 +02:00
SoXX	36d6ecd873	fix: Remove redundant error logging and tracing in TaskStart function All checks were successful Gitea Build Check / Build (push) Successful in 56s Details	2024-10-26 20:36:43 +02:00
SoXX	408c977432	fix: nil pointer for DB All checks were successful Gitea Build Check / Build (push) Successful in 57s Details	2024-10-26 20:30:12 +02:00
SoXX	107a317095	fix: Capitalize struct fields for public access All checks were successful Gitea Build Check / Build (push) Successful in 1m2s Details	2024-10-25 22:46:22 +02:00
SoXX	41882b9bfb	feat: Refactor task execution to use Plug interface and update algorithm implementation All checks were successful Gitea Build Check / Build (push) Successful in 1m3s Details	2024-10-25 22:33:13 +02:00
SoXX	684d52d64c	feat: pumped SDK version to 4.0.0 All checks were successful Gitea Build Check / Build (push) Successful in 58s Details Gitea Build Check / Build (pull_request) Successful in 58s Details	2024-10-16 12:20:21 +02:00
SoXX	a99c4457d2	fix: Handle multiple error returns in TaskStart by joining errors All checks were successful Gitea Build Check / Build (push) Successful in 58s Details Gitea Build Check / Build (pull_request) Successful in 58s Details	2024-10-16 12:12:20 +02:00
SoXX	cc2b0374da	fix: Refactor error handling by introducing errorString function to check for nil errors All checks were successful Gitea Build Check / Build (push) Successful in 59s Details Gitea Build Check / Build (pull_request) Successful in 59s Details	2024-10-16 10:22:35 +02:00
SoXX	2707806a08	feat: Update scrape history on task failure and completion All checks were successful Gitea Build Check / Build (push) Successful in 56s Details	2024-10-16 10:11:00 +02:00
SoXX	22b0edabfe	fix(grpc): force ID length to 25 chars All checks were successful Gitea Build Check / Build (push) Successful in 59s Details	2024-10-16 09:47:48 +02:00
SoXX	9de22d3176	feat: Add ScrapeHistory tracking and refactor task execution return type All checks were successful Gitea Build Check / Build (push) Successful in 58s Details - Add `ScrapeHistory` record creation and update during task execution - Modify `TaskExecution` function to return `TaskSummery` instead of using a cancel function - Reintroduce and move `removeTask` function for cleanup after task execution	2024-10-15 16:08:34 +02:00
SoXX	9132779673	chore(dependencies): update to latest version	2024-10-15 15:43:10 +02:00
SoXX	38cecba455	fix(grpc): spelling All checks were successful Gitea Build Check / Build (push) Successful in 48s Details Gitea Build Check / Build (pull_request) Successful in 50s Details	2024-09-04 14:56:27 +02:00
SoXX	b497348d2a	feat(grpc): add logging All checks were successful Gitea Build Check / Build (push) Successful in 51s Details Gitea Build Check / Build (pull_request) Successful in 47s Details	2024-09-04 14:53:01 +02:00
SoXX	2c8d6bd682	fix(grpc): get necessary information All checks were successful Gitea Build Check / Build (push) Successful in 51s Details Gitea Build Check / Build (pull_request) Successful in 53s Details getting data from database for the userSource	2024-09-04 14:17:00 +02:00
SoXX	601ce9eae7	feat(grpc): added GetUserMessages & SendMessage All checks were successful Gitea Build Check / Build (push) Successful in 50s Details Gitea Build Check / Build (pull_request) Successful in 53s Details finalized first draft for the endpoint integration	2024-09-04 14:01:04 +02:00
SoXX	42954c7cf1	feat(grpc): added ping added ping function	2024-09-04 13:41:29 +02:00