Anthrove/plug-sdk
4
0
Fork 0
Code Issues Pull Requests Actions Projects Releases 6 Activity

feat(database): Update SDK to v3 and refactor database

Browse Source
This commit is contained in:
SoXX 2024-08-14 16:06:53 +02:00
parent 2f10da2673
commit 9193a20bd6
5 changed files with 38 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -7,14 +7,14 @@ Anthrove Plug SDK is a Golang-based Software Development Kit (SDK) that provides
To install the Anthrove Plug SDK, you will need to have Go installed on your system. You can then use the go get command to fetch the SDK: To install the Anthrove Plug SDK, you will need to have Go installed on your system. You can then use the go get command to fetch the SDK:
```bash ```bash
go get git.anthrove.art/anthrove/plug-sdk/v2 go get git.anthrove.art/Anthrove/plug-sdk/v3
``` ```
## Usage ## Usage
Below is a basic example of how to use the SDK: Below is a basic example of how to use the SDK:
````go ````go
import "git.anthrove.art/anthrove/plug-sdk/v2/pkg/plug" import "git.anthrove.art/Anthrove/plug-sdk/v3/pkg/plug"
// Define what Source this Plug is used for // Define what Source this Plug is used for
source := models.Source{ source := models.Source{

14
go.mod
View File

@ -1,9 +1,9 @@
module git.anthrove.art/anthrove/plug-sdk/v2 module git.anthrove.art/Anthrove/plug-sdk/v3
go 1.22.0 go 1.22.0
require ( require (
git.anthrove.art/anthrove/otter-space-sdk/v2 v2.1.0 git.anthrove.art/Anthrove/otter-space-sdk/v3 v3.0.0
github.com/golang/protobuf v1.5.4 github.com/golang/protobuf v1.5.4
github.com/matoous/go-nanoid/v2 v2.1.0 github.com/matoous/go-nanoid/v2 v2.1.0
google.golang.org/grpc v1.61.1 google.golang.org/grpc v1.61.1
@ -19,14 +19,14 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect github.com/jinzhu/now v1.1.5 // indirect
github.com/lib/pq v1.10.9 // indirect github.com/lib/pq v1.10.9 // indirect
github.com/rubenv/sql-migrate v1.6.1 // indirect github.com/rubenv/sql-migrate v1.7.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect github.com/sirupsen/logrus v1.9.3 // indirect
golang.org/x/crypto v0.22.0 // indirect golang.org/x/crypto v0.22.0 // indirect
golang.org/x/net v0.21.0 // indirect golang.org/x/net v0.21.0 // indirect
golang.org/x/sync v0.7.0 // indirect golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.19.0 // indirect golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.14.0 // indirect golang.org/x/text v0.17.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
gorm.io/driver/postgres v1.5.9 // indirect gorm.io/driver/postgres v1.5.9 // indirect
gorm.io/gorm v1.25.10 // indirect gorm.io/gorm v1.25.11 // indirect
) )

18
pkg/otter/connect.go
View File

@ -2,19 +2,11 @@ package otter
import ( import (
"context" "context"
"git.anthrove.art/anthrove/otter-space-sdk/v2/pkg/database"
"git.anthrove.art/anthrove/otter-space-sdk/v2/pkg/models" "git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/database"
"git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/models"
) )
func ConnectToDatabase(ctx context.Context, config models.DatabaseConfig) (database.OtterSpace, error) { func ConnectToDatabase(ctx context.Context, config models.DatabaseConfig) error {
var otterSpace database.OtterSpace return database.Connect(ctx, config)
var err error
otterSpace = database.NewPostgresqlConnection()
err = otterSpace.Connect(ctx, config)
if err != nil {
return nil, err
}
return otterSpace, err
} }

15
pkg/plug/grpc.go
View File

@ -4,27 +4,24 @@ import (
"context" "context"
"log" "log"
"git.anthrove.art/anthrove/otter-space-sdk/v2/pkg/database" "git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/models"
"git.anthrove.art/anthrove/otter-space-sdk/v2/pkg/models"
"google.golang.org/protobuf/types/known/timestamppb" "google.golang.org/protobuf/types/known/timestamppb"
gRPC "git.anthrove.art/anthrove/plug-sdk/v2/pkg/grpc" gRPC "git.anthrove.art/Anthrove/plug-sdk/v3/pkg/grpc"
gonanoid "github.com/matoous/go-nanoid/v2" gonanoid "github.com/matoous/go-nanoid/v2"
) )
type server struct { type server struct {
gRPC.UnimplementedPlugConnectorServer gRPC.UnimplementedPlugConnectorServer
ctx map[string]context.CancelFunc ctx map[string]context.CancelFunc
database database.OtterSpace
taskExecutionFunction TaskExecution taskExecutionFunction TaskExecution
sendMessageExecution SendMessageExecution sendMessageExecution SendMessageExecution
getMessageExecution GetMessageExecution getMessageExecution GetMessageExecution
} }
func NewGrpcServer(database database.OtterSpace, taskExecutionFunction TaskExecution, sendMessageExecution SendMessageExecution, getMessageExecution GetMessageExecution) gRPC.PlugConnectorServer { func NewGrpcServer(taskExecutionFunction TaskExecution, sendMessageExecution SendMessageExecution, getMessageExecution GetMessageExecution) gRPC.PlugConnectorServer {
return &server{ return &server{
ctx: make(map[string]context.CancelFunc), ctx: make(map[string]context.CancelFunc),
database: database,
taskExecutionFunction: taskExecutionFunction, taskExecutionFunction: taskExecutionFunction,
sendMessageExecution: sendMessageExecution, sendMessageExecution: sendMessageExecution,
getMessageExecution: getMessageExecution, getMessageExecution: getMessageExecution,
@ -32,7 +29,7 @@ func NewGrpcServer(database database.OtterSpace, taskExecutionFunction TaskExecu
} }
func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation) (*gRPC.PlugTaskStatus, error) { func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation) (*gRPC.PlugTaskStatus, error) {
var anthroveUser models.User var user models.User
var plugTaskState gRPC.PlugTaskStatus var plugTaskState gRPC.PlugTaskStatus
var err error var err error
@ -44,7 +41,7 @@ func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation)
plugTaskState.TaskId = id plugTaskState.TaskId = id
plugTaskState.TaskState = gRPC.PlugTaskState_RUNNING plugTaskState.TaskState = gRPC.PlugTaskState_RUNNING
anthroveUser.ID = models.AnthroveUserID(creation.UserId) user.ID = models.UserID(creation.UserId)
// gRPC closes the context after the call ended. So the whole scrapping stopped without waiting // gRPC closes the context after the call ended. So the whole scrapping stopped without waiting
// by using this method we assign a new context to each new request we get. // by using this method we assign a new context to each new request we get.
@ -54,7 +51,7 @@ func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation)
go func() { go func() {
// FIXME: better implement this methode, works for now but needs refactoring // FIXME: better implement this methode, works for now but needs refactoring
err := s.taskExecutionFunction(ctx, s.database, creation.UserSourceName, anthroveUser, creation.DeepScrape, creation.ApiKey, func() { err := s.taskExecutionFunction(ctx, creation.UserSourceName, user, creation.DeepScrape, creation.ApiKey, func() {
s.removeTask(id) s.removeTask(id)
}) })
if err != nil { if err != nil {

34
pkg/plug/server.go
View File

@ -2,17 +2,17 @@ package plug
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"log" "log"
"net" "net"
"git.anthrove.art/anthrove/otter-space-sdk/v2/pkg/database" "git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/database"
otterError "git.anthrove.art/anthrove/otter-space-sdk/v2/pkg/error" otterError "git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/error"
"git.anthrove.art/anthrove/otter-space-sdk/v2/pkg/models" "git.anthrove.art/Anthrove/otter-space-sdk/v3/pkg/models"
"github.com/golang/protobuf/ptypes/timestamp" "github.com/golang/protobuf/ptypes/timestamp"
pb "git.anthrove.art/anthrove/plug-sdk/v2/pkg/grpc" pb "git.anthrove.art/Anthrove/plug-sdk/v3/pkg/grpc"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
@ -22,7 +22,7 @@ type Message struct {
CreatedAt *timestamp.Timestamp CreatedAt *timestamp.Timestamp
} }
type TaskExecution func(ctx context.Context, database database.OtterSpace, userSourceUsername string, anthroveUser models.User, deepScrape bool, apiKey string, cancelFunction func()) error type TaskExecution func(ctx context.Context, userSourceUsername string, anthroveUser models.User, deepScrape bool, apiKey string, cancelFunction func()) error
type SendMessageExecution func(ctx context.Context, userSourceID string, userSourceUsername string, message string) error type SendMessageExecution func(ctx context.Context, userSourceID string, userSourceUsername string, message string) error
type GetMessageExecution func(ctx context.Context, userSourceID string, userSourceUsername string) ([]Message, error) type GetMessageExecution func(ctx context.Context, userSourceID string, userSourceUsername string) ([]Message, error)
@ -30,7 +30,6 @@ type Plug struct {
address string address string
port string port string
ctx context.Context ctx context.Context
database database.OtterSpace
taskExecutionFunction TaskExecution taskExecutionFunction TaskExecution
sendMessageExecution SendMessageExecution sendMessageExecution SendMessageExecution
getMessageExecution GetMessageExecution getMessageExecution GetMessageExecution
@ -49,14 +48,21 @@ func NewPlug(ctx context.Context, address string, port string, source models.Sou
func (p *Plug) Listen() error { func (p *Plug) Listen() error {
var err error var err error
log.Printf("initilazing source!") log.Print("Check if source exists")
err = p.database.CreateSource(p.ctx, &p.source) _, err = database.GetSourceByDomain(p.ctx, p.source.Domain)
if err != nil { if err != nil {
if !errors.Is(err, &otterError.NoDataWritten{}) { if err.Error() == otterError.NoDataFound {
log.Panic(err) log.Printf("iIitializing source!")
if _, err = database.CreateSource(p.ctx, p.source); err != nil {
panic(err)
}
} else {
panic(err)
} }
} }
log.Print("Source exists")
lis, err := net.Listen("tcp", fmt.Sprintf("%s:%s", p.address, p.port)) lis, err := net.Listen("tcp", fmt.Sprintf("%s:%s", p.address, p.port))
if err != nil { if err != nil {
return err return err
@ -64,7 +70,7 @@ func (p *Plug) Listen() error {
grpcServer := grpc.NewServer() grpcServer := grpc.NewServer()
pb.RegisterPlugConnectorServer(grpcServer, NewGrpcServer(p.database, p.taskExecutionFunction, p.sendMessageExecution, p.getMessageExecution)) pb.RegisterPlugConnectorServer(grpcServer, NewGrpcServer(p.taskExecutionFunction, p.sendMessageExecution, p.getMessageExecution))
err = grpcServer.Serve(lis) err = grpcServer.Serve(lis)
if err != nil { if err != nil {
@ -74,10 +80,6 @@ func (p *Plug) Listen() error {
return nil return nil
} }
func (p *Plug) WithOtterSpace(graph database.OtterSpace) {
p.database = graph
}
func (p *Plug) TaskExecutionFunction(function TaskExecution) { func (p *Plug) TaskExecutionFunction(function TaskExecution) {
p.taskExecutionFunction = function p.taskExecutionFunction = function
} }