Skip to content

feat: switch to postgres for database tools #533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: switch to postgres for database tools #533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 0 additions & 21 deletions database/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,3 @@
module obot-platform/database

go 1.23.3

require (
github.com/gptscript-ai/go-gptscript v0.9.6-0.20250222170845-eee4337500a6
github.com/ncruces/go-sqlite3 v0.20.3
)

require (
github.com/getkin/kin-openapi v0.129.0 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80 // indirect
github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349 // indirect
github.com/perimeterx/marshmallow v1.1.5 // indirect
github.com/tetratelabs/wazero v1.8.2 // indirect
golang.org/x/sys v0.27.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
51 changes: 0 additions & 51 deletions database/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,51 +0,0 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/getkin/kin-openapi v0.129.0 h1:QGYTNcmyP5X0AtFQ2Dkou9DGBJsUETeLH9rFrJXZh30=
github.com/getkin/kin-openapi v0.129.0/go.mod h1:gmWI+b/J45xqpyK5wJmRRZse5wefA5H0RDMK46kLUtI=
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/gptscript-ai/go-gptscript v0.9.6-0.20250222170845-eee4337500a6 h1:vsZ09cWfNWUXT6AOVQc1GpfEdIxcLusUs6Hgo9IgAKs=
github.com/gptscript-ai/go-gptscript v0.9.6-0.20250222170845-eee4337500a6/go.mod h1:QvGPZoRuAiA8P5EzPI05kTrs+LZ0ipHywUGsKruSknw=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/ncruces/go-sqlite3 v0.20.3 h1:+4G4uEqOeusF0yRuQVUl9fuoEebUolwQSnBUjYBLYIw=
github.com/ncruces/go-sqlite3 v0.20.3/go.mod h1:ojLIAB243gtz68Eo283Ps+k9PyR3dvzS+9/RgId4+AA=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80 h1:nZspmSkneBbtxU9TopEAE0CY+SBJLxO8LPUlw2vG4pU=
github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80/go.mod h1:7tFDb+Y51LcDpn26GccuUgQXUk6t0CXZsivKjyimYX8=
github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349 h1:t05Ww3DxZutOqbMN+7OIuqDwXbhl32HiZGpLy26BAPc=
github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o=
github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s=
github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
150 changes: 29 additions & 121 deletions database/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,89 +2,60 @@ package main

import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"slices"

"obot-platform/database/pkg/cmd"

"github.com/gptscript-ai/go-gptscript"
_ "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"os"
)

var workspaceID = os.Getenv("DATABASE_WORKSPACE_ID")

func main() {
if len(os.Args) != 2 {
fmt.Println("Usage: gptscript-go-tool <command>")
os.Exit(1)
}
command := os.Args[1]
ctx := context.Background()

g, err := gptscript.NewGPTScript()
if err != nil {
fmt.Printf("Error creating GPTScript: %v\n", err)
os.Exit(1)
workspaceID := os.Getenv("DATABASE_WORKSPACE_ID")
if workspaceID == "" {
// TODO(njhale): Figure out why DATABASE_WORKSPACE_ID is not set here for the UI tools.
workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID")
}
defer g.Close()

var (
ctx = context.Background()
dbFileName = "obot.db"
dbWorkspacePath = "/databases/" + dbFileName
revisionID string = "-1"
initialDBData []byte
)

workspaceDB, err := g.ReadFileWithRevisionInWorkspace(ctx, dbWorkspacePath, gptscript.ReadFileInWorkspaceOptions{
WorkspaceID: workspaceID,
})

var notFoundErr *gptscript.NotFoundInWorkspaceError
if err != nil && !errors.As(err, &notFoundErr) {
fmt.Printf("Error reading DB file: %v\n", err)
os.Exit(1)
}
// Get admin DSN from environment variable
adminDSN := os.Getenv("POSTGRES_DSN")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this will need to get plumbed down as a credential from obot when we decide how postgres for this tool is going to be stood up/configured.


// Create a temporary file for the SQLite database
dbFile, err := os.CreateTemp("", dbFileName)
// Setup database and user with admin credentials
dsn, err := cmd.EnsureTenantSchema(ctx, adminDSN, workspaceID)
if err != nil {
fmt.Printf("Error creating temp file: %v\n", err)
fmt.Printf("Error setting up database: %v\n", err)
os.Exit(1)
}
defer dbFile.Close()
defer os.Remove(dbFile.Name())

// Write the data to the temporary file
if workspaceDB != nil && workspaceDB.Content != nil {
initialDBData = workspaceDB.Content
if err := os.WriteFile(dbFile.Name(), initialDBData, 0644); err != nil {
fmt.Printf("Error writing to temp file: %v\n", err)
os.Exit(1)
}
if workspaceDB.RevisionID != "" {
revisionID = workspaceDB.RevisionID
}
}

// Run the requested command
// Run the requested command using the user credentials
var result string
switch command {
case "listDatabaseTables":
result, err = cmd.ListDatabaseTables(ctx, dbFile)
result, err = cmd.ListDatabaseTables(ctx, dsn)

case "listDatabaseTableRows":
result, err = cmd.ListDatabaseTableRows(ctx, dbFile, os.Getenv("TABLE"))
table := os.Getenv("TABLE")
if table == "" {
err = fmt.Errorf("TABLE environment variable is required")
break
}
result, err = cmd.ListDatabaseTableRows(ctx, dsn, table)

case "runDatabaseSQL":
result, err = cmd.RunDatabaseCommand(ctx, dbFile, os.Getenv("SQL"), "-header")
if err == nil {
err = saveWorkspaceDB(ctx, g, dbWorkspacePath, revisionID, dbFile, initialDBData)
sql := os.Getenv("SQL")
if sql == "" {
err = fmt.Errorf("SQL environment variable is required")
break
}
result, err = cmd.RunDatabaseCommand(ctx, dsn, sql)

case "databaseContext":
result, err = cmd.DatabaseContext(ctx, dbFile)
result, err = cmd.DatabaseContext(ctx, dsn)

default:
err = fmt.Errorf("unknown command: %s", command)
}
Expand All @@ -96,66 +67,3 @@ func main() {

fmt.Print(result)
}

// saveWorkspaceDB saves the updated database file to the workspace if the content of the database has changed.
func saveWorkspaceDB(
ctx context.Context,
g *gptscript.GPTScript,
dbWorkspacePath string,
revisionID string,
dbFile *os.File,
initialDBData []byte,
) error {
updatedDBData, err := os.ReadFile(dbFile.Name())
if err != nil {
return fmt.Errorf("Error reading updated DB file: %v", err)
}

if hash(initialDBData) == hash(updatedDBData) {
return nil
}

if err := g.WriteFileInWorkspace(ctx, dbWorkspacePath, updatedDBData, gptscript.WriteFileInWorkspaceOptions{
WorkspaceID: workspaceID,
CreateRevision: &([]bool{true}[0]),
LatestRevisionID: revisionID,
}); err != nil {
return fmt.Errorf("Error writing updated DB file to workspace: %v", err)
}

// Delete old revisions after successfully writing the new revision
revisions, err := g.ListRevisionsForFileInWorkspace(ctx, dbWorkspacePath, gptscript.ListRevisionsForFileInWorkspaceOptions{
WorkspaceID: workspaceID,
})
if err != nil {
fmt.Fprintf(os.Stderr, "Error listing revisions: %v\n", err)
return nil
}

lastRevisionIndex := slices.IndexFunc(revisions, func(rev gptscript.FileInfo) bool {
return rev.RevisionID == revisionID
})

if lastRevisionIndex < 0 {
return nil
}

for _, rev := range revisions[:lastRevisionIndex+1] {
if err := g.DeleteRevisionForFileInWorkspace(ctx, dbWorkspacePath, rev.RevisionID, gptscript.DeleteRevisionForFileInWorkspaceOptions{
WorkspaceID: workspaceID,
}); err != nil {
fmt.Fprintf(os.Stderr, "Error deleting revision %s: %v\n", rev.RevisionID, err)
}
}

return nil
}

// hash computes the SHA-256 hash of the given data and returns it as a hexadecimal string
func hash(data []byte) string {
if data == nil {
return ""
}
hash := sha256.Sum256(data)
return hex.EncodeToString(hash[:])
}
Loading
Loading