diff --git a/Dockerfile b/Dockerfile index e3a5663..5a9829b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24.5-bookworm +FROM golang:1.25.1-alpine WORKDIR /app/ diff --git a/embeddings/embeddings.go b/embeddings/embeddings.go new file mode 100644 index 0000000..3d3252e --- /dev/null +++ b/embeddings/embeddings.go @@ -0,0 +1,25 @@ +package embeddings + +import ( + "context" + + "github.com/openai/openai-go/v2" + "github.com/openai/openai-go/v2/packages/param" +) + +func EmbedText(text string) ([]float64, error) { + client := openai.NewClient() + embd, err := client.Embeddings.New(context.TODO(), openai.EmbeddingNewParams{ + Input: openai.EmbeddingNewParamsInputUnion{OfString: param.Opt[string]{Value: text}}, + Model: openai.EmbeddingModelTextEmbedding3Small, + Dimensions: param.Opt[int64]{Value: 768}, + }) + if err != nil { + return nil, err + } + embds := [][]float64{} + for _, item := range embd.Data { + embds = append(embds, item.Embedding) + } + return embds[0], nil +} diff --git a/embeddings/embeddings_test.go b/embeddings/embeddings_test.go new file mode 100644 index 0000000..b217938 --- /dev/null +++ b/embeddings/embeddings_test.go @@ -0,0 +1,20 @@ +package embeddings + +import ( + "os" + "testing" +) + +func TestEmbedText(t *testing.T) { + _, ok := os.LookupEnv("OPENAI_API_KEY") + if !ok { + t.Skip("Skipping test because OPENAI_API_KEY is not in the environment") + } + embd, err := EmbedText("hello world!") + if err != nil { + t.Errorf("Expecting no error, got %s", err.Error()) + } + if len(embd) != 768 { + t.Errorf("Expecting the embedding to be of length 768, got %d", len(embd)) + } +} diff --git a/go.mod b/go.mod index 33b3ac8..b4a761e 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,13 @@ module github.com/AstraBert/git-push-blog -go 1.24.5 +go 1.25.1 require ( github.com/a-h/templ v0.3.943 github.com/blevesearch/bleve v1.0.14 github.com/gomarkdown/markdown v0.0.0-20250810172220-2e2c11897d1a + github.com/openai/openai-go/v2 v2.6.0 + github.com/takara-ai/serverlessVector v1.0.0 ) require ( @@ -26,6 +28,10 @@ require ( github.com/mschoch/smat v0.2.0 // indirect github.com/philhofer/fwd v1.0.0 // indirect github.com/steveyen/gtreap v0.1.0 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/tinylib/msgp v1.1.0 // indirect github.com/willf/bitset v1.1.10 // indirect go.etcd.io/bbolt v1.3.5 // indirect diff --git a/go.sum b/go.sum index 2303bc4..08d4982 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOl github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/openai/openai-go/v2 v2.6.0 h1:0t3e5AUr5fsgb9TotDJNTdpGqf/SSSfMX4pr8QrV9OY= +github.com/openai/openai-go/v2 v2.6.0/go.mod h1:sIUkR+Cu/PMUVkSKhkk742PRURkQOCFhiwJ7eRSBqmk= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -103,9 +105,21 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/takara-ai/serverlessVector v1.0.0 h1:BGXb20ZgNlgJ/4YnqBwzg9414zD4189vy2S1vfiaO/Y= +github.com/takara-ai/serverlessVector v1.0.0/go.mod h1:fCkqNXUgsyg91gqayAoP/2vLXMY1FwU94S5OAVE6pDY= github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg= github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= diff --git a/vectordb/vectordb.go b/vectordb/vectordb.go new file mode 100644 index 0000000..aad4e97 --- /dev/null +++ b/vectordb/vectordb.go @@ -0,0 +1,10 @@ +package vectordb + +import ( + "github.com/takara-ai/serverlessVector" +) + +func CreateDB() *serverlessVector.VectorDB { + db := serverlessVector.NewVectorDB(384) + return db +} diff --git a/vectordb/vectordb_test.go b/vectordb/vectordb_test.go new file mode 100644 index 0000000..36229ca --- /dev/null +++ b/vectordb/vectordb_test.go @@ -0,0 +1,21 @@ +package vectordb + +import "testing" + +func TestCreateVectorDB(t *testing.T) { + db := CreateDB() + if db.Size() > 0 { + t.Errorf("Expecting new DB to have size zero, got %d", db.Size()) + } + if dims, ok := db.GetStats()["dimension"]; ok { + if dimsInt, isInt := dims.(int); isInt { + if dimsInt != 384 { + t.Errorf("Expecting 'dimension' to be 384, but got %d", dimsInt) + } + } else { + t.Error("Expecting 'dimension' to be an integer, but they are not.") + } + } else { + t.Error("Expecting 'dimension' to be found in stats, but it is not.") + } +}