refactor: update logging level and improve config handling in service and server commands

.dockerignore

@@ -1,2 +1,2 @@
-/data
+/data*
 /tmp

.github/workflows/embedding.yaml

@@ -0,0 +1,102 @@
+name: Generate Embeddings
+on:
+    push:
+        branches:
+            - use-ollama-for-embeddings
+
+jobs:
+    generate-embeddings:
+        runs-on: ubuntu-24.04
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Fetch Data
+              run: make fetch-data
+
+            - name: Download current DB
+              run: make fetch-db
+
+            - name: Install Ollama
+              run: curl -fsSL https://ollama.com/install.sh | sh
+
+            - name: Pull Model
+              run: ollama pull mxbai-embed-large
+
+            - name: Generate Embeddings
+              run: go run . index
+              env:
+                CLIENT_ID: asd
+                CLIENT_SECRET: asd
+                FQDN: http://localhost
+
+            - name: Create DB.zip
+              run: zip -r db.zip db
+
+            - name: Upload binaries to release
+              uses: svenstaro/upload-release-action@v2
+              with:
+                repo_token: ${{ secrets.GITHUB_TOKEN }}
+                file: db.zip
+                asset_name: db.zip
+                tag: '0.0.1'
+                overwrite: true
+
+    build-image:
+      runs-on: ubuntu-24.04-arm
+      needs: generate-embeddings
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v4
+
+        - name: Log in to registry
+          uses: docker/login-action@v3
+          with:
+                registry: ghcr.io
+                username: ${{ github.actor }}
+                password: ${{ secrets.GITHUB_TOKEN }}
+
+        - name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@v2
+
+        - name: Fetch DB
+          run: make fetch-db
+
+        - name: Build and push
+          uses: docker/build-push-action@v6
+          with:
+            context: .
+            push: true
+            tags: |
+                ghcr.io/shopwarelabs/copilot-extension
+            cache-from: type=gha,scope=copilot
+            cache-to: type=gha,scope=copilot,mode=max
+            provenance: false
+
+    build-ollama:
+      runs-on: ubuntu-24.04-arm
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v4
+
+        - name: Log in to registry
+          uses: docker/login-action@v3
+          with:
+                registry: ghcr.io
+                username: ${{ github.actor }}
+                password: ${{ secrets.GITHUB_TOKEN }}
+
+        - name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@v2
+
+        - name: Build and push
+          uses: docker/build-push-action@v6
+          with:
+            context: .
+            file: Dockerfile.ollama
+            push: true
+            tags: |
+                ghcr.io/shopwarelabs/copilot-extension/ollama
+            cache-from: type=gha,scope=ollama
+            cache-to: type=gha,scope=ollama,mode=max
+            provenance: false

.gitignore

@@ -1,5 +1,5 @@
 /.idea
-/data
+/data*
 /db
 /.env
 /tmp

Dockerfile

@@ -12,7 +12,7 @@ RUN apk add --no-cache ca-certificates
 COPY --from=builder /rag /app/rag
 COPY --from=builder /app/db /app/db
 
-EXPOSE 8080
+EXPOSE 8000
 
 WORKDIR /app
 ENTRYPOINT ["/app/rag"]

Dockerfile.ollama

@@ -0,0 +1,8 @@
+FROM ollama/ollama
+
+ENV OLLAMA_MODELS /models
+ENV OLLAMA_DEBUG false
+ENV OLLAMA_KEEP_ALIVE -1 
+RUN ollama serve & sleep 5 && ollama pull mxbai-embed-large
+
+ENTRYPOINT ["ollama", "serve"]

Makefile

@@ -0,0 +1,35 @@
+fetch-data:
+	rm -rf data
+	git clone --depth=1 https://github.com/shopware/shopware.git data
+	composer install -d data --no-scripts
+	./data/bin/console list --format json > data/commands.json
+	node cli-console-markdown.mjs > all-commands.md
+	cd data && rm -rf .git && find . -mindepth 1 ! -regex '^./src.*' -delete
+	rm -rf data/src/WebInstaller
+	rm -f data/src/Administration/{README.md,LICENSE}
+	rm -rf data/src/Administration/Resources/app/administration/build
+	rm -rf data/src/Administration/Resources/app/administration/eslint-rules
+	rm -rf data/src/Administration/Resources/app/administration/patches
+	rm -rf data/src/Administration/Resources/app/administration/scripts
+	rm -rf data/src/Administration/Resources/app/administration/static
+	rm -rf data/src/Administration/Resources/app/administration/test
+	rm -rf data/src/Administration/Resources/app/administration/*.js
+	find data/src/Administration -name "*.spec.js" -o -name "*.spec.ts" -delete
+	rm data/src/Core/locales.php
+	rm -rf data/src/Storefront/Resources/app/storefront/test
+	rm -rf data/src/Storefront/Resources/app/storefront/static/draco
+	git clone --depth=1 https://github.com/shopware/docs.git data/docs
+	cd data/docs && rm -rf .git && cd ..
+	find data/docs -maxdepth 1 -type f -delete
+	rm -rf data/docs/.github data/docs/.vscode data/docs/assets
+	git clone --depth=1 https://github.com/shopware/frontends.git data/frontends_tmp
+	mv data/frontends_tmp/apps/docs/src data/frontends
+	rm -rf data/frontends_tmp
+	rm -rf data/frontends/ai data/frontends/.assets data/frontends/public
+	mv all-commands.md data/docs/
+
+fetch-db:
+	rm -rf db
+	curl -L -o db.zip https://github.com/shopwareLabs/copilot-extension/releases/download/0.0.1/db.zip
+	unzip -o db.zip
+	rm db.zip

README.md

@@ -16,7 +16,6 @@ and create a `.env` file with the following variables:
 CLIENT_ID=
 CLIENT_SECRET=
 FQDN=<where-the-app-runs>
-LOCAL_GITHUB_TOKEN=<github-token>
 ```
 
 For the client id and client secret you need to create an app in your github account like:

agent/service.go

@@ -93,20 +93,7 @@ func (s *Service) generateCompletion(ctx context.Context, integrationID, apiToke
 
 		startTime := time.Now()
 
-		embeddings, err := copilot.Embeddings(ctx, retryablehttp.NewClient(), integrationID, apiToken, &copilot.EmbeddingsRequest{
-			Model: copilot.ModelEmbeddings,
-			Input: []string{msg.Content},
-		})
-
-		if err != nil {
-			return fmt.Errorf("failed to get embeddings: %w", err)
-		}
-
-		log.Infof("Embeddings took %s", time.Since(startTime))
-
-		startTime = time.Now()
-
-		res, err := s.collection.QueryEmbedding(ctx, embeddings.Data[0].Embedding, 5, nil, nil)
+		res, err := s.collection.Query(ctx, msg.Content, 5, nil, nil)
 
 		if err != nil {
 			return fmt.Errorf("failed to query collection: %w", err)
@@ -170,7 +157,7 @@ func (s *Service) generateCompletion(ctx context.Context, integrationID, apiToke
 
 		if function != nil {
 			usedTools = append(usedTools, function.Name)
-			log.Debugf("Function CALL: %s", function.Name)
+			log.Infof("Function CALL: %s", function.Name)
 
 			msg, err := handleFunction(ctx, function)
 

cli-console-markdown.mjs

@@ -0,0 +1,34 @@
+import { readFileSync } from 'node:fs';
+
+const output = readFileSync('data/commands.json', 'utf8');
+
+const uselessCommands = ['help', 'version', 'exit', '_complete', 'clear', 'history', 'about', 'sync:composer:version', 'secrets:encrypt-from-local', 'secrets:decrypt-to-local', 'secrets:generate-keys', 'secrets:list', 'secrets:remove', 'secrets:reveal', 'secrets:set', 's3:set-visibility', 'completion'];
+
+for (const command of JSON.parse(output).commands) {
+    if (uselessCommands.includes(command.name)) {
+        continue;
+    }
+
+    console.log(`# ${command.usage}`);
+    console.log('')
+
+    if (command.description) {
+        console.log(command.description);
+    }
+
+    if (Object.keys(command.definition.options)) {
+        console.log('')
+        console.log('## Options')
+        console.log('')
+
+        for (const [name, option] of Object.entries(command.definition.options)) {
+            if (['help', 'version', 'silent', 'verbose', 'quiet', 'ansi', 'no-ansi', 'no-interaction', 'profile', 'no-debug', 'env'].includes(name)) {
+                continue;
+            }
+
+            console.log(`- \`--${name}${option.shortcut ? `|${option.shortcut}` : ''}\` - ${option.description}${option.default ? ` (default: ${option.default})` : ''}`);
+        }
+    }
+
+    console.log('')
+}

cmd_delete.go

@@ -10,7 +10,13 @@ var deleteCmd = &cobra.Command{
 	Short: "Delete document from vector db",
 	Args:  cobra.MinimumNArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
-		collection, err := config.GetCollection()
+		cfg, err := config.New()
+
+		if err != nil {
+			return err
+		}
+
+		collection, err := config.GetCollection(cfg)
 
 		if err != nil {
 			return err