diff --git a/Dockerfile b/Dockerfile index ac59c9b..467aa63 100755 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,14 @@ COPY ./src /usr/local/src # Extensions and libraries COPY --from=composer /usr/local/src/vendor /usr/local/vendor +RUN \ + apk update \ + && apk add --no-cache make automake autoconf gcc g++ git brotli-dev docker-cli curl + +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/kubectl + HEALTHCHECK --interval=30s --timeout=15s --start-period=60s --retries=3 CMD curl -s -H "Authorization: Bearer ${OPR_EXECUTOR_SECRET}" --fail http://127.0.0.1:80/v1/health CMD [ "php", "app/http.php" ] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9b6b416 --- /dev/null +++ b/Makefile @@ -0,0 +1,193 @@ +.PHONY: help start-docker test-docker helm-clean helm-install helm-install-local helm-upgrade helm-uninstall helm-test kind-create kind-delete kind-load kind-full docker-build docker-push deploy stop clean + +# Configuration +IMAGE_NAME ?= openruntimes/executor +IMAGE_TAG ?= latest +HELM_RELEASE ?= executor +HELM_NAMESPACE ?= default +KIND_CLUSTER ?= executor-cluster + +# Default target +help: + @echo "Open Runtimes Executor - Make Commands" + @echo "" + @echo "Docker Runner:" + @echo " make start-docker - Start executor with Docker runner" + @echo " make test-docker - Run tests against Docker executor" + @echo " make logs-docker - View Docker executor logs" + @echo "" + @echo "Helm Deployment:" + @echo " make helm-clean - Clean up conflicting resources" + @echo " make helm-install - Install executor with Helm" + @echo " make helm-upgrade - Upgrade executor Helm release" + @echo " make helm-uninstall - Uninstall executor Helm release" + @echo " make helm-test - Port-forward and test Helm deployment" + @echo "" + @echo "Kind (Local Kubernetes):" + @echo " make kind-create - Create Kind cluster" + @echo " make kind-delete - Delete Kind cluster" + @echo " make kind-load - Build and load image to Kind" + @echo " make kind-full - Full setup: create cluster, load image, install" + @echo "" + @echo "Docker Image:" + @echo " make docker-build - Build executor Docker image" + @echo " make docker-push - Push image to registry" + @echo "" + @echo "Combined:" + @echo " make deploy - Build, load to kind, and install with helm" + @echo "" + @echo "General:" + @echo " make stop - Stop Docker executor" + @echo " make clean - Clean up everything" + +# Docker Runner targets +start-docker: + @echo "Starting Executor with Docker runner..." + docker-compose up -d + @echo "Executor is running on http://localhost:80" + +logs-docker: + docker-compose logs -f + +# Helm targets +helm-clean: + @echo "Cleaning up existing resources that conflict with Helm..." + -kubectl delete serviceaccount executor-sa -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete role executor-role -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete rolebinding executor-rolebinding -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete deployment executor-k8s -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete service executor-k8s -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete configmap executor-k8s-config -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete secret executor-k8s-secret -n $(HELM_NAMESPACE) 2>/dev/null || true + @echo "Cleanup complete!" + +helm-install: helm-clean + @echo "Installing Executor with Helm..." + helm install $(HELM_RELEASE) ./deploy \ + --namespace $(HELM_NAMESPACE) \ + --create-namespace \ + --wait + @echo "" + @echo "Executor installed successfully!" + @echo "To access the executor, run:" + @echo " make helm-test" + +helm-install-local: helm-clean + @echo "Installing Executor with Helm (local Kind values)..." + helm install $(HELM_RELEASE) ./deploy \ + -f ./deploy/values-local.yaml \ + --namespace $(HELM_NAMESPACE) \ + --create-namespace \ + --wait + @echo "" + @echo "Executor installed successfully!" + @echo "To access the executor, run:" + @echo " make helm-test" + +helm-upgrade: + @echo "Upgrading Executor..." + helm upgrade $(HELM_RELEASE) ./deploy \ + --namespace $(HELM_NAMESPACE) \ + --wait + @echo "Executor upgraded successfully!" + +helm-uninstall: + @echo "Uninstalling Executor..." + helm uninstall $(HELM_RELEASE) --namespace $(HELM_NAMESPACE) + @echo "Executor uninstalled!" + +helm-test: + @echo "Port-forwarding to executor service..." + @echo "Executor will be available at http://localhost:8080" + @echo "Press Ctrl+C to stop" + @kubectl port-forward -n $(HELM_NAMESPACE) svc/$(HELM_RELEASE)-openruntimes-executor 8080:80 + +# Kind targets +kind-create: + @echo "Creating Kind cluster..." + @if kind get clusters | grep -q $(KIND_CLUSTER); then \ + echo "Cluster $(KIND_CLUSTER) already exists"; \ + else \ + kind create cluster --name $(KIND_CLUSTER); \ + echo "Cluster created successfully!"; \ + fi + +kind-delete: + @echo "Deleting Kind cluster..." + kind delete cluster --name $(KIND_CLUSTER) + @echo "Cluster deleted!" + +kind-load: + @echo "Building and loading image to Kind..." + docker build -t $(IMAGE_NAME):$(IMAGE_TAG) . + kind load docker-image $(IMAGE_NAME):$(IMAGE_TAG) --name $(KIND_CLUSTER) + @echo "Image loaded successfully!" + +kind-full: kind-create kind-load helm-install-local + @echo "" + @echo "✅ Full setup complete!" + @echo "Run 'make helm-test' to access the executor" + +# Docker image targets +docker-build: + @echo "Building Docker image..." + docker build -t $(IMAGE_NAME):$(IMAGE_TAG) . + @echo "Image built successfully!" + +docker-push: docker-build + @echo "Pushing Docker image..." + docker push $(IMAGE_NAME):$(IMAGE_TAG) + @echo "Image pushed successfully!" + +# Combined deployment +deploy: kind-load helm-upgrade + @echo "" + @echo "✅ Deployment complete!" + @echo "Run 'make helm-test' to access the executor" + +# Testing targets +test-docker: + @echo "Running tests against Docker executor..." + composer test + +# Cleanup targets +stop: + @echo "Stopping Docker executor..." + -docker-compose down + +clean: + @echo "Cleaning up..." + @echo "Stopping Docker executor..." + -docker-compose down + @echo "Uninstalling Helm release..." + -helm uninstall $(HELM_RELEASE) --namespace $(HELM_NAMESPACE) 2>/dev/null || true + @echo "Cleaning up conflicting Kubernetes resources..." + -kubectl delete serviceaccount executor-sa -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete role executor-role -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete rolebinding executor-rolebinding -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete deployment executor-k8s -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete service executor-k8s -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete configmap executor-k8s-config -n $(HELM_NAMESPACE) 2>/dev/null || true + -kubectl delete secret executor-k8s-secret -n $(HELM_NAMESPACE) 2>/dev/null || true + @echo "Deleting Kind cluster..." + -kind delete cluster --name $(KIND_CLUSTER) 2>/dev/null || true + @echo "Cleaning Docker..." + -docker system prune -f + @echo "Cleanup complete!" + +# Development targets +install: + composer install + +lint: + composer lint + +format: + composer format + +check: + composer check + +# Build target +build: + docker-compose build diff --git a/README.md b/README.md index d4ca40f..780b583 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,60 @@ curl -H "authorization: Bearer executor-secret-key" -H "Content-Type: applicatio docker compose down ``` +## Kubernetes Deployment + +For deploying to Kubernetes with Helm: + +### Quick Start with Kind (Local) + +```bash +# Create local Kubernetes cluster +make kind-create + +# Build and deploy +make kind-full + +# Test the executor +make helm-test +``` + +### Production Deployment + +```bash +# Install with Helm +helm install executor ./deploy \ + --set secret.executorSecret="your-secure-secret-key" \ + --set ingress.enabled=true \ + --set ingress.hosts[0].host="executor.example.com" \ + --set sharedStorage.enabled=true \ + --set sharedStorage.storageClass="nfs-storage" + +# Or use production values +helm install executor ./deploy -f ./deploy/values-production.yaml +``` + +### Shared Storage (Required for Function Execution) + +For full function execution in Kubernetes, you need shared storage between executor and runtime pods: + +```bash +# Configure shared storage (ReadWriteMany required) +helm install executor ./deploy \ + --set sharedStorage.enabled=true \ + --set sharedStorage.storageClass="efs-sc" # AWS EFS + --set sharedStorage.size="100Gi" +``` + +Supported storage classes: +- **AWS**: EFS (`efs-sc`) +- **Azure**: Azure Files (`azurefile`) +- **GCP**: Filestore (`filestore-sc`) +- **On-Premises**: NFS, Ceph, GlusterFS + +See the [shared storage documentation](./deploy/STORAGE.md) for detailed setup instructions. + +See the [deployment documentation](./deploy/README.md) for more details. + ## API Endpoints | Method | Endpoint | Description | Params | @@ -192,6 +246,7 @@ docker compose down | Variable name | Description | |------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| | OPR_EXECUTOR_ENV | Environment mode of the executor, ex. `development` | +| OPR_EXECUTOR_RUNNER | Runner type for the executor: `docker` (default) or `kubernetes` | | OPR_EXECUTOR_RUNTIMES | Comma-separated list of supported runtimes `(ex: php-8.1,dart-2.18,deno-1.24,..)`. These runtimes should be available as container images. | | OPR_EXECUTOR_CONNECTION_STORAGE | DSN string that represents a connection to your storage device, ex: `file://localhost` for local storage | | OPR_EXECUTOR_INACTIVE_THRESHOLD | Threshold time (in seconds) for detecting inactive runtimes, ex: `60` | @@ -200,8 +255,11 @@ docker compose down | OPR_EXECUTOR_SECRET | Secret key used by the executor for authentication | | OPR_EXECUTOR_LOGGING_PROVIDER | Deprecated: use `OPR_EXECUTOR_LOGGING_CONFIG` with DSN instead. External logging provider used by the executor, ex: `sentry` | | OPR_EXECUTOR_LOGGING_CONFIG | External logging provider DSN used by the executor, ex: `sentry://PROJECT_ID:SENTRY_API_KEY@SENTRY_HOST/` | -| OPR_EXECUTOR_DOCKER_HUB_USERNAME | Username for Docker Hub authentication (if applicable) | -| OPR_EXECUTOR_DOCKER_HUB_PASSWORD | Password for Docker Hub authentication (if applicable) | +| OPR_EXECUTOR_DOCKER_HUB_USERNAME | Username for Docker Hub authentication (if applicable, used when runner is `docker`) | +| OPR_EXECUTOR_DOCKER_HUB_PASSWORD | Password for Docker Hub authentication (if applicable, used when runner is `docker`) | +| OPR_EXECUTOR_K8S_URL | Kubernetes API URL (if applicable, used when runner is `kubernetes`), ex: `https://kubernetes.default.svc` | +| OPR_EXECUTOR_K8S_NAMESPACE | Kubernetes namespace for runtime pods (if applicable, used when runner is `kubernetes`), ex: `default` | +| OPR_EXECUTOR_K8S_TOKEN | Kubernetes API token for external access (optional, auto-detected from ServiceAccount when running in-cluster) | | OPR_EXECUTOR_RUNTIME_VERSIONS | Version tag for runtime environments, ex: `v5` | | OPR_EXECUTOR_RETRY_ATTEMPTS | Number of retry attempts for failed executions, ex: `5` | | OPR_EXECUTOR_RETRY_DELAY_MS | Delay (in milliseconds) between retry attempts, ex: `500` | diff --git a/app/http.php b/app/http.php index ca1a2ee..02dbfa1 100644 --- a/app/http.php +++ b/app/http.php @@ -8,6 +8,7 @@ require_once __DIR__ . '/controllers.php'; use OpenRuntimes\Executor\Runner\Docker; +use OpenRuntimes\Executor\Runner\Kubernetes; use Swoole\Runtime; use Utopia\Console; use Utopia\Http\Http; @@ -15,6 +16,7 @@ use Utopia\Http\Adapter\Swoole\Server; use Utopia\System\System; use Utopia\Orchestration\Adapter\DockerAPI; +use Utopia\Orchestration\Adapter\K8s; use Utopia\Orchestration\Orchestration; use function Swoole\Coroutine\run; @@ -25,19 +27,76 @@ Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL); Http::setMode((string)System::getEnv('OPR_EXECUTOR_ENV', Http::MODE_TYPE_PRODUCTION)); + +// Determine runner type from environment variable +$runnerType = System::getEnv('OPR_EXECUTOR_RUNNER', 'docker'); + Http::onRequest() ->inject('response') - ->action(function (Response $response) { - $response->addHeader('Server', 'Executor'); + ->action(function (Response $response) use ($runnerType) { + $serverHeader = $runnerType === 'kubernetes' ? 'Executor-K8s' : 'Executor'; + $response->addHeader('Server', $serverHeader); }); -run(function () { - $orchestration = new Orchestration(new DockerAPI( - System::getEnv('OPR_EXECUTOR_DOCKER_HUB_USERNAME', ''), - System::getEnv('OPR_EXECUTOR_DOCKER_HUB_PASSWORD', '') - )); +run(function () use ($runnerType) { $networks = explode(',', System::getEnv('OPR_EXECUTOR_NETWORK') ?: 'openruntimes-runtimes'); - $runner = new Docker($orchestration, $networks); + + if ($runnerType === 'kubernetes') { + Console::info("Initializing Kubernetes orchestration..."); + + // Kubernetes in-cluster configuration + $k8sUrl = System::getEnv('OPR_EXECUTOR_K8S_URL', 'https://kubernetes.default.svc'); + $k8sNamespace = System::getEnv('OPR_EXECUTOR_K8S_NAMESPACE', 'default'); + + Console::info("K8s API URL: $k8sUrl"); + Console::info("K8s Namespace: $k8sNamespace"); + + // Build authentication configuration + $auth = []; + + // Try to read ServiceAccount token from the mounted secret (in-cluster) + $tokenPath = '/var/run/secrets/kubernetes.io/serviceaccount/token'; + $caPath = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'; + + if (file_exists($tokenPath)) { + $tokenContent = file_get_contents($tokenPath); + if ($tokenContent !== false) { + $auth['token'] = trim($tokenContent); + Console::info("Using ServiceAccount token from: $tokenPath"); + } + } elseif ($token = System::getEnv('OPR_EXECUTOR_K8S_TOKEN', '')) { + // Fallback to environment variable (for external access) + $auth['token'] = $token; + Console::info("Using token from environment variable"); + } + + if (file_exists($caPath)) { + $auth['ca'] = $caPath; + Console::info("Using CA certificate from: $caPath"); + } + + if (empty($auth['token'])) { + Console::warning("No authentication token found! Make sure ServiceAccount is properly configured."); + } + + $orchestration = new Orchestration(new K8s($k8sUrl, $k8sNamespace ?? 'default', $auth)); + + Console::info("Creating Kubernetes runner..."); + $runner = new Kubernetes($orchestration, $networks); + + Console::success('Kubernetes Executor is ready.'); + } else { + Console::info("Initializing Docker orchestration..."); + + $orchestration = new Orchestration(new DockerAPI( + System::getEnv('OPR_EXECUTOR_DOCKER_HUB_USERNAME', ''), + System::getEnv('OPR_EXECUTOR_DOCKER_HUB_PASSWORD', '') + )); + + $runner = new Docker($orchestration, $networks); + + Console::success('Docker Executor is ready.'); + } Http::setResource('runner', fn () => $runner); @@ -50,7 +109,5 @@ $server = new Server('0.0.0.0', '80', $settings); $http = new Http($server, 'UTC'); - Console::success('Executor is ready.'); - $http->start(); }); diff --git a/deploy/.helmignore b/deploy/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deploy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/Chart.yaml b/deploy/Chart.yaml new file mode 100644 index 0000000..eb1a374 --- /dev/null +++ b/deploy/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: openruntimes-executor +description: A Helm chart for OpenRuntimes Executor on Kubernetes +type: application +version: 1.0.0 +appVersion: "latest" +keywords: + - openruntimes + - executor + - serverless + - functions +home: https://github.com/open-runtimes/executor +sources: + - https://github.com/open-runtimes/executor +maintainers: + - name: OpenRuntimes Team + url: https://github.com/open-runtimes diff --git a/deploy/templates/NOTES.txt b/deploy/templates/NOTES.txt new file mode 100644 index 0000000..c32dfd2 --- /dev/null +++ b/deploy/templates/NOTES.txt @@ -0,0 +1,28 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openruntimes-executor.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openruntimes-executor.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openruntimes-executor.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openruntimes-executor.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} + +2. Test the executor: + curl -H "Authorization: Bearer {{ .Values.secret.executorSecret }}" http://localhost:8080/v1/health + +3. View logs: + kubectl logs -f -l "app.kubernetes.io/name={{ include "openruntimes-executor.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -n {{ .Release.Namespace }} diff --git a/deploy/templates/_helpers.tpl b/deploy/templates/_helpers.tpl new file mode 100644 index 0000000..0f60a24 --- /dev/null +++ b/deploy/templates/_helpers.tpl @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "openruntimes-executor.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "openruntimes-executor.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "openruntimes-executor.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "openruntimes-executor.labels" -}} +helm.sh/chart: {{ include "openruntimes-executor.chart" . }} +{{ include "openruntimes-executor.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "openruntimes-executor.selectorLabels" -}} +app.kubernetes.io/name: {{ include "openruntimes-executor.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "openruntimes-executor.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "openruntimes-executor.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/templates/configmap.yaml b/deploy/templates/configmap.yaml new file mode 100644 index 0000000..d4f8a91 --- /dev/null +++ b/deploy/templates/configmap.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} +data: + OPR_EXECUTOR_ENV: {{ .Values.config.env | quote }} + OPR_EXECUTOR_RUNNER: {{ .Values.config.runner | quote }} + OPR_EXECUTOR_RUNTIMES: {{ .Values.config.runtimes | quote }} + OPR_EXECUTOR_RUNTIME_VERSIONS: {{ .Values.config.runtimeVersions | quote }} + OPR_EXECUTOR_CONNECTION_STORAGE: {{ .Values.config.connectionStorage | quote }} + OPR_EXECUTOR_INACTIVE_THRESHOLD: {{ .Values.config.inactiveThreshold | quote }} + OPR_EXECUTOR_MAINTENANCE_INTERVAL: {{ .Values.config.maintenanceInterval | quote }} + OPR_EXECUTOR_NETWORK: {{ .Values.config.network | quote }} + OPR_EXECUTOR_IMAGE_PULL: {{ .Values.config.imagePull | quote }} + OPR_EXECUTOR_K8S_URL: {{ .Values.config.kubernetes.url | quote }} + OPR_EXECUTOR_K8S_NAMESPACE: {{ .Values.config.kubernetes.namespace | default .Release.Namespace | quote }} + OPR_EXECUTOR_RETRY_ATTEMPTS: {{ .Values.config.retryAttempts | quote }} + OPR_EXECUTOR_RETRY_DELAY_MS: {{ .Values.config.retryDelayMs | quote }} + {{- if .Values.sharedStorage.enabled }} + OPR_EXECUTOR_STORAGE_BUILDS: {{ .Values.sharedStorage.builds.mountPath | quote }} + OPR_EXECUTOR_STORAGE_FUNCTIONS: {{ .Values.sharedStorage.functions.mountPath | quote }} + OPR_EXECUTOR_STORAGE_BUILDS_PVC: {{ include "openruntimes-executor.fullname" . }}-builds + OPR_EXECUTOR_STORAGE_FUNCTIONS_PVC: {{ include "openruntimes-executor.fullname" . }}-functions + {{- end }} diff --git a/deploy/templates/deployment.yaml b/deploy/templates/deployment.yaml new file mode 100644 index 0000000..3e6ea8b --- /dev/null +++ b/deploy/templates/deployment.yaml @@ -0,0 +1,113 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "openruntimes-executor.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "openruntimes-executor.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "openruntimes-executor.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["php", "app/http.php"] + ports: + - name: http + containerPort: 80 + protocol: TCP + envFrom: + - configMapRef: + name: {{ include "openruntimes-executor.fullname" . }} + - secretRef: + name: {{ include "openruntimes-executor.fullname" . }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: {{ .Values.livenessProbe.httpGet.path }} + port: {{ .Values.livenessProbe.httpGet.port }} + httpHeaders: + - name: Authorization + value: Bearer {{ .Values.secret.executorSecret }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: {{ .Values.readinessProbe.httpGet.path }} + port: {{ .Values.readinessProbe.httpGet.port }} + httpHeaders: + - name: Authorization + value: Bearer {{ .Values.secret.executorSecret }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + {{- if .Values.sharedStorage.enabled }} + - name: builds + mountPath: {{ .Values.sharedStorage.builds.mountPath }} + - name: functions + mountPath: {{ .Values.sharedStorage.functions.mountPath }} + {{- end }} + volumes: + - name: tmp + {{- if .Values.persistence.emptyDir }} + emptyDir: + sizeLimit: {{ .Values.persistence.size }} + {{- else }} + persistentVolumeClaim: + claimName: {{ include "openruntimes-executor.fullname" . }} + {{- end }} + {{- if .Values.sharedStorage.enabled }} + - name: builds + persistentVolumeClaim: + claimName: {{ include "openruntimes-executor.fullname" . }}-builds + - name: functions + persistentVolumeClaim: + claimName: {{ include "openruntimes-executor.fullname" . }}-functions + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/templates/hpa.yaml b/deploy/templates/hpa.yaml new file mode 100644 index 0000000..66cfb15 --- /dev/null +++ b/deploy/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "openruntimes-executor.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/deploy/templates/ingress.yaml b/deploy/templates/ingress.yaml new file mode 100644 index 0000000..e8f24c7 --- /dev/null +++ b/deploy/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "openruntimes-executor.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/templates/pdb.yaml b/deploy/templates/pdb.yaml new file mode 100644 index 0000000..cfd44ab --- /dev/null +++ b/deploy/templates/pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} +spec: + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + selector: + matchLabels: + {{- include "openruntimes-executor.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/templates/pvc.yaml b/deploy/templates/pvc.yaml new file mode 100644 index 0000000..dc73a6c --- /dev/null +++ b/deploy/templates/pvc.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.emptyDir) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} +--- +{{- if .Values.sharedStorage.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "openruntimes-executor.fullname" . }}-builds + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} + app.kubernetes.io/component: storage +spec: + accessModes: + - {{ .Values.sharedStorage.accessMode }} + {{- if .Values.sharedStorage.storageClass }} + storageClassName: {{ .Values.sharedStorage.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.sharedStorage.size }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "openruntimes-executor.fullname" . }}-functions + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} + app.kubernetes.io/component: storage +spec: + accessModes: + - {{ .Values.sharedStorage.accessMode }} + {{- if .Values.sharedStorage.storageClass }} + storageClassName: {{ .Values.sharedStorage.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.sharedStorage.size }} +{{- end }} + diff --git a/deploy/templates/rbac.yaml b/deploy/templates/rbac.yaml new file mode 100644 index 0000000..6416252 --- /dev/null +++ b/deploy/templates/rbac.yaml @@ -0,0 +1,60 @@ +{{- if .Values.rbac.create -}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} +rules: + # Permissions for managing pods + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "create", "delete", "watch", "update", "patch"] + + # Permissions for pod logs + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get", "list"] + + # Permissions for executing commands in pods + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create", "get"] + + # Permissions for pod status + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["get", "list", "watch"] + + # Permissions for network policies + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "list", "create", "delete", "update", "patch"] + + # Permissions for secrets (if needed for storing runtime secrets) + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + + # Permissions for config maps (if needed) + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "openruntimes-executor.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "openruntimes-executor.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/templates/secret.yaml b/deploy/templates/secret.yaml new file mode 100644 index 0000000..62e83e5 --- /dev/null +++ b/deploy/templates/secret.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} +type: Opaque +stringData: + OPR_EXECUTOR_SECRET: {{ .Values.secret.executorSecret | quote }} + {{- if .Values.secret.dockerHub.username }} + OPR_EXECUTOR_DOCKER_HUB_USERNAME: {{ .Values.secret.dockerHub.username | quote }} + {{- end }} + {{- if .Values.secret.dockerHub.password }} + OPR_EXECUTOR_DOCKER_HUB_PASSWORD: {{ .Values.secret.dockerHub.password | quote }} + {{- end }} diff --git a/deploy/templates/service.yaml b/deploy/templates/service.yaml new file mode 100644 index 0000000..1054059 --- /dev/null +++ b/deploy/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "openruntimes-executor.fullname" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "openruntimes-executor.selectorLabels" . | nindent 4 }} diff --git a/deploy/templates/serviceaccount.yaml b/deploy/templates/serviceaccount.yaml new file mode 100644 index 0000000..6c9a0a4 --- /dev/null +++ b/deploy/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "openruntimes-executor.serviceAccountName" . }} + labels: + {{- include "openruntimes-executor.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/values-local.yaml b/deploy/values-local.yaml new file mode 100644 index 0000000..6f17baa --- /dev/null +++ b/deploy/values-local.yaml @@ -0,0 +1,44 @@ +# Values for local Kind development +replicaCount: 1 + +image: + repository: openruntimes/executor + pullPolicy: IfNotPresent + tag: "latest" + +config: + runner: "kubernetes" + env: "development" + runtimes: "php-8.3,node-21.0,python-3.12" + runtimeVersions: "v5" + inactiveThreshold: "60" + maintenanceInterval: "300" + imagePull: "disabled" # Use local images + +secret: + executorSecret: "local-dev-secret-key" + +resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 250m + memory: 256Mi + +autoscaling: + enabled: false + +persistence: + emptyDir: true + size: 5Gi + +# Shared storage for local development +# Kind doesn't support ReadWriteMany PVCs by default +# For local dev, we'll disable shared storage and use emptyDir +# In production, use NFS or cloud provider RWX storage +sharedStorage: + enabled: false + storageClass: "standard" + accessMode: ReadWriteOnce + size: 10Gi diff --git a/deploy/values-production.yaml b/deploy/values-production.yaml new file mode 100644 index 0000000..c27c96c --- /dev/null +++ b/deploy/values-production.yaml @@ -0,0 +1,82 @@ +# Values for production deployment +replicaCount: 3 + +image: + repository: openruntimes/executor + pullPolicy: Always + tag: "latest" + +config: + runner: "kubernetes" + env: "production" + runtimes: "php-8.3,node-21.0,python-3.12,deno-1.40,ruby-3.3,go-1.21" + runtimeVersions: "v5" + inactiveThreshold: "300" + maintenanceInterval: "600" + imagePull: "enabled" + +secret: + # IMPORTANT: Change this in production! + executorSecret: "CHANGE-THIS-IN-PRODUCTION" + +resources: + limits: + cpu: 2000m + memory: 4Gi + requests: + cpu: 1000m + memory: 2Gi + +autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 20 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + +persistence: + emptyDir: false + storageClass: "fast-ssd" + size: 50Gi + +# Shared storage for production - requires ReadWriteMany storage class +# Examples: NFS, EFS (AWS), Azure Files, GCP Filestore +sharedStorage: + enabled: true + storageClass: "nfs-storage" # Change to your RWX storage class + accessMode: ReadWriteMany + size: 100Gi + +podDisruptionBudget: + enabled: true + minAvailable: 2 + +ingress: + enabled: true + className: "nginx" + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/rate-limit: "100" + hosts: + - host: executor.example.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: executor-tls + hosts: + - executor.example.com + +# Node affinity for production workloads +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - openruntimes-executor + topologyKey: kubernetes.io/hostname diff --git a/deploy/values.yaml b/deploy/values.yaml new file mode 100644 index 0000000..6308d2f --- /dev/null +++ b/deploy/values.yaml @@ -0,0 +1,189 @@ +# Default values for openruntimes-executor. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: openruntimes/executor + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "executor-sa" + +rbac: + # Specifies whether RBAC resources should be created + create: true + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + annotations: {} + +ingress: + enabled: false + className: "nginx" + annotations: {} + # cert-manager.io/cluster-issuer: letsencrypt-prod + # nginx.ingress.kubernetes.io/rewrite-target: / + hosts: + - host: executor.example.com + paths: + - path: / + pathType: Prefix + tls: [] + # - secretName: executor-tls + # hosts: + # - executor.example.com + +resources: + limits: + cpu: 2000m + memory: 2Gi + requests: + cpu: 500m + memory: 512Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Executor configuration +config: + # Runner type: docker or kubernetes + runner: "kubernetes" + + # Environment mode + env: "production" + + # Comma-separated list of supported runtimes + runtimes: "php-8.3,node-21.0,python-3.12,deno-1.40" + + # Runtime versions + runtimeVersions: "v5" + + # Storage connection + connectionStorage: "file://localhost" + + # Inactive threshold in seconds + inactiveThreshold: "300" + + # Maintenance interval in seconds + maintenanceInterval: "600" + + # Network name + network: "openruntimes-runtimes" + + # Image pull policy: enabled or disabled + imagePull: "enabled" + + # Kubernetes configuration + kubernetes: + url: "https://kubernetes.default.svc" + namespace: "default" + + # Retry configuration + retryAttempts: "5" + retryDelayMs: "500" + +# Secret configuration +secret: + # Executor secret key for authentication + # IMPORTANT: Change this in production! + executorSecret: "change-this-secret-key-in-production" + + # Docker Hub credentials (optional, for docker runner) + dockerHub: + username: "" + password: "" + +# Persistent volume for /tmp (executor pod only) +persistence: + enabled: true + storageClass: "" + accessMode: ReadWriteOnce + size: 10Gi + # Use emptyDir instead of PVC + emptyDir: true + +# Shared storage for code/functions between executor and runtime pods +sharedStorage: + enabled: true + # Storage class - leave empty for default, or specify (e.g., "standard", "gp2", "local-path") + storageClass: "" + accessMode: ReadWriteMany + size: 20Gi + # Mount paths + builds: + mountPath: /storage/builds + functions: + mountPath: /storage/functions + +# Liveness and readiness probes +livenessProbe: + enabled: true + httpGet: + path: /v1/health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + enabled: true + httpGet: + path: /v1/health + port: http + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + +# Pod Disruption Budget +podDisruptionBudget: + enabled: false + minAvailable: 1 + +# Network Policy +networkPolicy: + enabled: false + policyTypes: + - Ingress + - Egress + ingress: [] + egress: [] diff --git a/docker-compose.yml b/docker-compose.yml index ab4519e..2330fda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,7 @@ services: - ./tests/resources/sites:/storage/sites:rw environment: - OPR_EXECUTOR_ENV + - OPR_EXECUTOR_RUNNER - OPR_EXECUTOR_RUNTIMES - OPR_EXECUTOR_CONNECTION_STORAGE - OPR_EXECUTOR_INACTIVE_THRESHOLD @@ -44,6 +45,9 @@ services: - OPR_EXECUTOR_LOGGING_CONFIG - OPR_EXECUTOR_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD + - OPR_EXECUTOR_K8S_URL + - OPR_EXECUTOR_K8S_NAMESPACE + - OPR_EXECUTOR_K8S_TOKEN - OPR_EXECUTOR_RUNTIME_VERSIONS - OPR_EXECUTOR_RETRY_ATTEMPTS - OPR_EXECUTOR_RETRY_DELAY_MS diff --git a/src/Executor/Runner/Kubernetes.php b/src/Executor/Runner/Kubernetes.php new file mode 100644 index 0000000..5e93f66 --- /dev/null +++ b/src/Executor/Runner/Kubernetes.php @@ -0,0 +1,1305 @@ +activeRuntimes = new Table(4096); + + $this->activeRuntimes->column('version', Table::TYPE_STRING, 32); + $this->activeRuntimes->column('created', Table::TYPE_FLOAT); + $this->activeRuntimes->column('updated', Table::TYPE_FLOAT); + $this->activeRuntimes->column('name', Table::TYPE_STRING, 1024); + $this->activeRuntimes->column('hostname', Table::TYPE_STRING, 1024); + $this->activeRuntimes->column('status', Table::TYPE_STRING, 256); + $this->activeRuntimes->column('key', Table::TYPE_STRING, 1024); + $this->activeRuntimes->column('listening', Table::TYPE_INT, 1); + $this->activeRuntimes->column('image', Table::TYPE_STRING, 1024); + $this->activeRuntimes->column('initialised', Table::TYPE_INT, 0); + $this->activeRuntimes->create(); + + $this->stats = new Stats(); + + $this->init($networks); + } + + /** + * @param string[] $networks + * @return void + * @throws \Utopia\Http\Exception + */ + private function init(array $networks): void + { + /* + * Remove residual runtimes and networks + */ + Console::info('Removing orphan runtimes and networks...'); + $this->cleanUp(); + Console::success("Orphan runtimes and networks removal finished."); + + /** + * Create and store Kubernetes networks (NetworkPolicies) used for communication between executor and runtimes + */ + Console::info('Creating networks...'); + $createdNetworks = $this->createNetworks($networks); + $this->networks = $createdNetworks; + + /** + * Warmup: make sure images are ready to run fast 🚀 + */ + $allowList = empty(System::getEnv('OPR_EXECUTOR_RUNTIMES')) ? [] : \explode(',', System::getEnv('OPR_EXECUTOR_RUNTIMES')); + + if (System::getEnv('OPR_EXECUTOR_IMAGE_PULL', 'enabled') === 'disabled') { + // Useful to prevent auto-pulling from remote when testing local images + Console::info("Skipping image pulling"); + } else { + $runtimeVersions = \explode(',', System::getEnv('OPR_EXECUTOR_RUNTIME_VERSIONS', 'v5') ?? 'v5'); + foreach ($runtimeVersions as $runtimeVersion) { + Console::success("Pulling $runtimeVersion images..."); + $runtimes = new Runtimes($runtimeVersion); + $runtimes = $runtimes->getAll(true, $allowList); + $callables = []; + foreach ($runtimes as $runtime) { + $callables[] = function () use ($runtime) { + Console::log('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...'); + $response = $this->orchestration->pull($runtime['image']); + if ($response) { + Console::info("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); + } else { + Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!"); + } + }; + } + + batch($callables); + } + } + + Console::success("Image pulling finished."); + + /** + * Run a maintenance worker every X seconds to remove inactive runtimes + */ + Console::info('Starting maintenance interval...'); + $interval = (int)System::getEnv('OPR_EXECUTOR_MAINTENANCE_INTERVAL', '3600'); // In seconds + Timer::tick($interval * 1000, function () { + Console::info("Running maintenance task ..."); + // Stop idling runtimes + foreach ($this->activeRuntimes as $runtimeName => $runtime) { + $inactiveThreshold = \time() - \intval(System::getEnv('OPR_EXECUTOR_INACTIVE_THRESHOLD', '60')); + if ($runtime['updated'] < $inactiveThreshold) { + go(function () use ($runtimeName, $runtime) { + try { + $this->orchestration->remove($runtime['name'], true); + Console::success("Successfully removed {$runtime['name']}"); + } catch (\Throwable $th) { + Console::error('Inactive Runtime deletion failed: ' . $th->getMessage()); + } finally { + $this->activeRuntimes->del($runtimeName); + } + }); + } + } + + // Clear leftover build folders + $localDevice = new Local(); + $tmpPath = DIRECTORY_SEPARATOR . 'tmp' . DIRECTORY_SEPARATOR; + $entries = $localDevice->getFiles($tmpPath); + $prefix = $tmpPath . System::getHostname() . '-'; + foreach ($entries as $entry) { + if (\str_starts_with($entry, $prefix)) { + $isActive = false; + + foreach ($this->activeRuntimes as $runtimeName => $runtime) { + if (\str_ends_with($entry, $runtimeName)) { + $isActive = true; + break; + } + } + + if (!$isActive) { + $localDevice->deletePath($entry); + } + } + } + + Console::success("Maintanance task finished."); + }); + + Console::success('Maintenance interval started.'); + + /** + * Get usage stats every X seconds to update swoole table + */ + Console::info('Starting stats interval...'); + $getStats = function (): void { + // Get usage stats + $usage = new Usage($this->orchestration); + $usage->run(); + $this->stats->updateStats($usage); + }; + + // Load initial stats in blocking way + $getStats(); + + // Setup infinite recursion in non-blocking way + \go(fn () => Timer::after(1000, fn () => $getStats())); + + Console::success('Stats interval started.'); + + Process::signal(SIGINT, fn () => $this->cleanUp($this->networks)); + Process::signal(SIGQUIT, fn () => $this->cleanUp($this->networks)); + Process::signal(SIGKILL, fn () => $this->cleanUp($this->networks)); + Process::signal(SIGTERM, fn () => $this->cleanUp($this->networks)); + } + + /** + * @param string $runtimeId + * @param int $timeout + * @param Response $response + * @param Log $log + * @return void + */ + public function getLogs(string $runtimeId, int $timeout, Response $response, Log $log): void + { + $runtimeName = System::getHostname() . '-' . $runtimeId; + + $tmpFolder = "tmp/$runtimeName/"; + $tmpLogging = "/{$tmpFolder}logging"; // Build logs + + // Wait for runtime (pod) to exist + for ($i = 0; $i < 10; $i++) { + try { + $runtime = $this->activeRuntimes->get($runtimeName); + if (!empty($runtime)) { + break; + } + } catch (\Throwable $th) { + } + + if ($i === 9) { + throw new \Exception('Runtime not ready. Pod not found.'); + } + + \usleep(500000); // 0.5s + } + + // Wait for state + $version = null; + $checkStart = \microtime(true); + while (true) { + if (\microtime(true) - $checkStart >= 10) { // Enforced timeout of 10s + throw new Exception(Exception::RUNTIME_TIMEOUT); + } + + $runtime = $this->activeRuntimes->get($runtimeName); + if (!empty($runtime)) { + $version = $runtime['version']; + break; + } + + \usleep(500000); // 0.5s + } + + if ($version === 'v2') { + return; + } + + // Wait for logging files + $checkStart = \microtime(true); + while (true) { + if (\microtime(true) - $checkStart >= $timeout) { + throw new Exception(Exception::LOGS_TIMEOUT); + } + + if (\file_exists($tmpLogging . '/logs.txt') && \file_exists($tmpLogging . '/timings.txt')) { + $timings = \file_get_contents($tmpLogging . '/timings.txt') ?: ''; + if (\strlen($timings) > 0) { + break; + } + } + + // Ensure runtime is still present + $runtime = $this->activeRuntimes->get($runtimeName); + if (empty($runtime)) { + return; + } + + \usleep(500000); // 0.5s + } + + /** + * @var mixed $logsChunk + */ + $logsChunk = ''; + + /** + * @var mixed $logsProcess + */ + $logsProcess = null; + + $streamInterval = 1000; // 1 second + $activeRuntimes = $this->activeRuntimes; + $timerId = Timer::tick($streamInterval, function () use (&$logsProcess, &$logsChunk, $response, $activeRuntimes, $runtimeName) { + $runtime = $activeRuntimes->get($runtimeName); + if ($runtime['initialised'] === 1) { + if (!empty($logsChunk)) { + $write = $response->write($logsChunk); + $logsChunk = ''; + } + + if (!empty($logsProcess)) { + \proc_terminate($logsProcess, 9); + } + return; + } + + if (empty($logsChunk)) { + return; + } + + $write = $response->write($logsChunk); + $logsChunk = ''; + + if (!$write) { + if (!empty($logsProcess)) { + \proc_terminate($logsProcess, 9); + } + } + }); + + $descriptor = [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ]; + + $logsProcess = \proc_open("tail -f -n +1 {$tmpLogging}/logs.txt", $descriptor, $pipes); + + $read = [$pipes[1]]; + $write = null; + $except = null; + + while (true) { + $runtime = $activeRuntimes->get($runtimeName); + if (empty($runtime)) { + break; + } + + $modified = \stream_select($read, $write, $except, 1); + + if ($modified > 0) { + $chunk = \fgets($pipes[1], 1024); + + if ($chunk !== false) { + $logsChunk .= $chunk; + } + } + + if ($runtime['initialised'] === 1) { + break; + } + + $read = [$pipes[1]]; + } + + if ($timerId !== false) { + Timer::clear($timerId); + } + + \fclose($pipes[0]); + \fclose($pipes[1]); + \fclose($pipes[2]); + if (\is_resource($logsProcess)) { + \proc_terminate($logsProcess, 9); + \proc_close($logsProcess); + } + } + + /** + * @param string $runtimeId + * @param string $command + * @param int $timeout + * @return string + */ + public function executeCommand(string $runtimeId, string $command, int $timeout): string + { + $runtimeName = System::getHostname() . '-' . $runtimeId; + + if (!$this->activeRuntimes->exists($runtimeName)) { + throw new Exception(Exception::RUNTIME_NOT_FOUND); + } + + $commands = ['sh', '-c', $command]; + $output = ''; + + try { + $this->orchestration->execute($runtimeName, $commands, $output, [], $timeout); + return $output; + } catch (TimeoutException $e) { + throw new Exception(Exception::COMMAND_TIMEOUT, previous: $e); + } catch (OrchestrationException $e) { + throw new Exception(Exception::COMMAND_FAILED, previous: $e); + } + } + + /** + * @param string $runtimeId + * @param string $secret + * @param string $image + * @param string $entrypoint + * @param string $source + * @param string $destination + * @param string[] $variables + * @param string $runtimeEntrypoint + * @param string $command + * @param int $timeout + * @param bool $remove + * @param float $cpus + * @param int $memory + * @param string $version + * @param string $restartPolicy + * @param Log $log + * @return mixed + */ + public function createRuntime( + string $runtimeId, + string $secret, + string $image, + string $entrypoint, + string $source, + string $destination, + array $variables, + string $runtimeEntrypoint, + string $command, + int $timeout, + bool $remove, + float $cpus, + int $memory, + string $version, + string $restartPolicy, + Log $log, + string $region = '', + ): mixed { + $runtimeName = System::getHostname() . '-' . $runtimeId; + $runtimeHostname = \bin2hex(\random_bytes(16)); + + if ($this->activeRuntimes->exists($runtimeName)) { + if ($this->activeRuntimes->get($runtimeName)['status'] == 'pending') { + throw new Exception(Exception::RUNTIME_CONFLICT, 'A runtime with the same ID is already being created. Attempt a execution soon.'); + } + + throw new Exception(Exception::RUNTIME_CONFLICT); + } + + $container = []; + $output = []; + $startTime = \microtime(true); + + $this->activeRuntimes->set($runtimeName, [ + 'version' => $version, + 'listening' => 0, + 'name' => $runtimeName, + 'hostname' => $runtimeHostname, + 'created' => $startTime, + 'updated' => $startTime, + 'status' => 'pending', + 'key' => $secret, + 'image' => $image, + 'initialised' => 0, + ]); + + /** + * Temporary file paths in the executor + */ + $buildFile = "code.tar.gz"; + if (($variables['OPEN_RUNTIMES_BUILD_COMPRESSION'] ?? '') === 'none') { + $buildFile = "code.tar"; + } + + $sourceFile = "code.tar.gz"; + if (!empty($source) && \pathinfo($source, PATHINFO_EXTENSION) === 'tar') { + $sourceFile = "code.tar"; + } + + $tmpFolder = "tmp/$runtimeName/"; + $tmpSource = "/{$tmpFolder}src/$sourceFile"; + $tmpBuild = "/{$tmpFolder}builds/$buildFile"; + $tmpLogging = "/{$tmpFolder}logging"; // Build logs + $tmpLogs = "/{$tmpFolder}logs"; // Runtime logs + + $sourceDevice = $this->getStorageDevice("/"); + $localDevice = new Local(); + + try { + /** + * Copy code files from source to a temporary location on the executor + */ + if (!empty($source)) { + if (!$sourceDevice->transfer($source, $tmpSource, $localDevice)) { + throw new \Exception('Failed to copy source code to temporary directory'); + }; + } + + /** + * Create the mount folder + */ + if (!$localDevice->createDirectory(\dirname($tmpBuild))) { + throw new \Exception("Failed to create temporary directory"); + } + + $this->orchestration + ->setCpus($cpus) + ->setMemory($memory); + + if (empty($runtimeEntrypoint)) { + if ($version === 'v2' && empty($command)) { + $runtimeEntrypointCommands = []; + } else { + $runtimeEntrypointCommands = ['tail', '-f', '/dev/null']; + } + } else { + $runtimeEntrypointCommands = ['bash', '-c', $runtimeEntrypoint]; + } + + $codeMountPath = $version === 'v2' ? '/usr/code' : '/mnt/code'; + $workdir = $version === 'v2' ? '/usr/code' : ''; + + $network = $this->networks[array_rand($this->networks)]; + + $volumes = [ + \dirname($tmpSource) . ':/tmp:rw', + \dirname($tmpBuild) . ':' . $codeMountPath . ':rw', + ]; + + if ($version === 'v5') { + $volumes[] = \dirname($tmpLogs . '/logs') . ':/mnt/logs:rw'; + $volumes[] = \dirname($tmpLogging . '/logging') . ':/tmp/logging:rw'; + } + + /** Keep the container alive if we have commands to be executed */ + $containerId = $this->orchestration->run( + image: $image, + name: $runtimeName, + command: $runtimeEntrypointCommands, + workdir: $workdir, + volumes: $volumes, + vars: $variables, + labels: [ + 'openruntimes-executor' => System::getHostname(), + 'openruntimes-runtime-id' => $runtimeId + ], + hostname: $runtimeHostname, + network: $network, + restart: $restartPolicy + ); + + if (empty($containerId)) { + throw new \Exception('Failed to create runtime'); + } + + /** + * Execute any commands if they were provided + */ + if (!empty($command)) { + if ($version === 'v2') { + $commands = [ + 'sh', + '-c', + 'touch /var/tmp/logs.txt && (' . $command . ') >> /var/tmp/logs.txt 2>&1 && cat /var/tmp/logs.txt' + ]; + } else { + $commands = [ + 'bash', + '-c', + 'mkdir -p /tmp/logging && touch /tmp/logging/timings.txt && touch /tmp/logging/logs.txt && script --log-out /tmp/logging/logs.txt --flush --log-timing /tmp/logging/timings.txt --return --quiet --command "' . \str_replace('"', '\"', $command) . '"' + ]; + } + + try { + $stdout = ''; + $status = $this->orchestration->execute( + name: $runtimeName, + command: $commands, + output: $stdout, + timeout: $timeout + ); + + if (!$status) { + throw new Exception(Exception::RUNTIME_FAILED, "Failed to create runtime: $stdout"); + } + + if ($version === 'v2') { + $stdout = \mb_substr($stdout ?: 'Runtime created successfully!', -MAX_BUILD_LOG_SIZE); // Limit to 1MB + $output[] = [ + 'timestamp' => Logs::getTimestamp(), + 'content' => $stdout + ]; + } else { + $output = Logs::get($runtimeName); + } + } catch (Throwable $err) { + throw new Exception(Exception::RUNTIME_FAILED, $err->getMessage(), null, $err); + } + } + + /** + * Move built code to expected build directory + */ + if (!empty($destination)) { + // Check if the build was successful by checking if file exists + if (!$localDevice->exists($tmpBuild)) { + throw new \Exception('Something went wrong when starting runtime.'); + } + + $size = $localDevice->getFileSize($tmpBuild); + $container['size'] = $size; + + $destinationDevice = $this->getStorageDevice($destination); + $path = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo($tmpBuild, PATHINFO_EXTENSION)); + + if (!$localDevice->transfer($tmpBuild, $path, $destinationDevice)) { + throw new \Exception('Failed to move built code to storage'); + }; + + $container['path'] = $path; + } + + $endTime = \microtime(true); + $duration = $endTime - $startTime; + + $container = array_merge($container, [ + 'output' => $output, + 'startTime' => $startTime, + 'duration' => $duration, + ]); + + $activeRuntime = $this->activeRuntimes->get($runtimeName); + $activeRuntime['updated'] = \microtime(true); + $activeRuntime['status'] = 'Up ' . \round($duration, 2) . 's'; + $activeRuntime['initialised'] = 1; + $this->activeRuntimes->set($runtimeName, $activeRuntime); + } catch (Throwable $th) { + if ($version === 'v2') { + $message = !empty($output) ? $output : $th->getMessage(); + try { + $logs = ''; + $status = $this->orchestration->execute( + name: $runtimeName, + command: ['sh', '-c', 'cat /var/tmp/logs.txt'], + output: $logs, + timeout: 15 + ); + + if (!empty($logs)) { + $message = $logs; + } + + $message = \mb_substr($message, -MAX_BUILD_LOG_SIZE); // Limit to 1MB + } catch (Throwable $err) { + // Ignore, use fallback error message + } + + $output = [ + 'timestamp' => Logs::getTimestamp(), + 'content' => $message + ]; + } else { + $output = Logs::get($runtimeName); + $output = \count($output) > 0 ? $output : [[ + 'timestamp' => Logs::getTimestamp(), + 'content' => $th->getMessage() + ]]; + } + + if ($remove) { + \sleep(2); // Allow time to read logs + } + + // Silently try to kill container + try { + $this->orchestration->remove($runtimeName, true); + } catch (Throwable $th) { + } + + $localDevice->deletePath($tmpFolder); + $this->activeRuntimes->del($runtimeName); + + $message = ''; + foreach ($output as $chunk) { + $message .= $chunk['content']; + } + + throw new \Exception($message, $th->getCode() ?: 500, $th); + } + + // Container cleanup + if ($remove) { + \sleep(2); // Allow time to read logs + + // Silently try to kill container + try { + $this->orchestration->remove($runtimeName, true); + } catch (Throwable $th) { + } + + $localDevice->deletePath($tmpFolder); + $this->activeRuntimes->del($runtimeName); + } + + // Remove weird symbol characters (for example from Next.js) + if (\is_array($container['output'])) { + foreach ($container['output'] as $index => &$chunk) { + $chunk['content'] = \mb_convert_encoding($chunk['content'] ?? '', 'UTF-8', 'UTF-8'); + } + } + + return $container; + } + + /** + * @param string $runtimeId + * @param Log $log + * @return void + */ + public function deleteRuntime(string $runtimeId, Log $log): void + { + $runtimeName = System::getHostname() . '-' . $runtimeId; + + if (!$this->activeRuntimes->exists($runtimeName)) { + throw new Exception(Exception::RUNTIME_NOT_FOUND); + } + + $this->orchestration->remove($runtimeName, true); + $this->activeRuntimes->del($runtimeName); + } + + /** + * @param string $runtimeId + * @param string|null $payload + * @param string $path + * @param string $method + * @param mixed $headers + * @param int $timeout + * @param string $image + * @param string $source + * @param string $entrypoint + * @param mixed $variables + * @param float $cpus + * @param int $memory + * @param string $version + * @param string $runtimeEntrypoint + * @param bool $logging + * @param string $restartPolicy + * @param Log $log + * @return mixed + * @throws Exception + */ + public function createExecution( + string $runtimeId, + ?string $payload, + string $path, + string $method, + mixed $headers, + int $timeout, + string $image, + string $source, + string $entrypoint, + mixed $variables, + float $cpus, + int $memory, + string $version, + string $runtimeEntrypoint, + bool $logging, + string $restartPolicy, + Log $log, + string $region = '', + ): mixed { + $runtimeName = System::getHostname() . '-' . $runtimeId; + + $variables = \array_merge($variables, [ + 'INERNAL_EXECUTOR_HOSTNAME' => System::getHostname() + ]); + + $prepareStart = \microtime(true); + + // Prepare runtime + if (!$this->activeRuntimes->exists($runtimeName)) { + if (empty($image) || empty($source)) { + throw new Exception(Exception::RUNTIME_NOT_FOUND, 'Runtime not found. Please start it first or provide runtime-related parameters.'); + } + + // Prepare request to executor + $sendCreateRuntimeRequest = function () use ($runtimeId, $image, $source, $entrypoint, $variables, $cpus, $memory, $version, $restartPolicy, $runtimeEntrypoint) { + $ch = \curl_init(); + + $body = \json_encode([ + 'runtimeId' => $runtimeId, + 'image' => $image, + 'source' => $source, + 'entrypoint' => $entrypoint, + 'variables' => $variables, + 'cpus' => $cpus, + 'memory' => $memory, + 'version' => $version, + 'restartPolicy' => $restartPolicy, + 'runtimeEntrypoint' => $runtimeEntrypoint + ]); + + \curl_setopt($ch, CURLOPT_URL, "http://127.0.0.1/v1/runtimes"); + \curl_setopt($ch, CURLOPT_POST, true); + \curl_setopt($ch, CURLOPT_POSTFIELDS, $body); + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + + \curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Content-Type: application/json', + 'Content-Length: ' . \strlen($body ?: ''), + 'authorization: Bearer ' . System::getEnv('OPR_EXECUTOR_SECRET', '') + ]); + + $executorResponse = \curl_exec($ch); + + $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); + + $error = \curl_error($ch); + + $errNo = \curl_errno($ch); + + \curl_close($ch); + + return [ + 'errNo' => $errNo, + 'error' => $error, + 'statusCode' => $statusCode, + 'executorResponse' => $executorResponse + ]; + }; + + // Prepare runtime + while (true) { + // If timeout is passed, stop and return error + if (\microtime(true) - $prepareStart >= $timeout) { + throw new Exception(Exception::RUNTIME_TIMEOUT); + } + + ['errNo' => $errNo, 'error' => $error, 'statusCode' => $statusCode, 'executorResponse' => $executorResponse] = \call_user_func($sendCreateRuntimeRequest); + + if ($errNo === 0) { + if (\is_string($executorResponse)) { + $body = \json_decode($executorResponse, true); + } else { + $body = []; + } + + if ($statusCode >= 500) { + // If the runtime has not yet attempted to start, it will return 500 + $error = $body['message']; + } elseif ($statusCode >= 400 && $statusCode !== 409) { + // If the runtime fails to start, it will return 400, except for 409 + // which indicates that the runtime is already being created + $error = $body['message']; + throw new \Exception('An internal curl error has occurred while starting runtime! Error Msg: ' . $error, 500); + } else { + break; + } + } elseif ($errNo !== 111) { + // Connection refused - see https://openswoole.com/docs/swoole-error-code + throw new \Exception('An internal curl error has occurred while starting runtime! Error Msg: ' . $error, 500); + } + + \usleep(500000); // 0.5s + } + } + + // Lower timeout by time it took to prepare container + $timeout -= (\microtime(true) - $prepareStart); + + // Update swoole table + $runtime = $this->activeRuntimes->get($runtimeName) ?? []; + + $runtime['updated'] = \time(); + $this->activeRuntimes->set($runtimeName, $runtime); + + // Ensure runtime started + $launchStart = \microtime(true); + while (true) { + // If timeout is passed, stop and return error + if (\microtime(true) - $launchStart >= $timeout) { + throw new Exception(Exception::RUNTIME_TIMEOUT); + } + + if ($this->activeRuntimes->get($runtimeName)['status'] !== 'pending') { + break; + } + + \usleep(500000); // 0.5s + } + + // Lower timeout by time it took to launch container + $timeout -= (\microtime(true) - $launchStart); + + // Ensure we have secret + $runtime = $this->activeRuntimes->get($runtimeName); + $hostname = $runtime['hostname']; + $secret = $runtime['key']; + if (empty($secret)) { + throw new \Exception('Runtime secret not found. Please re-create the runtime.', 500); + } + + $executeV2 = function () use ($variables, $payload, $secret, $hostname, $timeout): array { + $statusCode = 0; + $errNo = -1; + $executorResponse = ''; + + $ch = \curl_init(); + + $body = \json_encode([ + 'variables' => $variables, + 'payload' => $payload, + 'headers' => [] + ], JSON_FORCE_OBJECT); + + \curl_setopt($ch, CURLOPT_URL, "http://" . $hostname . ":3000/"); + \curl_setopt($ch, CURLOPT_POST, true); + \curl_setopt($ch, CURLOPT_POSTFIELDS, $body); + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + \curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); + \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + + \curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Content-Type: application/json', + 'Content-Length: ' . \strlen($body ?: ''), + 'x-internal-challenge: ' . $secret, + 'host: null' + ]); + + $executorResponse = \curl_exec($ch); + + $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); + + $error = \curl_error($ch); + + $errNo = \curl_errno($ch); + + \curl_close($ch); + + if ($errNo !== 0) { + return [ + 'errNo' => $errNo, + 'error' => $error, + 'statusCode' => $statusCode, + 'body' => '', + 'logs' => '', + 'errors' => '', + 'headers' => [] + ]; + } + + // Extract response + $executorResponse = json_decode(\strval($executorResponse), false); + + $res = $executorResponse->response ?? ''; + if (is_array($res)) { + $res = json_encode($res, JSON_UNESCAPED_UNICODE); + } elseif (is_object($res)) { + $res = json_encode($res, JSON_UNESCAPED_UNICODE | JSON_FORCE_OBJECT); + } + + $stderr = $executorResponse->stderr ?? ''; + $stdout = $executorResponse->stdout ?? ''; + + return [ + 'errNo' => $errNo, + 'error' => $error, + 'statusCode' => $statusCode, + 'body' => $res, + 'logs' => $stdout, + 'errors' => $stderr, + 'headers' => [] + ]; + }; + + $executeV5 = function () use ($path, $method, $headers, $payload, $secret, $hostname, $timeout, $runtimeName, $logging): array { + $statusCode = 0; + $errNo = -1; + $executorResponse = ''; + + $ch = \curl_init(); + + $responseHeaders = []; + + if (!(\str_starts_with($path, '/'))) { + $path = '/' . $path; + } + + \curl_setopt($ch, CURLOPT_URL, "http://" . $hostname . ":3000" . $path); + \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); + \curl_setopt($ch, CURLOPT_NOBODY, \strtoupper($method) === 'HEAD'); + + if (!empty($payload)) { + \curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + } + + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + \curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($curl, $header) use (&$responseHeaders) { + $len = strlen($header); + $header = explode(':', $header, 2); + if (count($header) < 2) { // ignore invalid headers + return $len; + } + + $key = strtolower(trim($header[0])); + $value = trim($header[1]); + + if (\in_array($key, ['x-open-runtimes-log-id'])) { + $value = \urldecode($value); + } + + if (\array_key_exists($key, $responseHeaders)) { + if (is_array($responseHeaders[$key])) { + $responseHeaders[$key][] = $value; + } else { + $responseHeaders[$key] = [$responseHeaders[$key], $value]; + } + } else { + $responseHeaders[$key] = $value; + } + + return $len; + }); + + \curl_setopt($ch, CURLOPT_TIMEOUT, $timeout + 5); // Gives extra 5s after safe timeout to recieve response + \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5); + if ($logging === true) { + $headers['x-open-runtimes-logging'] = 'enabled'; + } else { + $headers['x-open-runtimes-logging'] = 'disabled'; + } + + $headers['Authorization'] = 'Basic ' . \base64_encode('opr:' . $secret); + $headers['x-open-runtimes-secret'] = $secret; + + $headers['x-open-runtimes-timeout'] = \max(\intval($timeout), 1); + $headersArr = []; + foreach ($headers as $key => $value) { + $headersArr[] = $key . ': ' . $value; + } + + \curl_setopt($ch, CURLOPT_HEADEROPT, CURLHEADER_UNIFIED); + \curl_setopt($ch, CURLOPT_HTTPHEADER, $headersArr); + + $executorResponse = \curl_exec($ch); + + $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE); + + $error = \curl_error($ch); + + $errNo = \curl_errno($ch); + + \curl_close($ch); + + if ($errNo !== 0) { + return [ + 'errNo' => $errNo, + 'error' => $error, + 'statusCode' => $statusCode, + 'body' => '', + 'logs' => '', + 'errors' => '', + 'headers' => $responseHeaders + ]; + } + + // Extract logs and errors from file based on fileId in header + $fileId = $responseHeaders['x-open-runtimes-log-id'] ?? ''; + if (\is_array($fileId)) { + $fileId = $fileId[0] ?? ''; + } + $logs = ''; + $errors = ''; + if (!empty($fileId)) { + $logFile = '/tmp/' . $runtimeName . '/logs/' . $fileId . '_logs.log'; + $errorFile = '/tmp/' . $runtimeName . '/logs/' . $fileId . '_errors.log'; + + $logDevice = new Local(); + + if ($logDevice->exists($logFile)) { + if ($logDevice->getFileSize($logFile) > MAX_LOG_SIZE) { + $maxToRead = MAX_LOG_SIZE; + $logs = $logDevice->read($logFile, 0, $maxToRead); + $logs .= "\nLog file has been truncated to " . number_format(MAX_LOG_SIZE / 1048576, 2) . "MB."; + } else { + $logs = $logDevice->read($logFile); + } + + $logDevice->delete($logFile); + } + + if ($logDevice->exists($errorFile)) { + if ($logDevice->getFileSize($errorFile) > MAX_LOG_SIZE) { + $maxToRead = MAX_LOG_SIZE; + $errors = $logDevice->read($errorFile, 0, $maxToRead); + $errors .= "\nError file has been truncated to " . number_format(MAX_LOG_SIZE / 1048576, 2) . "MB."; + } else { + $errors = $logDevice->read($errorFile); + } + + $logDevice->delete($errorFile); + } + } + + $outputHeaders = []; + foreach ($responseHeaders as $key => $value) { + if (\str_starts_with($key, 'x-open-runtimes-')) { + continue; + } + + $outputHeaders[$key] = $value; + } + + return [ + 'errNo' => $errNo, + 'error' => $error, + 'statusCode' => $statusCode, + 'body' => $executorResponse, + 'logs' => $logs, + 'errors' => $errors, + 'headers' => $outputHeaders + ]; + }; + + // From here we calculate billable duration of execution + $startTime = \microtime(true); + + $listening = $runtime['listening']; + + if (empty($listening)) { + // Wait for cold-start to finish (app listening on port) + $pingStart = \microtime(true); + $validator = new TCP(); + while (true) { + // If timeout is passed, stop and return error + if (\microtime(true) - $pingStart >= $timeout) { + throw new Exception(Exception::RUNTIME_TIMEOUT); + } + + $online = $validator->isValid($hostname . ':' . 3000); + if ($online) { + break; + } + + \usleep(500000); // 0.5s + } + + // Update swoole table + $runtime = $this->activeRuntimes->get($runtimeName); + $runtime['listening'] = 1; + $this->activeRuntimes->set($runtimeName, $runtime); + + // Lower timeout by time it took to cold-start + $timeout -= (\microtime(true) - $pingStart); + } + + // Execute function + $executionRequest = $version === 'v2' ? $executeV2 : $executeV5; + + $retryDelayMs = \intval(System::getEnv('OPR_EXECUTOR_RETRY_DELAY_MS', '500')); + $retryAttempts = \intval(System::getEnv('OPR_EXECUTOR_RETRY_ATTEMPTS', '5')); + + $attempts = 0; + do { + $executionResponse = \call_user_func($executionRequest); + if ($executionResponse['errNo'] === CURLE_OK) { + break; + } + + // Not retryable, return error immediately + if (!in_array($executionResponse['errNo'], [ + CURLE_COULDNT_RESOLVE_HOST, // 6 + CURLE_COULDNT_CONNECT, // 7 + ])) { + break; + } + + usleep($retryDelayMs * 1000); + } while ((++$attempts < $retryAttempts) || (\microtime(true) - $startTime < $timeout)); + + // Error occurred + if ($executionResponse['errNo'] !== CURLE_OK) { + // Intended timeout error for v2 functions + if ($version === 'v2' && $executionResponse['errNo'] === SOCKET_ETIMEDOUT) { + throw new Exception(Exception::EXECUTION_TIMEOUT, $executionResponse['error'], 400); + } + + throw new \Exception('Internal curl error has occurred within the executor! Error Number: ' . $executionResponse['errNo'], 500); + } + + // Successful execution + ['statusCode' => $statusCode, 'body' => $body, 'logs' => $logs, 'errors' => $errors, 'headers' => $headers] = $executionResponse; + + $endTime = \microtime(true); + $duration = $endTime - $startTime; + + if ($version === 'v2') { + $logs = \mb_strcut($logs, 0, MAX_BUILD_LOG_SIZE); + $errors = \mb_strcut($errors, 0, MAX_BUILD_LOG_SIZE); + } + + $execution = [ + 'statusCode' => $statusCode, + 'headers' => $headers, + 'body' => $body, + 'logs' => $logs, + 'errors' => $errors, + 'duration' => $duration, + 'startTime' => $startTime, + ]; + + // Update swoole table + $runtime = $this->activeRuntimes->get($runtimeName); + $runtime['updated'] = \microtime(true); + $this->activeRuntimes->set($runtimeName, $runtime); + + return $execution; + } + + /** + * @param string[] $networks + * @return void + */ + private function cleanUp(array $networks = []): void + { + Console::log('Cleaning up pods and networks...'); + + $functionsToRemove = $this->orchestration->list(['label' => 'openruntimes-executor=' . System::getHostname()]); + + if (\count($functionsToRemove) === 0) { + Console::info('No pods found to clean up.'); + } + + $jobsRuntimes = []; + foreach ($functionsToRemove as $container) { + $jobsRuntimes[] = function () use ($container) { + try { + $this->orchestration->remove($container->getId(), true); + + $activeRuntimeId = $container->getName(); + + if (!$this->activeRuntimes->exists($activeRuntimeId)) { + $this->activeRuntimes->del($activeRuntimeId); + } + + Console::success('Removed pod ' . $container->getName()); + } catch (\Throwable $th) { + Console::error('Failed to remove pod: ' . $container->getName()); + Console::error($th); + } + }; + } + batch($jobsRuntimes); + + $jobsNetworks = []; + foreach ($networks as $network) { + $jobsNetworks[] = function () use ($network) { + try { + $this->orchestration->removeNetwork($network); + Console::success("Removed network: $network"); + } catch (Exception $e) { + Console::error("Failed to remove network $network: " . $e->getMessage()); + } + }; + } + batch($jobsNetworks); + + Console::success('Cleanup finished.'); + } + + /** + * @param string[] $networks + * @return string[] + */ + private function createNetworks(array $networks): array + { + $jobs = []; + $createdNetworks = []; + foreach ($networks as $network) { + $jobs[] = function () use ($network, &$createdNetworks) { + if (!$this->orchestration->networkExists($network)) { + try { + $this->orchestration->createNetwork($network, false); + Console::success("Created network: $network"); + $createdNetworks[] = $network; + } catch (\Throwable $e) { + Console::error("Failed to create network $network: " . $e->getMessage()); + } + } else { + Console::info("Network $network already exists"); + $createdNetworks[] = $network; + } + }; + } + batch($jobs); + + Console::info('Kubernetes runner does not require executor network connection.'); + + return $createdNetworks; + } + + public function getRuntimes(): mixed + { + $runtimes = []; + foreach ($this->activeRuntimes as $runtime) { + $runtimes[] = $runtime; + } + return $runtimes; + } + + public function getRuntime(string $name): mixed + { + if (!$this->activeRuntimes->exists($name)) { + throw new Exception(Exception::RUNTIME_NOT_FOUND); + } + + return $this->activeRuntimes->get($name); + } + + public function getStats(): Stats + { + return $this->stats; + } +} diff --git a/test-k8s-quick.sh b/test-k8s-quick.sh new file mode 100755 index 0000000..d69728d --- /dev/null +++ b/test-k8s-quick.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +set -e + +# Configuration +EXECUTOR_URL="http://localhost:8080" +SECRET_KEY="local-dev-secret-key" +RUNTIME_ID="test-$(date +%s)" + +echo "🚀 Testing Kubernetes Executor" +echo "Runtime ID: $RUNTIME_ID" +echo "" + +# Health check +echo "📊 Health Check" +curl -s -X GET "${EXECUTOR_URL}/v1/health" \ + -H "Authorization: Bearer ${SECRET_KEY}" | jq . +echo "" + +# List runtimes (should be empty) +echo "📋 List Runtimes (should be empty)" +curl -s -X GET "${EXECUTOR_URL}/v1/runtimes" \ + -H "Authorization: Bearer ${SECRET_KEY}" | jq . +echo "" + +# Create runtime +echo "🔧 Create Runtime" +curl -s -X POST "${EXECUTOR_URL}/v1/runtimes" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${SECRET_KEY}" \ + -d "{ + \"runtimeId\": \"${RUNTIME_ID}\", + \"image\": \"openruntimes/php:v5-8.3\", + \"entrypoint\": \"index.php\", + \"version\": \"v5\", + \"cpus\": 0.5, + \"memory\": 256, + \"timeout\": 60 + }" | jq . +echo "" + +# Wait for runtime +echo "⏳ Waiting for runtime to be ready..." +sleep 3 + +# Get runtime details +echo "🔍 Get Runtime Details" +curl -s -X GET "${EXECUTOR_URL}/v1/runtimes/${RUNTIME_ID}" \ + -H "Authorization: Bearer ${SECRET_KEY}" | jq . +echo "" + +# Execute command +echo "💻 Execute Command" +curl -s -X POST "${EXECUTOR_URL}/v1/runtimes/${RUNTIME_ID}/commands" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${SECRET_KEY}" \ + -d '{ + "command": "php -v", + "timeout": 10 + }' | jq . +echo "" + +# Delete runtime +echo "🗑️ Delete Runtime" +curl -s -X DELETE "${EXECUTOR_URL}/v1/runtimes/${RUNTIME_ID}" \ + -H "Authorization: Bearer ${SECRET_KEY}" +echo "" + +# Verify deletion +echo "✅ Verify Deletion" +sleep 2 +curl -s -X GET "${EXECUTOR_URL}/v1/runtimes" \ + -H "Authorization: Bearer ${SECRET_KEY}" | jq . +echo "" + +echo "✨ Test complete!"