Skip to content

feat(syncctl): make sync pod resource limits configurable via env vars #1241

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mpanius wants to merge 1 commit into
base: newjitsu
Choose a base branch
from
Open

feat(syncctl): make sync pod resource limits configurable via env vars #1241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions bulker/sync-controller/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,15 @@ Request body:
| `SYNCCTL_CONTAINER_INIT_TIMEOUT_SECONDS` | no | timeout for Pod initialization | `180` |
| `SYNCCTL_SIDECAR_DATABASE_URL` | no | Must lead to the same db instance as `SYNCCTL_DATABASE_URL`. Use it when in k8s environment database is reachable through a different hostname or IP. | =`SYNCCTL_DATABASE_URL` |
| `SYNCCTL_INSTANCE_ID` | no | ID of syncctl instance. It is used for metrics. If is not set, instance id will be generated and persisted to disk (`~/.syncctl/instance_id`) and reused on next restart. | random uuid |
| `SYNCCTL_SOURCE_CPU_REQUEST_MILLICORES` | no | CPU request for source container in millicores | `100` |
| `SYNCCTL_SOURCE_CPU_LIMIT_MILLICORES` | no | CPU limit for source container in millicores | `1000` |
| `SYNCCTL_SOURCE_MEMORY_REQUEST_MI` | no | Memory request for source container in MiB | `256` |
| `SYNCCTL_SOURCE_MEMORY_LIMIT_MI` | no | Memory limit for source container in MiB. Must be adjusted together with `SOURCE_JAVA_OPTS` — JVM heap (Xmx) should be ~1Gi less than memory limit. E.g.: 2048 MiB limit → `-Xmx1500m`, 4096 MiB limit → `-Xmx3500m` | `8192` |
| `SYNCCTL_SOURCE_JAVA_OPTS` | no | JAVA_OPTS for source container. Controls JVM heap size. Must be consistent with `SOURCE_MEMORY_LIMIT_MI` | `-Xmx7000m` |
| `SYNCCTL_SIDECAR_CPU_REQUEST_MILLICORES` | no | CPU request for sidecar container in millicores | `0` |
| `SYNCCTL_SIDECAR_CPU_LIMIT_MILLICORES` | no | CPU limit for sidecar container in millicores | `500` |
| `SYNCCTL_SIDECAR_MEMORY_REQUEST_MI` | no | Memory request for sidecar container in MiB | `0` |
| `SYNCCTL_SIDECAR_MEMORY_LIMIT_MI` | no | Memory limit for sidecar container in MiB | `4096` |


## Database schema
Expand Down
28 changes: 28 additions & 0 deletions bulker/sync-controller/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,34 @@ type Config struct {

TaskTimeoutHours int `mapstructure:"TASK_TIMEOUT_HOURS" default:"48"`

// # Sync Pod Resources - resource limits and requests for sync task pods.
// Allows overriding the default resource allocations for source and sidecar containers
// spawned by syncctl. Values in millicores for CPU and mebibytes (MiB) for memory.

// SourceCPURequest CPU request for the source (Airbyte connector) container in millicores. Default: 100
SourceCPURequest int `mapstructure:"SOURCE_CPU_REQUEST_MILLICORES" default:"100"`
// SourceCPULimit CPU limit for the source (Airbyte connector) container in millicores. Default: 1000
SourceCPULimit int `mapstructure:"SOURCE_CPU_LIMIT_MILLICORES" default:"1000"`
// SourceMemoryRequest memory request for the source container in MiB. Default: 256
SourceMemoryRequest int `mapstructure:"SOURCE_MEMORY_REQUEST_MI" default:"256"`
// SourceMemoryLimit memory limit for the source container in MiB. Default: 8192 (8Gi).
// NOTE: must be adjusted together with SourceJavaOpts — JVM heap (Xmx) should be ~1Gi less
// than the memory limit to leave room for non-heap JVM memory (metaspace, thread stacks, etc).
// Example: 2048 MiB limit → -Xmx1500m, 4096 MiB limit → -Xmx3500m
SourceMemoryLimit int `mapstructure:"SOURCE_MEMORY_LIMIT_MI" default:"8192"`
// SourceJavaOpts JAVA_OPTS env var for the source container (controls JVM heap). Default: -Xmx7000m.
// Must be consistent with SourceMemoryLimit — see note above.
SourceJavaOpts string `mapstructure:"SOURCE_JAVA_OPTS" default:"-Xmx7000m"`

// SidecarCPURequest CPU request for the sidecar container in millicores. Default: 0
SidecarCPURequest int `mapstructure:"SIDECAR_CPU_REQUEST_MILLICORES" default:"0"`
// SidecarCPULimit CPU limit for the sidecar container in millicores. Default: 500
SidecarCPULimit int `mapstructure:"SIDECAR_CPU_LIMIT_MILLICORES" default:"500"`
// SidecarMemoryRequest memory request for the sidecar container in MiB. Default: 0
SidecarMemoryRequest int `mapstructure:"SIDECAR_MEMORY_REQUEST_MI" default:"0"`
// SidecarMemoryLimit memory limit for the sidecar container in MiB. Default: 4096 (4Gi)
SidecarMemoryLimit int `mapstructure:"SIDECAR_MEMORY_LIMIT_MI" default:"4096"`

SidecarImage string `mapstructure:"SIDECAR_IMAGE" default:"jitsucom/sidecar:latest"`
PodsServiceAccount string `mapstructure:"PODS_SERVICE_ACCOUNT"`

Expand Down
22 changes: 9 additions & 13 deletions bulker/sync-controller/job_runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -841,19 +841,17 @@ func (j *JobRunner) createPod(podName string, task TaskDescriptor, configuration
Command: []string{"sh", "-c", fmt.Sprintf("eval \"$AIRBYTE_ENTRYPOINT %s\" 2> /pipes/stderr > /pipes/stdout", command)},
Env: []v1.EnvVar{{Name: "USE_STREAM_CAPABLE_STATE", Value: "true"},
{Name: "AUTO_DETECT_SCHEMA", Value: "true"},
{Name: "JAVA_OPTS", Value: "-Xmx7000m"}},
{Name: "JAVA_OPTS", Value: j.config.SourceJavaOpts}},

VolumeMounts: volumeMounts,
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(int64(1000), resource.DecimalSI),
// 8Gi
v1.ResourceMemory: *resource.NewQuantity(int64(math.Pow(2, 33)), resource.BinarySI),
v1.ResourceCPU: *resource.NewMilliQuantity(int64(j.config.SourceCPULimit), resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(int64(float64(j.config.SourceMemoryLimit)*math.Pow(2, 20)), resource.BinarySI),
},
Requests: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(int64(100), resource.DecimalSI),
// 256Mi
v1.ResourceMemory: *resource.NewQuantity(int64(math.Pow(2, 28)), resource.BinarySI),
v1.ResourceCPU: *resource.NewMilliQuantity(int64(j.config.SourceCPURequest), resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(int64(float64(j.config.SourceMemoryRequest)*math.Pow(2, 20)), resource.BinarySI),
},
},
},
Expand All @@ -865,14 +863,12 @@ func (j *JobRunner) createPod(podName string, task TaskDescriptor, configuration
VolumeMounts: volumeMounts,
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(int64(500), resource.DecimalSI),
// 4Gi
v1.ResourceMemory: *resource.NewQuantity(int64(math.Pow(2, 32)), resource.BinarySI),
v1.ResourceCPU: *resource.NewMilliQuantity(int64(j.config.SidecarCPULimit), resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(int64(float64(j.config.SidecarMemoryLimit)*math.Pow(2, 20)), resource.BinarySI),
},
Requests: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(int64(0), resource.DecimalSI),
// 256
v1.ResourceMemory: *resource.NewQuantity(int64(0), resource.BinarySI),
v1.ResourceCPU: *resource.NewMilliQuantity(int64(j.config.SidecarCPURequest), resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(int64(float64(j.config.SidecarMemoryRequest)*math.Pow(2, 20)), resource.BinarySI),
},
},
},
Expand Down