[core] Disable system proxy by default for Mesos, DCS, DDscheduler

Author: teo

core/config.go

@@ -67,6 +67,7 @@ func setDefaults() error {
 	viper.SetDefault("mesosAuthMode", env("AUTH_MODE", ""))
 	viper.SetDefault("mesosCheckpoint", true)
 	viper.SetDefault("mesosCompression", false)
+	viper.SetDefault("mesosUseSystemProxy", false)
 	viper.SetDefault("mesosCredentials.username", env("AUTH_USER", ""))
 	viper.SetDefault("mesosCredentials.passwordFile", env("AUTH_PASSWORD_FILE", ""))
 	viper.SetDefault("mesosExecutorImage", env("EXEC_IMAGE", ""))
@@ -95,7 +96,9 @@ func setDefaults() error {
 	viper.SetDefault("dumpWorkflows", false)
 	viper.SetDefault("configServiceUri", "apricot://127.0.0.1:47101")
 	viper.SetDefault("dcsServiceEndpoint", "127.0.0.1:50051")
+	viper.SetDefault("dcsServiceUseSystemProxy", false)
 	viper.SetDefault("ddSchedulerEndpoint", "127.0.0.1:50052")
+	viper.SetDefault("ddSchedulerUseSystemProxy", false)
 	viper.SetDefault("integrationPlugins", []string{})
 	viper.SetDefault("coreConfigEntry", "settings")
 	viper.SetDefault("fmqPlugin", "OCClite")
@@ -115,6 +118,7 @@ func setFlags() error {
 	pflag.String("mesosAuthMode", viper.GetString("mesosAuthMode"), "Method to use for Mesos authentication; specify '"+schedutil.AuthModeBasic+"' for simple HTTP authentication")
 	pflag.Bool("mesosCheckpoint", viper.GetBool("mesosCheckpoint"), "Enable/disable agent checkpointing for framework tasks (recover from agent failure)")
 	pflag.Bool("mesosCompression", viper.GetBool("mesosCompression"), "When true attempt to use compression for HTTP streams")
+	pflag.Bool("mesosUseSystemProxy", viper.GetBool("mesosUseSystemProxy"), "When true the https_proxy, http_proxy and no_proxy environment variables are obeyed")
 	pflag.String("mesosExecutorImage", viper.GetString("mesosExecutorImage"), "Name of the docker image to run the executor")
 	pflag.Duration("mesosFailoverTimeout", viper.GetDuration("mesosFailoverTimeout"), "Framework failover timeout (recover from scheduler failure)")
 	pflag.String("mesosFrameworkHostname", viper.GetString("mesosFrameworkHostname"), "Framework hostname that is advertised to the master")
@@ -140,7 +144,9 @@ func setFlags() error {
 	pflag.Bool("dumpWorkflows", viper.GetBool("dumpWorkflows"), "Dump unprocessed and processed workflow files (`$PWD/wf-{,un}processed-<timestamp>.json`)")
 	pflag.String("configServiceUri", viper.GetString("configServiceUri"), "URI of the Apricot instance (`apricot://host:port`), Consul server (`consul://`) or YAML configuration file, entry point for all configuration")
 	pflag.String("dcsServiceEndpoint", viper.GetString("dcsServiceEndpoint"), "Endpoint of the DCS gRPC service (`host:port`)")
+	pflag.Bool("dcsServiceUseSystemProxy", viper.GetBool("dcsServiceUseSystemProxy"), "When true the https_proxy, http_proxy and no_proxy environment variables are obeyed")
 	pflag.String("ddSchedulerEndpoint", viper.GetString("ddSchedulerEndpoint"), "Endpoint of the DD scheduler gRPC service (`host:port`)")
+	pflag.Bool("ddSchedulerUseSystemProxy", viper.GetBool("ddSchedulerUseSystemProxy"), "When true the https_proxy, http_proxy and no_proxy environment variables are obeyed")
 	pflag.StringSlice("integrationPlugins", viper.GetStringSlice("integrationPlugins"), "List of integration plugins to load (default: empty)")
 	pflag.String("coreConfigEntry", viper.GetString("coreConfigEntry"), "key for AliECS core configuration within the `aliecs` component [EXPERT SETTING]")
 	pflag.String("fmqPlugin", viper.GetString("fmqPlugin"), "Name of the plugin for FairMQ tasks")

core/integration/dcs/client.go

@@ -30,6 +30,7 @@ import (
 	"github.com/AliceO2Group/Control/common/logger"
 	dcspb "github.com/AliceO2Group/Control/core/integration/dcs/protos"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 )
@@ -46,7 +47,18 @@ func NewClient(cxt context.Context, cancel context.CancelFunc, endpoint string)
 	log.WithFields(logrus.Fields{
 		"endpoint": endpoint,
 	}).Debug("dialing DCS client")
-	conn, err := grpc.DialContext(cxt, endpoint, grpc.WithInsecure(), grpc.WithBlock())
+
+	dialOptions := []grpc.DialOption {
+		grpc.WithInsecure(),
+		grpc.WithBlock(),
+	}
+	if !viper.GetBool("dcsServiceUseSystemProxy") {
+		dialOptions = append(dialOptions, grpc.WithNoProxy())
+	}
+	conn, err := grpc.DialContext(cxt,
+		endpoint,
+		dialOptions...,
+	)
 	if err != nil {
 		log.WithField("error", err.Error()).
 			WithField("endpoint", endpoint).

core/integration/ddsched/client.go

@@ -30,6 +30,7 @@ import (
 	"github.com/AliceO2Group/Control/common/logger"
 	ddpb "github.com/AliceO2Group/Control/core/integration/ddsched/protos"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 )
@@ -46,7 +47,18 @@ func NewClient(cxt context.Context, cancel context.CancelFunc, endpoint string)
 	log.WithFields(logrus.Fields{
 		"endpoint": endpoint,
 	}).Debug("dialing DD scheduler client")
-	conn, err := grpc.DialContext(cxt, endpoint, grpc.WithInsecure(), grpc.WithBlock())
+
+	dialOptions := []grpc.DialOption {
+		grpc.WithInsecure(),
+		grpc.WithBlock(),
+	}
+	if !viper.GetBool("ddSchedulerUseSystemProxy") {
+		dialOptions = append(dialOptions, grpc.WithNoProxy())
+	}
+	conn, err := grpc.DialContext(cxt,
+			endpoint,
+			dialOptions...,
+		)
 	if err != nil {
 		log.WithField("error", err.Error()).
 			WithField("endpoint", endpoint).

core/task/schedutil/mesosutil.go

@@ -30,20 +30,21 @@ package schedutil
 import (
 	"errors"
 	"fmt"
+	"net/http"
+	"net/url"
 	"path/filepath"
 	"time"
 
 	"github.com/AliceO2Group/Control/common/logger"
-	"github.com/mesos/mesos-go/api/v1/lib/encoding/codecs"
-	"github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
-
 	"github.com/AliceO2Group/Control/common/product"
 	"github.com/mesos/mesos-go/api/v1/lib"
+	"github.com/mesos/mesos-go/api/v1/lib/encoding/codecs"
 	"github.com/mesos/mesos-go/api/v1/lib/httpcli"
 	"github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched"
 	"github.com/mesos/mesos-go/api/v1/lib/resources"
 	"github.com/mesos/mesos-go/api/v1/lib/scheduler/calls"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
 	proto "google.golang.org/protobuf/proto"
 )
 
@@ -133,6 +134,13 @@ func BuildHTTPSched(creds credentials) calls.Caller {
 		httpcli.Do(httpcli.With(
 			authConfigOpt,
 			httpcli.Timeout(viper.GetDuration("mesosApiTimeout")),
+			httpcli.Transport(func(transport *http.Transport) {
+				if !viper.GetBool("mesosUseSystemProxy") {
+					transport.Proxy = func(request *http.Request) (*url.URL, error) {
+						return nil, nil
+					}
+				}
+			}),
 		)),
 	)
 	if viper.GetBool("mesosCompression") {