Merge commit 'f745a1155953145d26737a7ec8746a9974b15ba5'

Author: Johanna Smith

.gitmodules

@@ -22,9 +22,6 @@
 [submodule "src/google.golang.org/grpc"]
 	path = src/google.golang.org/grpc
 	url = https://github.com/grpc/grpc-go
-[submodule "src/github.com/cloudfoundry/loggregator-api"]
-	path = src/github.com/cloudfoundry/loggregator-api
-	url = https://github.com/cloudfoundry/loggregator-api
 [submodule "src/github.com/crewjam/rfc5424"]
 	path = src/github.com/crewjam/rfc5424
 	url = https://github.com/crewjam/rfc5424
@@ -37,3 +34,18 @@
 [submodule "src/github.com/cloudfoundry/diodes"]
 	path = src/github.com/cloudfoundry/diodes
 	url = https://github.com/cloudfoundry/diodes
+[submodule "src/code.cloudfoundry.org/rfc5424"]
+	path = src/code.cloudfoundry.org/rfc5424
+	url = https://github.com/cloudfoundry-incubator/rfc5424
+[submodule "src/github.com/cloudfoundry/dropsonde"]
+	path = src/github.com/cloudfoundry/dropsonde
+	url = https://github.com/cloudfoundry/dropsonde
+[submodule "src/github.com/cloudfoundry/sonde-go"]
+	path = src/github.com/cloudfoundry/sonde-go
+	url = https://github.com/cloudfoundry/sonde-go
+[submodule "src/github.com/gogo/protobuf"]
+	path = src/github.com/gogo/protobuf
+	url = https://github.com/gogo/protobuf
+[submodule "src/github.com/nu7hatch/gouuid"]
+	path = src/github.com/nu7hatch/gouuid
+	url = https://github.com/nu7hatch/gouuid

README.md

@@ -1,4 +1,4 @@
-# Scalable Syslog Release [![slack.cloudfoundry.org][slack-badge]][loggregator-slack]
+# Scalable Syslog Release [![slack.cloudfoundry.org][slack-badge]][loggregator-slack] [![CI Badge][ci-badge]][ci-pipeline]
 
 Scalable syslog is a [Bosh][bosh] release that works in conjunction with
 [Loggregator][loggregator] to bind applications to syslog readers. It can be
@@ -108,3 +108,5 @@ Assuming you have these two CAs, run the following commands:
 [cf-deployment]:            https://code.cloudfoundry.org/cf-deployment
 [sample-manifest]:          https://code.cloudfoundry.org/scalable-syslog-release/blob/master/manifests/scalable-syslog.yml
 [common-cloud-config]:      https://code.cloudfoundry.org/bosh-deployment/blob/master/warden/cloud-config.yml
+[ci-badge]:                 https://loggregator.ci.cf-app.com/api/v1/pipelines/scalable-syslog/jobs/tests/badge
+[ci-pipeline]:              https://loggregator.ci.cf-app.com/teams/main/pipelines/scalable-syslog

jobs/scheduler/spec

@@ -15,6 +15,7 @@ packages:
 consumes:
 - name: adapter_addrs
   type: addr
+  optional: true
 
 properties:
   scalablesyslog.scheduler.require_opt_in:
@@ -29,6 +30,15 @@ properties:
     description: "The port for pprof on the scheduler"
     default: 0
 
+  scalablesyslog.adapter.addrs:
+    description: |
+      A collection of adapter addresses. This is used only if the
+      adapter_adders link is not provided.
+    default: []
+  scalablesyslog.adapter.port:
+    description: "The port used to communicate with the adapters via gRPC."
+    default: 4443
+
   scalablesyslog.scheduler.tls.client.ca:
     description: "The TLS CA for the scalable syslog"
   scalablesyslog.scheduler.tls.client.cert:

jobs/scheduler/templates/ctl.erb

@@ -23,6 +23,15 @@ ulimit -n 8192
   blacklisted_ips = blacklisted_ranges.map do |range|
     "#{range['start']}-#{range['end']}"
   end.join(",")
+  adapter_addrs = []
+  adapter_port = 0
+  if_link("adapter_addrs") do |l|
+    adapter_addrs = l.instances.map(&:address).join(",")
+    adapter_port = l.p('scalablesyslog.adapter.port')
+  end.else do
+    adapter_addrs = p('scalablesyslog.adapter.addrs').join(",")
+    adapter_port = p('scalablesyslog.adapter.port')
+  end
 %>
 
 echo $$ > $PIDFILE
@@ -41,8 +50,8 @@ exec chpst -u vcap:vcap ./scheduler \
   --api-cn=<%= p("scalablesyslog.scheduler.tls.api.cn") %> \
   --api-skip-cert-verify=<%= p("scalablesyslog.scheduler.tls.api.skip_cert_verify") %> \
   --api-polling-interval=<%= p("scalablesyslog.scheduler.api.polling_interval") %> \
-  --adapter-addrs=<%= link("adapter_addrs").instances.map(&:address).join(",") %> \
-  --adapter-port=<%= link("adapter_addrs").p('scalablesyslog.adapter.port') %> \
+  --adapter-addrs=<%= adapter_addrs %> \
+  --adapter-port=<%= adapter_port %> \
   --blacklist-ranges=<%= blacklisted_ips %> \
   --metric-ingress-addr=<%= p("scalablesyslog.scheduler.metric.addr") %> \
   --metric-ingress-cn=<%= p("scalablesyslog.scheduler.metric.cn") %> \

manifests/scalable-syslog.yml

@@ -55,6 +55,7 @@ instance_groups:
 - name: adapter
   azs:
   - z1
+  - z2
   instances: 3
   persistent_disk_type: 1GB
   vm_type: default

packages/adapter/spec

@@ -8,21 +8,7 @@ files:
 - code.cloudfoundry.org/go-loggregator/*.go # gosub
 - code.cloudfoundry.org/go-loggregator/pulseemitter/*.go # gosub
 - code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/genproto/googleapis/rpc/status/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/codes/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/credentials/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/grpclog/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/internal/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/keepalive/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/metadata/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/naming/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/peer/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/stats/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/status/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/tap/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/transport/*.go # gosub
+- code.cloudfoundry.org/rfc5424/*.go # gosub
 - code.cloudfoundry.org/scalable-syslog/adapter/*.go # gosub
 - code.cloudfoundry.org/scalable-syslog/adapter/app/*.go # gosub
 - code.cloudfoundry.org/scalable-syslog/adapter/internal/binding/*.go # gosub
@@ -34,7 +20,6 @@ files:
 - code.cloudfoundry.org/scalable-syslog/internal/diodes/*.go # gosub
 - code.cloudfoundry.org/scalable-syslog/internal/health/*.go # gosub
 - github.com/cloudfoundry/diodes/*.go # gosub
-- github.com/crewjam/rfc5424/*.go # gosub
 - github.com/golang/protobuf/proto/*.go # gosub
 - github.com/golang/protobuf/ptypes/any/*.go # gosub
 - golang.org/x/net/context/*.go # gosub

packages/fake_logs_provider/spec

@@ -8,21 +8,6 @@ files:
 - code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2/*.go # gosub
 - code.cloudfoundry.org/go-loggregator/testhelpers/*.go # gosub
 - code.cloudfoundry.org/go-loggregator/testhelpers/fakes/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/genproto/googleapis/rpc/status/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/codes/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/credentials/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/grpclog/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/internal/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/keepalive/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/metadata/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/naming/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/peer/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/stats/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/status/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/tap/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/transport/*.go # gosub
 - code.cloudfoundry.org/scalable-syslog/tools/fake_logs_provider/*.go # gosub
 - github.com/golang/protobuf/proto/*.go # gosub
 - github.com/golang/protobuf/ptypes/any/*.go # gosub
@@ -37,3 +22,17 @@ files:
 - golang.org/x/text/transform/*.go # gosub
 - golang.org/x/text/unicode/bidi/*.go # gosub
 - golang.org/x/text/unicode/norm/*.go # gosub
+- google.golang.org/genproto/googleapis/rpc/status/*.go # gosub
+- google.golang.org/grpc/*.go # gosub
+- google.golang.org/grpc/codes/*.go # gosub
+- google.golang.org/grpc/credentials/*.go # gosub
+- google.golang.org/grpc/grpclog/*.go # gosub
+- google.golang.org/grpc/internal/*.go # gosub
+- google.golang.org/grpc/keepalive/*.go # gosub
+- google.golang.org/grpc/metadata/*.go # gosub
+- google.golang.org/grpc/naming/*.go # gosub
+- google.golang.org/grpc/peer/*.go # gosub
+- google.golang.org/grpc/stats/*.go # gosub
+- google.golang.org/grpc/status/*.go # gosub
+- google.golang.org/grpc/tap/*.go # gosub
+- google.golang.org/grpc/transport/*.go # gosub

packages/scheduler/spec

@@ -8,21 +8,6 @@ files:
 - code.cloudfoundry.org/go-loggregator/*.go # gosub
 - code.cloudfoundry.org/go-loggregator/pulseemitter/*.go # gosub
 - code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/genproto/googleapis/rpc/status/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/codes/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/credentials/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/grpclb/grpc_lb_v1/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/grpclog/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/internal/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/keepalive/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/metadata/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/naming/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/peer/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/stats/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/status/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/tap/*.go # gosub
-- code.cloudfoundry.org/go-loggregator/vendor/google.golang.org/grpc/transport/*.go # gosub
 - code.cloudfoundry.org/scalable-syslog/internal/api/*.go # gosub
 - code.cloudfoundry.org/scalable-syslog/internal/api/v1/*.go # gosub
 - code.cloudfoundry.org/scalable-syslog/internal/health/*.go # gosub