Skip to content

Commit 17b832c

Browse files
apollo13apollo13
committed
First stab at enabling ACLs for a nomad cluster.
1 parent 254d9af commit 17b832c

File tree

5 files changed

+134
-69
lines changed

5 files changed

+134
-69
lines changed

‎pkg/clients/nomad/nomad.go‎

Lines changed: 115 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import (
1818
//go:generate mockery --name Nomad --filename nomad.go
1919
type Nomad interface {
2020
// SetConfig for the client, path is a valid Nomad JSON config file
21-
SetConfig(address string, port, nodes int) error
21+
SetConfig(address string, port, nodes int, acl_token string) error
2222
// Create jobs in the provided files
2323
Create(files []string) error
2424
// Stop jobs in the provided files
@@ -30,9 +30,11 @@ type Nomad interface {
3030
// HealthCheckAPI uses the Nomad API to check that all servers and nodes
3131
// are ready. The function will block until either all nodes are healthy or the
3232
// timeout period elapses.
33-
HealthCheckAPI(time.Duration) error
33+
HealthCheckAPI(time.Duration, bool) error
3434
// Endpoints returns a list of endpoints for a cluster
3535
Endpoints(job, group, task string) ([]map[string]string, error)
36+
// Bootstrap ACLs
37+
BootstrapACLs() error
3638
}
3739

3840
// NomadImpl is an implementation of the Nomad interface
@@ -43,6 +45,7 @@ type NomadImpl struct {
4345
address string
4446
port int
4547
clientNodes int
48+
aclToken string
4649
}
4750

4851
// NewNomad creates a new Nomad client
@@ -59,17 +62,52 @@ type createRequest struct {
5962
Job string
6063
}
6164

65+
func (n *NomadImpl) setAuthHeaders(rq *http.Request) {
66+
if n.aclToken != "" {
67+
rq.Header.Set("X-Nomad-Token", n.aclToken)
68+
}
69+
}
70+
6271
// SetConfig loads the Nomad config from a file
63-
func (n *NomadImpl) SetConfig(address string, port, nodes int) error {
72+
func (n *NomadImpl) SetConfig(address string, port, nodes int, acl_token string) error {
6473
n.address = address
6574
n.port = port
6675
n.clientNodes = nodes
76+
n.aclToken = acl_token
6777

6878
return nil
6979
}
7080

71-
// HealthCheckAPI executes a HTTP heath check for a Nomad cluster
72-
func (n *NomadImpl) HealthCheckAPI(timeout time.Duration) error {
81+
func (n *NomadImpl) BootstrapACLs() error {
82+
if n.aclToken != "" {
83+
n.l.Debug("Bootstrapping ACLs", "address", n.address)
84+
85+
jsonBody := []byte(fmt.Sprintf(`{"BootstrapSecret":"%s"}`, n.aclToken))
86+
bodyReader := bytes.NewReader(jsonBody)
87+
88+
rq, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s:%d/v1/acl/bootstrap", n.address, n.port), bodyReader)
89+
if err != nil {
90+
return err
91+
}
92+
93+
resp, err := n.httpClient.Do(rq)
94+
if err != nil {
95+
return xerrors.Errorf("Unable to bootstrap ACLs: %w", err)
96+
}
97+
defer resp.Body.Close()
98+
99+
if resp.StatusCode != http.StatusOK {
100+
// try to read the body for the error
101+
d, _ := ioutil.ReadAll(resp.Body)
102+
return xerrors.Errorf("Error bootstrapping ACLs, got status code %d, error: %s", resp.StatusCode, string(d))
103+
}
104+
105+
}
106+
return nil
107+
}
108+
109+
// HealthCheckAPI executes a HTTP health check for a Nomad cluster
110+
func (n *NomadImpl) HealthCheckAPI(timeout time.Duration, simple bool) error {
73111
n.l.Debug("Performing Nomad health check", "address", n.address)
74112
st := time.Now()
75113
for {
@@ -79,75 +117,87 @@ func (n *NomadImpl) HealthCheckAPI(timeout time.Duration) error {
79117
return fmt.Errorf("Timeout waiting for Nomad healthcheck %s", n.address)
80118
}
81119

82-
rq, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s:%d/v1/nodes", n.address, n.port), nil)
83-
if err != nil {
84-
return err
85-
}
86-
87-
resp, err := n.httpClient.Do(rq)
88-
if err == nil && resp.StatusCode == 200 {
89-
nodes := []map[string]interface{}{}
90-
// check number of nodes
91-
json.NewDecoder(resp.Body).Decode(&nodes)
92-
93-
// loop nodes and check ready
94-
readyCount := 0
95-
for _, node := range nodes {
96-
// get the node status
97-
nodeStatus := node["Status"].(string)
98-
nodeName := node["Name"].(string)
99-
nodeEligable := node["SchedulingEligibility"].(string)
100-
101-
n.l.Debug("Node status", "node", nodeName, "status", nodeStatus, "eligible", nodeEligable)
102-
// get the driver status
103-
drivers, ok := node["Drivers"].(map[string]interface{})
104-
if !ok {
105-
continue
106-
}
107-
108-
var driversHealthy = true
109-
var dockerDetected = false
110-
for k, v := range drivers {
111-
driver, ok := v.(map[string]interface{})
112-
if !ok {
113-
continue
114-
}
120+
if simple {
121+
rq, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s:%d/v1/status/leader", n.address, n.port), nil)
122+
if err != nil {
123+
return err
124+
}
125+
resp, err := n.httpClient.Do(rq)
126+
if err == nil && resp.StatusCode == http.StatusOK {
127+
return nil
128+
}
129+
} else {
130+
rq, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s:%d/v1/nodes", n.address, n.port), nil)
131+
n.setAuthHeaders(rq)
132+
if err != nil {
133+
return err
134+
}
115135

116-
healthy, ok := driver["Healthy"].(bool)
136+
resp, err := n.httpClient.Do(rq)
137+
if err == nil && resp.StatusCode == http.StatusOK {
138+
nodes := []map[string]interface{}{}
139+
// check number of nodes
140+
json.NewDecoder(resp.Body).Decode(&nodes)
141+
142+
// loop nodes and check ready
143+
readyCount := 0
144+
for _, node := range nodes {
145+
// get the node status
146+
nodeStatus := node["Status"].(string)
147+
nodeName := node["Name"].(string)
148+
nodeEligable := node["SchedulingEligibility"].(string)
149+
150+
n.l.Debug("Node status", "node", nodeName, "status", nodeStatus, "eligible", nodeEligable)
151+
// get the driver status
152+
drivers, ok := node["Drivers"].(map[string]interface{})
117153
if !ok {
118154
continue
119155
}
120156

121-
detected, ok := driver["Detected"].(bool)
122-
if !ok || !detected {
123-
continue
124-
}
157+
var driversHealthy = true
158+
var dockerDetected = false
159+
for k, v := range drivers {
160+
driver, ok := v.(map[string]interface{})
161+
if !ok {
162+
continue
163+
}
164+
165+
healthy, ok := driver["Healthy"].(bool)
166+
if !ok {
167+
continue
168+
}
169+
170+
detected, ok := driver["Detected"].(bool)
171+
if !ok || !detected {
172+
continue
173+
}
174+
175+
// we need to make a special case to check the docker driver is
176+
// present as if the nomad server starts before docker then the
177+
// presence of docker will not be detected
178+
if k == "docker" {
179+
dockerDetected = true
180+
}
181+
182+
n.l.Debug("Driver status", "node", nodeName, "driver", k, "healthy", healthy)
183+
if !healthy {
184+
driversHealthy = false
185+
}
125186

126-
// we need to make a special case to check the docker driver is
127-
// present as if the nomad server starts before docker then the
128-
// presence of docker will not be detected
129-
if k == "docker" {
130-
dockerDetected = true
131187
}
132188

133-
n.l.Debug("Driver status", "node", nodeName, "driver", k, "healthy", healthy)
134-
if !healthy {
135-
driversHealthy = false
189+
if nodeStatus == "ready" && nodeEligable == "eligible" && driversHealthy && dockerDetected {
190+
readyCount++
136191
}
137-
138192
}
139193

140-
if nodeStatus == "ready" && nodeEligable == "eligible" && driversHealthy && dockerDetected {
141-
readyCount++
194+
if readyCount == n.clientNodes {
195+
n.l.Debug("Nomad check complete", "address", n.address)
196+
return nil
142197
}
143-
}
144198

145-
if readyCount == n.clientNodes {
146-
n.l.Debug("Nomad check complete", "address", n.address)
147-
return nil
199+
n.l.Debug("Nodes not ready", "ready", readyCount, "nodes", n.clientNodes)
148200
}
149-
150-
n.l.Debug("Nodes not ready", "ready", readyCount, "nodes", n.clientNodes)
151201
}
152202

153203
// backoff
@@ -171,6 +221,7 @@ func (n *NomadImpl) Create(files []string) error {
171221
cr := fmt.Sprintf(`{"Job": %s}`, string(jsonJob))
172222

173223
r, err := http.NewRequest(http.MethodPost, addr, bytes.NewReader([]byte(cr)))
224+
n.setAuthHeaders(r)
174225
if err != nil {
175226
return xerrors.Errorf("Unable to create http request: %w", err)
176227
}
@@ -201,6 +252,7 @@ func (n *NomadImpl) Stop(files []string) error {
201252

202253
// stop the job
203254
r, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s:%d/v1/job/%s", n.address, n.port, id), nil)
255+
n.setAuthHeaders(r)
204256
if err != nil {
205257
return xerrors.Errorf("Unable to create http request: %w", err)
206258
}
@@ -235,6 +287,7 @@ func (n *NomadImpl) ParseJob(file string) ([]byte, error) {
235287

236288
// validate the config with the Nomad API
237289
r, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s:%d/v1/jobs/parse", n.address, n.port), bytes.NewReader(jobData))
290+
n.setAuthHeaders(r)
238291
if err != nil {
239292
return nil, xerrors.Errorf("Unable to create http request: %w", err)
240293
}
@@ -313,6 +366,7 @@ func (n *NomadImpl) Endpoints(job, group, task string) ([]map[string]string, err
313366
}
314367

315368
r, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s:%d/v1/allocation/%s", n.address, n.port, j["ID"]), nil)
369+
n.setAuthHeaders(r)
316370
if err != nil {
317371
return nil, xerrors.Errorf("Unable to create http request: %w", err)
318372
}
@@ -393,6 +447,7 @@ func (n *NomadImpl) Endpoints(job, group, task string) ([]map[string]string, err
393447
func (n *NomadImpl) getJobAllocations(job string) ([]map[string]interface{}, error) {
394448
// get the allocations for the job
395449
r, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s:%d/v1/job/%s/allocations", n.address, n.port, job), nil)
450+
n.setAuthHeaders(r)
396451
if err != nil {
397452
return nil, xerrors.Errorf("Unable to create http request: %w", err)
398453
}

‎pkg/clients/nomad/nomad_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func setupNomadTests(t *testing.T) (Nomad, string, *mocks.HTTP) {
4040
)
4141

4242
c := NewNomad(mh, 1*time.Millisecond, logger.NewTestLogger(t))
43-
c.SetConfig("local", 4646, 1)
43+
c.SetConfig("local", 4646, 1, "")
4444

4545
return c, tmpDir, mh
4646
}

‎pkg/config/resources/nomad/provider_cluster.go‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,8 @@ func (p *ClusterProvider) Refresh() error {
145145

146146
wg.Wait()
147147

148-
p.nomadClient.SetConfig(fmt.Sprintf("http://%s", p.config.ExternalIP), p.config.APIPort, p.config.ClientNodes+1)
149-
err := p.nomadClient.HealthCheckAPI(startTimeout)
148+
p.nomadClient.SetConfig(fmt.Sprintf("http://%s", p.config.ExternalIP), p.config.APIPort, p.config.ClientNodes+1, p.config.ACLToken)
149+
err := p.nomadClient.HealthCheckAPI(startTimeout, false)
150150
if err != nil {
151151
return err
152152
}
@@ -174,8 +174,8 @@ func (p *ClusterProvider) Refresh() error {
174174
p.log.Debug("Successfully created client node", "ref", p.config.ID, "client", fqdn)
175175
}
176176

177-
p.nomadClient.SetConfig(fmt.Sprintf("http://%s", p.config.ExternalIP), p.config.APIPort, p.config.ClientNodes+1)
178-
err := p.nomadClient.HealthCheckAPI(startTimeout)
177+
p.nomadClient.SetConfig(fmt.Sprintf("http://%s", p.config.ExternalIP), p.config.APIPort, p.config.ClientNodes+1, p.config.ACLToken)
178+
err := p.nomadClient.HealthCheckAPI(startTimeout, false)
179179
if err != nil {
180180
return err
181181
}
@@ -391,8 +391,16 @@ func (p *ClusterProvider) createNomad() error {
391391
}
392392

393393
// ensure all client nodes are up
394-
p.nomadClient.SetConfig(fmt.Sprintf("http://%s", p.config.ExternalIP), p.config.APIPort, clientNodes)
395-
err = p.nomadClient.HealthCheckAPI(startTimeout)
394+
p.nomadClient.SetConfig(fmt.Sprintf("http://%s", p.config.ExternalIP), p.config.APIPort, clientNodes, p.config.ACLToken)
395+
err = p.nomadClient.HealthCheckAPI(startTimeout, true)
396+
if err != nil {
397+
return err
398+
}
399+
err = p.nomadClient.BootstrapACLs()
400+
if err != nil {
401+
return err
402+
}
403+
err = p.nomadClient.HealthCheckAPI(startTimeout, false)
396404
if err != nil {
397405
return err
398406
}

‎pkg/config/resources/nomad/provider_job.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func (p *JobProvider) Create() error {
4646
nomadCluster := p.config.Cluster
4747

4848
// load the config
49-
p.client.SetConfig(fmt.Sprintf("http://%s", nomadCluster.ExternalIP), nomadCluster.APIPort, nomadCluster.ClientNodes)
49+
p.client.SetConfig(fmt.Sprintf("http://%s", nomadCluster.ExternalIP), nomadCluster.APIPort, nomadCluster.ClientNodes, nomadCluster.ACLToken)
5050

5151
err := p.client.Create(p.config.Paths)
5252
if err != nil {
@@ -99,7 +99,7 @@ func (p *JobProvider) Destroy() error {
9999
nomadCluster := p.config.Cluster
100100

101101
// load the config
102-
p.client.SetConfig(fmt.Sprintf("http://%s", nomadCluster.ExternalIP), nomadCluster.APIPort, nomadCluster.ClientNodes)
102+
p.client.SetConfig(fmt.Sprintf("http://%s", nomadCluster.ExternalIP), nomadCluster.APIPort, nomadCluster.ClientNodes, nomadCluster.ACLToken)
103103

104104
err := p.client.Stop(p.config.Paths)
105105
if err != nil {

‎pkg/config/resources/nomad/resource_cluster.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ type NomadCluster struct {
3636
Ports ctypes.Ports `hcl:"port,block" json:"ports,omitempty"` // ports to expose
3737
PortRanges ctypes.PortRanges `hcl:"port_range,block" json:"port_ranges,omitempty"` // range of ports to expose
3838

39+
ACLToken string `hcl:"acl_token,optional" json:"acl_token,omitempty"`
40+
3941
// Output Parameters
4042

4143
// The APIPort the server is running on

0 commit comments

Comments
 (0)