From 2cc07732d5c1c73187b097ac1927c80eaf145008 Mon Sep 17 00:00:00 2001 From: Ben Vinson Date: Wed, 5 Jun 2019 13:37:12 -0400 Subject: [PATCH] AWS: Parameter Store - Adds the ability to use secrets --- pkg/manifest/manifest.go | 97 ++++++++++++++++++++++++ pkg/manifest/manifest_test.go | 12 ++- pkg/manifest/secrets.go | 3 + pkg/manifest/service.go | 19 +++++ pkg/manifest/testdata/env.yml | 2 + pkg/manifest/testdata/simple.yml | 2 + provider/aws/formation/service.json.tmpl | 6 ++ 7 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 pkg/manifest/secrets.go diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index bd17cfbe19..9e3df065b3 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -21,10 +21,12 @@ type Manifest struct { Params Params `yaml:"params,omitempty"` Resources Resources `yaml:"resources,omitempty"` Services Services `yaml:"services,omitempty"` + Secrets Secrets `yaml:"secrets,omitempty"` Timers Timers `yaml:"timers,omitempty"` attributes map[string]bool env map[string]string + secrets map[string]string } func init() { @@ -66,6 +68,14 @@ func Load(data []byte, env map[string]string) (*Manifest, error) { return nil, err } + if err := m.CombineSecrets(); err != nil { + return nil, err + } + + if err := m.ValidateSecrets(); err != nil { + return nil, err + } + return &m, nil } @@ -137,6 +147,21 @@ func (m *Manifest) CombineEnv() error { return nil } +// used only for tests +func (m *Manifest) SetSecrets(secrets map[string]string) { + m.secrets = secrets +} + +func (m *Manifest) CombineSecrets() error { + for i, s := range m.Services { + me := make([]string, len(m.Secrets)) + copy(me, m.Secrets) + m.Services[i].Secrets = append(me, s.Secrets...) + } + + return nil +} + func (m *Manifest) Service(name string) (*Service, error) { for _, s := range m.Services { if s.Name == name { @@ -194,6 +219,53 @@ func (m *Manifest) ServiceEnvironment(service string) (map[string]string, error) return env, nil } +func (m *Manifest) ServiceSecrets(service string) (map[string]string, error) { + s, err := m.Service(service) + if err != nil { + return nil, err + } + + secrets := map[string]string{} + + missing := []string{} + + for _, e := range s.Secrets { + parts := strings.SplitN(e, "=", 2) + + switch len(parts) { + case 1: + if parts[0] == "*" { + for k, v := range m.secrets { + secrets[k] = v + } + } else { + v, ok := m.secrets[parts[0]] + if !ok { + missing = append(missing, parts[0]) + } + secrets[parts[0]] = v + } + case 2: + v, ok := m.secrets[parts[0]] + if ok { + secrets[parts[0]] = v + } else { + secrets[parts[0]] = parts[1] + } + default: + return nil, fmt.Errorf("invalid secrets declaration: %s", e) + } + } + + if len(missing) > 0 { + sort.Strings(missing) + + return nil, fmt.Errorf("required secrets: %s", strings.Join(missing, ", ")) + } + + return secrets, nil +} + // ValidateEnv returns an error if required env vars for a service are not available // It also filters m.env to the union of all service env vars defined in the manifest func (m *Manifest) ValidateEnv() error { @@ -219,6 +291,31 @@ func (m *Manifest) ValidateEnv() error { return nil } +// ValidateSecrets returns an error if required secrets vars for a service are not available +// It also filters m.secrets to the union of all service secrets vars defined in the manifest +func (m *Manifest) ValidateSecrets() error { + keys := map[string]bool{} + + for _, s := range m.Services { + secrets, err := m.ServiceSecrets(s.Name) + if err != nil { + return err + } + + for k := range secrets { + keys[k] = true + } + } + + for k := range m.secrets { + if !keys[k] { + delete(m.secrets, k) + } + } + + return nil +} + func (m *Manifest) ApplyDefaults() error { for i, s := range m.Services { if s.Build.Path == "" && s.Image == "" { diff --git a/pkg/manifest/manifest_test.go b/pkg/manifest/manifest_test.go index 0ef60b0de4..a41aee5f44 100644 --- a/pkg/manifest/manifest_test.go +++ b/pkg/manifest/manifest_test.go @@ -42,6 +42,7 @@ func TestManifestLoad(t *testing.T) { "DEVELOPMENT=false", "SECRET", }, + Secrets: []string{}, Health: manifest.ServiceHealth{ Grace: 10, Path: "/", @@ -74,6 +75,7 @@ func TestManifestLoad(t *testing.T) { Environment: []string{ "SECRET", }, + Secrets: []string{}, Port: manifest.ServicePort{Port: 2000, Scheme: "https"}, Scale: manifest.ServiceScale{ Count: manifest.ServiceScaleCount{Min: 1, Max: 1}, @@ -91,6 +93,7 @@ func TestManifestLoad(t *testing.T) { Command: "foo", Domains: []string{"baz.example.org", "qux.example.org"}, Drain: 60, + Secrets: []string{}, Health: manifest.ServiceHealth{ Grace: 2, Interval: 5, @@ -114,6 +117,7 @@ func TestManifestLoad(t *testing.T) { }, Command: "", Drain: 30, + Secrets: []string{}, Health: manifest.ServiceHealth{ Grace: 5, Interval: 5, @@ -135,6 +139,7 @@ func TestManifestLoad(t *testing.T) { }, Command: "", Drain: 30, + Secrets: []string{}, Health: manifest.ServiceHealth{ Grace: 5, Interval: 5, @@ -177,6 +182,7 @@ func TestManifestLoad(t *testing.T) { Environment: []string{ "SECRET", }, + Secrets: []string{}, Port: manifest.ServicePort{Port: 2000, Scheme: "https"}, Scale: manifest.ServiceScale{ Count: manifest.ServiceScaleCount{Min: 1, Max: 1}, @@ -200,6 +206,7 @@ func TestManifestLoad(t *testing.T) { Path: ".", }, Drain: 30, + Secrets: []string{}, Health: manifest.ServiceHealth{ Grace: 5, Path: "/", @@ -335,6 +342,9 @@ func TestManifestLoadSimple(t *testing.T) { "REQUIRED", "DEFAULT=true", }, + Secrets: manifest.Secrets{ + "DATABASE=sql://localhost", + }, Health: manifest.ServiceHealth{ Grace: 5, Interval: 5, @@ -351,7 +361,7 @@ func TestManifestLoadSimple(t *testing.T) { }, } - n.SetAttributes([]string{"services", "services.web", "services.web.build", "services.web.environment"}) + n.SetAttributes([]string{"services", "services.web", "services.web.build", "services.web.environment", "services.web.secrets"}) n.SetEnv(map[string]string{"REQUIRED": "test"}) // env processing that normally happens as part of load diff --git a/pkg/manifest/secrets.go b/pkg/manifest/secrets.go new file mode 100644 index 0000000000..36afc3ea23 --- /dev/null +++ b/pkg/manifest/secrets.go @@ -0,0 +1,3 @@ +package manifest + +type Secrets []string diff --git a/pkg/manifest/service.go b/pkg/manifest/service.go index 5c1f0915d1..d737c214a9 100644 --- a/pkg/manifest/service.go +++ b/pkg/manifest/service.go @@ -16,6 +16,7 @@ type Service struct { Domains ServiceDomains `yaml:"domain,omitempty"` Drain int `yaml:"drain,omitempty"` Environment Environment `yaml:"environment,omitempty"` + Secrets Secrets `yaml:"secrets,omitempty"` Health ServiceHealth `yaml:"health,omitempty"` Image string `yaml:"image,omitempty"` Init bool `yaml:"init,omitempty"` @@ -135,6 +136,24 @@ func (s Service) EnvironmentKeys() string { return strings.Join(keys, ",") } +func (s Service) SecretstKeys() string { + kh := map[string]bool{} + + for _, e := range s.Secrets { + kh[strings.Split(e, "=")[0]] = true + } + + keys := []string{} + + for k := range kh { + keys = append(keys, k) + } + + sort.Strings(keys) + + return strings.Join(keys, ",") +} + func (s Service) GetName() string { return s.Name } diff --git a/pkg/manifest/testdata/env.yml b/pkg/manifest/testdata/env.yml index bd9bb245ed..961d27c286 100644 --- a/pkg/manifest/testdata/env.yml +++ b/pkg/manifest/testdata/env.yml @@ -8,6 +8,8 @@ services: q-train-intent: environment: - QUEUE_NAME=train-intent + secrets: + - DATABASE=test://example/db q-delete-intent: environment: - QUEUE_NAME=delete-intent diff --git a/pkg/manifest/testdata/simple.yml b/pkg/manifest/testdata/simple.yml index 57b85bb809..8c0d71b171 100644 --- a/pkg/manifest/testdata/simple.yml +++ b/pkg/manifest/testdata/simple.yml @@ -4,3 +4,5 @@ services: environment: - REQUIRED - DEFAULT=true + secrets: + - DATABASE=sql://localhost \ No newline at end of file diff --git a/provider/aws/formation/service.json.tmpl b/provider/aws/formation/service.json.tmpl index d73b804dec..24564b290a 100644 --- a/provider/aws/formation/service.json.tmpl +++ b/provider/aws/formation/service.json.tmpl @@ -507,6 +507,12 @@ { "Ref": "AWS::NoValue" } ], "Ulimits": [ { "Name": "nofile", "SoftLimit": "1024000", "HardLimit": "1024000" } ] + "Secrets": [ + {{ range $k, $v := .Secrets }} + { "Name": "{{$k}}", "ValueFrom": {{ safe $v }} }, + {{ end }} + { "Ref": "AWS::NoValue" } + ] } ], "Cpu": { "Fn::If": [ "Fargate", { "Ref": "Cpu" }, { "Ref": "AWS::NoValue" } ] },