From b5c05a3ba5c7c27f2b757e4f30ce062dacd44795 Mon Sep 17 00:00:00 2001 From: Siim Aus Date: Thu, 5 Oct 2023 18:05:21 +0300 Subject: [PATCH 1/2] Add enhanchements to safe export and import functions: - It is possible now export without converting Data key values to string. If this option is used then export format is not compatible with v2 format. - It is now possible to import with append only mode - this will take latest Alive version from import file and checks if this data matches to current data in vault. If not then it will be added as next version. This is useful when you want only to add newest values to vault and not kill your entire existing history for secret. --- main.go | 38 +++++++++++++++++--------- vault/secret.go | 72 +++++++++++++++++++++++++++++++++++++++++++++---- vault/tree.go | 54 +++++++++++++++++++++++++++++++++---- vault/vault.go | 47 +++++++++++++++++++++++--------- 4 files changed, 175 insertions(+), 36 deletions(-) diff --git a/main.go b/main.go index a942d65..23cb89f 100644 --- a/main.go +++ b/main.go @@ -222,14 +222,16 @@ type Options struct { All bool `cli:"-a, --all"` Deleted bool `cli:"-d, --deleted"` //These do nothing but are kept for backwards-compat - OnlyAlive bool `cli:"-o, --only-alive"` - Shallow bool `cli:"-s, --shallow"` + OnlyAlive bool `cli:"-o, --only-alive"` + Shallow bool `cli:"-s, --shallow"` + NoStringConversion bool `cli:"-n, --no-string-conversion"` } `cli:"export"` Import struct { IgnoreDestroyed bool `cli:"-I, --ignore-destroyed"` IgnoreDeleted bool `cli:"-i, --ignore-deleted"` Shallow bool `cli:"-s, --shallow"` + AppendOnly bool `cli:"-a, --appendonly"` } `cli:"import"` Move struct { @@ -2378,7 +2380,7 @@ redeleting them. r.Dispatch("export", &Help{ Summary: "Export one or more subtrees for migration / backup purposes", - Usage: "safe export [-ad] PATH [PATH ...]", + Usage: "safe export [-adn] PATH [PATH ...]", Type: NonDestructiveCommand, Description: ` Normally, the export will get only the latest version of each secret, and encode it in a format that is backwards- @@ -2387,6 +2389,7 @@ compatible with pre-1.0.0 versions of safe (and newer versions). incompatible with versions of safe prior to v1.0.0 -d (--deleted) will cause safe to undelete, read, and then redelete deleted secrets in order to encode them in the backup. Without this, deleted versions will be ignored. +-n (--no-string-conversion) will use v3 export and does not convert values to strings. This is incompatible with v1/v2 `}, func(command string, args ...string) error { rc.Apply(opt.UseTarget) if len(args) < 1 { @@ -2431,6 +2434,7 @@ backup. Without this, deleted versions will be ignored. FetchAllVersions: opt.Export.All, GetDeletedVersions: opt.Export.Deleted, AllowDeletedSecrets: opt.Export.Deleted, + AsStrings: !opt.Export.NoStringConversion, }) if err != nil { return err @@ -2440,6 +2444,7 @@ backup. Without this, deleted versions will be ignored. } var mustV2Export bool + mustV2Export = opt.Export.All || opt.Export.NoStringConversion //Determine if we can get away with a v1 export for _, s := range secrets { if len(s.Versions) > 1 { @@ -2459,7 +2464,12 @@ backup. Without this, deleted versions will be ignored. } v2Export := func() error { - export := exportFormat{ExportVersion: 2, Data: map[string]exportSecret{}, RequiresVersioning: map[string]bool{}} + + exportVersionNum := uint(2) + if opt.Export.NoStringConversion { + exportVersionNum = 3 + } + export := exportFormat{ExportVersion: exportVersionNum, Data: map[string]exportSecret{}, RequiresVersioning: map[string]bool{}} for _, secret := range secrets { if len(secret.Versions) > 1 { @@ -2477,11 +2487,11 @@ backup. Without this, deleted versions will be ignored. thisVersion := exportVersion{ Deleted: version.State == vault.SecretStateDeleted && opt.Export.Deleted, Destroyed: version.State == vault.SecretStateDestroyed || (version.State == vault.SecretStateDeleted && !opt.Export.Deleted), - Value: map[string]string{}, + Value: map[string]interface{}{}, } for _, key := range version.Data.Keys() { - thisVersion.Value[key] = version.Data.Get(key) + thisVersion.Value[key] = version.Data.GetAsInterface(key) } thisSecret.Versions = append(thisSecret.Versions, thisVersion) @@ -2524,6 +2534,7 @@ backup. Without this, deleted versions will be ignored. rting garbage data and then destroying it (which is originally done to preserve version numbering). -i (--ignore-deleted) will ignore deleted versions from being written during the import. -s (--shallow) will write only the latest version for each secret. +-a (--appendonly) will only write latest alive version if exists. `}, func(command string, args ...string) error { rc.Apply(opt.UseTarget) b, err := ioutil.ReadAll(os.Stdin) @@ -2622,7 +2633,7 @@ rting garbage data and then destroying it (which is originally done to preserve } data := vault.NewSecret() for k, v := range secret.Versions[i].Value { - data.Set(k, v, false) + data.SetAsInterface(k, v, false) } s.Versions = append(s.Versions, vault.SecretVersion{ Number: firstVersion + uint(i), @@ -2632,8 +2643,9 @@ rting garbage data and then destroying it (which is originally done to preserve } err := s.Copy(v, s.Path, vault.TreeCopyOpts{ - Clear: true, - Pad: !(opt.Import.IgnoreDestroyed || opt.Import.Shallow), + Clear: !opt.Import.AppendOnly, + Pad: !(opt.Import.IgnoreDestroyed || opt.Import.Shallow), + AppendOnly: opt.Import.AppendOnly, }) if err != nil { return err @@ -2654,7 +2666,7 @@ rting garbage data and then destroying it (which is originally done to preserve if len(v) == 1 { if meta, isMap := (v[0]).(map[string]interface{}); isMap { version, isFloat64 := meta["export_version"].(float64) - if isFloat64 && version == 2 { + if (isFloat64 && version == 2) || (isFloat64 && version == 3) { fn = v2Import } } @@ -4478,7 +4490,7 @@ type exportSecret struct { } type exportVersion struct { - Deleted bool `json:"deleted,omitempty"` - Destroyed bool `json:"destroyed,omitempty"` - Value map[string]string `json:"value,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Destroyed bool `json:"destroyed,omitempty"` + Value map[string]interface{} `json:"value,omitempty"` } diff --git a/vault/secret.go b/vault/secret.go index 504fd99..ceb19de 100644 --- a/vault/secret.go +++ b/vault/secret.go @@ -18,11 +18,11 @@ import ( // A Secret contains a set of key/value pairs that store anything you // want, including passwords, RSAKey keys, usernames, etc. type Secret struct { - data map[string]string + data map[string]interface{} } func NewSecret() *Secret { - return &Secret{make(map[string]string)} + return &Secret{make(map[string]interface{})} } func (s Secret) MarshalJSON() ([]byte, error) { @@ -41,8 +41,29 @@ func (s *Secret) Has(key string) bool { // Get retrieves the value of the given key, or "" if no such key exists. func (s *Secret) Get(key string) string { - x, _ := s.data[key] - return x + xx, ok := s.data[key] + if !ok { + return "" + } + switch x := xx.(type) { + case string: + return x + case []byte: + return string(x) + default: + res, _ := json.Marshal(x) + return string(res) + } + +} + +func (s *Secret) GetAsInterface(key string) interface{} { + xx, ok := s.data[key] + if !ok { + // return empty interface + return nil + } + return xx } func (s *Secret) Keys() []string { @@ -64,6 +85,15 @@ func (s *Secret) Set(key, value string, skipIfExists bool) error { return nil } +// Set interface +func (s *Secret) SetAsInterface(key string, value interface{}, skipIfExists bool) error { + if s.Has(key) && skipIfExists { + return ansi.Errorf("@R{BUG: Something tried to overwrite the} @C{%s} @R{key, but it already existed, and --no-clobber was specified}", key) + } + s.data[key] = value + return nil +} + // Delete removes the entry with the given key from the Secret. // Returns true if there was a matching object to delete. False otherwise. func (s *Secret) Delete(key string) bool { @@ -269,6 +299,29 @@ func (s *Secret) YAML() string { return string(b) } +// Eqals compares two secrets and returns true if they are equal +func (s *Secret) Equals(in *Secret) bool { + + if (s == nil && in == nil) || (s == in) { + return true + } + if (s == nil && in != nil) || (s != nil && in == nil) { + return false + } + if (s.data == nil && in.data != nil) || (s.data != nil && in.data == nil) { + return false + } + a, err := json.Marshal(s.data) + if err != nil { + return false + } + b, err := json.Marshal(in.data) + if err != nil { + return false + } + return string(a) == string(b) +} + // SingleValue converts a secret to a string representing the value extracted. // Returns an error if there are not exactly one results in the secret // object @@ -278,7 +331,16 @@ func (s *Secret) SingleValue() (string, error) { } var ret string for _, v := range s.data { - ret = v + + switch x := v.(type) { + case string: + ret = x + case []byte: + ret = string(x) + default: + res, _ := json.Marshal(x) + ret = string(res) + } } return ret, nil } diff --git a/vault/tree.go b/vault/tree.go index cb3afb8..dba7a03 100644 --- a/vault/tree.go +++ b/vault/tree.go @@ -113,7 +113,7 @@ type secretTree struct { Branches []secretTree Type uint MountVersion uint - Value string + Value interface{} Version uint Deleted bool Destroyed bool @@ -238,7 +238,8 @@ func (t secretTree) convertToSecrets() Secrets { } for _, key := range version.Branches { - thisVersion.Data.Set(key.Basename(), key.Value, false) + //thisVersion.Data.Set(key.Basename(), key.Value, false) + thisVersion.Data.SetAsInterface(key.Basename(), key.Value, false) } thisEntry.Versions = append(thisEntry.Versions, thisVersion) @@ -308,6 +309,8 @@ type TreeOpts struct { GetDeletedVersions bool //Only perform gets. If the target is not a secret, then an error is returned GetOnly bool + //All key values are retrieved as strings + AsStrings bool } func (v *Vault) constructTree(path string, opts TreeOpts) (*secretTree, error) { @@ -479,6 +482,8 @@ type TreeCopyOpts struct { Clear bool //Pad will insert dummy versions that have been truncated by Vault Pad bool + //AppendOnly + AppendOnly bool } func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error { @@ -489,6 +494,40 @@ func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error { } } + if opts.AppendOnly { + var latest *SecretVersion + if len(s.Versions) > 0 && s.Versions[len(s.Versions)-1].State == SecretStateAlive { + latest = &s.Versions[len(s.Versions)-1] + } + + if latest != nil { + updateRequired := false + // get latest version from client + existingSecret := &Secret{ + data: make(map[string]interface{}), + } + + _, err := v.client.Get(dst, &existingSecret.data, &vaultkv.KVGetOpts{Version: 0}) + // if err is nil, then the secret exists + if err == nil { + // compare the latest version of the secret with the existing secret + // if they are the same, then we don't need to do anything + updateRequired = !latest.Data.Equals(existingSecret) + + } else { + updateRequired = true + } + if updateRequired { + _, err = v.Client().Set(dst, latest.Data.data, nil) + if err != nil { + return fmt.Errorf("Could not write secret to path `%s': %s", dst, err) + } + } + + } + return nil + } + var toDelete, toDestroy []uint if opts.Pad && len(s.Versions) > 0 { @@ -503,9 +542,9 @@ func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error { } for _, version := range s.Versions { - var toWrite map[string]string + var toWrite map[string]interface{} if version.State == SecretStateDestroyed { - toWrite = map[string]string{"TO_DESTROY": "TO_DESTROY"} + toWrite = map[string]interface{}{"TO_DESTROY": "TO_DESTROY"} } else { toWrite = version.Data.data } @@ -810,6 +849,7 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) { } s, err := w.vault.Read(EncodePath(path, "", uint64(t.Version))) + //For v1 backends, this is the first non-list Vault access. // If we're unable to get a path that we could list because of permissions, // don't explode. @@ -820,6 +860,10 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) { return nil, err } + if w.opts.AsStrings { + s, err = w.vault.DataAsString(s) + } + if t.Deleted { w.vault.client.Delete(path, &vaultkv.KVDeleteOpts{Versions: []uint{t.Version}}) if err != nil { @@ -838,7 +882,7 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) { ret = append(ret, secretTree{ Name: path + ":" + key, Type: treeTypeKey, - Value: string(s.data[key]), + Value: s.data[key], Version: version, Deleted: t.Deleted, }) diff --git a/vault/vault.go b/vault/vault.go index 1c30bb7..8bb7aec 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -154,25 +154,46 @@ func (v *Vault) Read(path string) (secret *Secret, err error) { } raw = map[string]interface{}{key: val} } - - for k, v := range raw { - if (key != "" && k == key) || key == "" { - if s, ok := v.(string); ok { - secret.data[k] = s - } else { - var b []byte - b, err = json.Marshal(v) - if err != nil { - return + secret.data = raw + /* + for k, v := range raw { + if (key != "" && k == key) || key == "" { + if s, ok := v.(string); ok { + secret.data[k] = s + } else { + var b []byte + b, err = json.Marshal(v) + if err != nil { + return + } + secret.data[k] = string(b) } - secret.data[k] = string(b) } } - } - + */ return } + + +func (v *Vault) DataAsString(in *Secret) (out *Secret, err error) { + out = in + + for k, v := range in.data { + if s, ok := v.(string); ok { + out.data[k] = s + } else { + var b []byte + b, err = json.Marshal(v) + if err != nil { + return + } + out.data[k] = string(b) + } + } + return out, nil +} + // List returns the set of (relative) paths that are directly underneath // the given path. Intermediate path nodes are suffixed with a single "/", // whereas leaf nodes (the secrets themselves) are not. From ebbe4f16c1ed333544beefa7d4762310d63b78fe Mon Sep 17 00:00:00 2001 From: Siim Aus Date: Thu, 5 Oct 2023 18:05:21 +0300 Subject: [PATCH 2/2] Add enhanchements to safe export and import functions: - It is possible now export without converting Data key values to string. If this option is used then export format is not compatible with v2 format. - It is now possible to import with append only mode - this will take latest Alive version from import file and checks if this data matches to current data in vault. If not then it will be added as next version. This is useful when you want only to add newest values to vault and not kill your entire existing history for secret. --- main.go | 38 +++++++++++++++++--------- vault/secret.go | 72 +++++++++++++++++++++++++++++++++++++++++++++---- vault/tree.go | 54 +++++++++++++++++++++++++++++++++---- vault/vault.go | 47 +++++++++++++++++++++++--------- 4 files changed, 175 insertions(+), 36 deletions(-) diff --git a/main.go b/main.go index a942d65..23cb89f 100644 --- a/main.go +++ b/main.go @@ -222,14 +222,16 @@ type Options struct { All bool `cli:"-a, --all"` Deleted bool `cli:"-d, --deleted"` //These do nothing but are kept for backwards-compat - OnlyAlive bool `cli:"-o, --only-alive"` - Shallow bool `cli:"-s, --shallow"` + OnlyAlive bool `cli:"-o, --only-alive"` + Shallow bool `cli:"-s, --shallow"` + NoStringConversion bool `cli:"-n, --no-string-conversion"` } `cli:"export"` Import struct { IgnoreDestroyed bool `cli:"-I, --ignore-destroyed"` IgnoreDeleted bool `cli:"-i, --ignore-deleted"` Shallow bool `cli:"-s, --shallow"` + AppendOnly bool `cli:"-a, --appendonly"` } `cli:"import"` Move struct { @@ -2378,7 +2380,7 @@ redeleting them. r.Dispatch("export", &Help{ Summary: "Export one or more subtrees for migration / backup purposes", - Usage: "safe export [-ad] PATH [PATH ...]", + Usage: "safe export [-adn] PATH [PATH ...]", Type: NonDestructiveCommand, Description: ` Normally, the export will get only the latest version of each secret, and encode it in a format that is backwards- @@ -2387,6 +2389,7 @@ compatible with pre-1.0.0 versions of safe (and newer versions). incompatible with versions of safe prior to v1.0.0 -d (--deleted) will cause safe to undelete, read, and then redelete deleted secrets in order to encode them in the backup. Without this, deleted versions will be ignored. +-n (--no-string-conversion) will use v3 export and does not convert values to strings. This is incompatible with v1/v2 `}, func(command string, args ...string) error { rc.Apply(opt.UseTarget) if len(args) < 1 { @@ -2431,6 +2434,7 @@ backup. Without this, deleted versions will be ignored. FetchAllVersions: opt.Export.All, GetDeletedVersions: opt.Export.Deleted, AllowDeletedSecrets: opt.Export.Deleted, + AsStrings: !opt.Export.NoStringConversion, }) if err != nil { return err @@ -2440,6 +2444,7 @@ backup. Without this, deleted versions will be ignored. } var mustV2Export bool + mustV2Export = opt.Export.All || opt.Export.NoStringConversion //Determine if we can get away with a v1 export for _, s := range secrets { if len(s.Versions) > 1 { @@ -2459,7 +2464,12 @@ backup. Without this, deleted versions will be ignored. } v2Export := func() error { - export := exportFormat{ExportVersion: 2, Data: map[string]exportSecret{}, RequiresVersioning: map[string]bool{}} + + exportVersionNum := uint(2) + if opt.Export.NoStringConversion { + exportVersionNum = 3 + } + export := exportFormat{ExportVersion: exportVersionNum, Data: map[string]exportSecret{}, RequiresVersioning: map[string]bool{}} for _, secret := range secrets { if len(secret.Versions) > 1 { @@ -2477,11 +2487,11 @@ backup. Without this, deleted versions will be ignored. thisVersion := exportVersion{ Deleted: version.State == vault.SecretStateDeleted && opt.Export.Deleted, Destroyed: version.State == vault.SecretStateDestroyed || (version.State == vault.SecretStateDeleted && !opt.Export.Deleted), - Value: map[string]string{}, + Value: map[string]interface{}{}, } for _, key := range version.Data.Keys() { - thisVersion.Value[key] = version.Data.Get(key) + thisVersion.Value[key] = version.Data.GetAsInterface(key) } thisSecret.Versions = append(thisSecret.Versions, thisVersion) @@ -2524,6 +2534,7 @@ backup. Without this, deleted versions will be ignored. rting garbage data and then destroying it (which is originally done to preserve version numbering). -i (--ignore-deleted) will ignore deleted versions from being written during the import. -s (--shallow) will write only the latest version for each secret. +-a (--appendonly) will only write latest alive version if exists. `}, func(command string, args ...string) error { rc.Apply(opt.UseTarget) b, err := ioutil.ReadAll(os.Stdin) @@ -2622,7 +2633,7 @@ rting garbage data and then destroying it (which is originally done to preserve } data := vault.NewSecret() for k, v := range secret.Versions[i].Value { - data.Set(k, v, false) + data.SetAsInterface(k, v, false) } s.Versions = append(s.Versions, vault.SecretVersion{ Number: firstVersion + uint(i), @@ -2632,8 +2643,9 @@ rting garbage data and then destroying it (which is originally done to preserve } err := s.Copy(v, s.Path, vault.TreeCopyOpts{ - Clear: true, - Pad: !(opt.Import.IgnoreDestroyed || opt.Import.Shallow), + Clear: !opt.Import.AppendOnly, + Pad: !(opt.Import.IgnoreDestroyed || opt.Import.Shallow), + AppendOnly: opt.Import.AppendOnly, }) if err != nil { return err @@ -2654,7 +2666,7 @@ rting garbage data and then destroying it (which is originally done to preserve if len(v) == 1 { if meta, isMap := (v[0]).(map[string]interface{}); isMap { version, isFloat64 := meta["export_version"].(float64) - if isFloat64 && version == 2 { + if (isFloat64 && version == 2) || (isFloat64 && version == 3) { fn = v2Import } } @@ -4478,7 +4490,7 @@ type exportSecret struct { } type exportVersion struct { - Deleted bool `json:"deleted,omitempty"` - Destroyed bool `json:"destroyed,omitempty"` - Value map[string]string `json:"value,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Destroyed bool `json:"destroyed,omitempty"` + Value map[string]interface{} `json:"value,omitempty"` } diff --git a/vault/secret.go b/vault/secret.go index 504fd99..ceb19de 100644 --- a/vault/secret.go +++ b/vault/secret.go @@ -18,11 +18,11 @@ import ( // A Secret contains a set of key/value pairs that store anything you // want, including passwords, RSAKey keys, usernames, etc. type Secret struct { - data map[string]string + data map[string]interface{} } func NewSecret() *Secret { - return &Secret{make(map[string]string)} + return &Secret{make(map[string]interface{})} } func (s Secret) MarshalJSON() ([]byte, error) { @@ -41,8 +41,29 @@ func (s *Secret) Has(key string) bool { // Get retrieves the value of the given key, or "" if no such key exists. func (s *Secret) Get(key string) string { - x, _ := s.data[key] - return x + xx, ok := s.data[key] + if !ok { + return "" + } + switch x := xx.(type) { + case string: + return x + case []byte: + return string(x) + default: + res, _ := json.Marshal(x) + return string(res) + } + +} + +func (s *Secret) GetAsInterface(key string) interface{} { + xx, ok := s.data[key] + if !ok { + // return empty interface + return nil + } + return xx } func (s *Secret) Keys() []string { @@ -64,6 +85,15 @@ func (s *Secret) Set(key, value string, skipIfExists bool) error { return nil } +// Set interface +func (s *Secret) SetAsInterface(key string, value interface{}, skipIfExists bool) error { + if s.Has(key) && skipIfExists { + return ansi.Errorf("@R{BUG: Something tried to overwrite the} @C{%s} @R{key, but it already existed, and --no-clobber was specified}", key) + } + s.data[key] = value + return nil +} + // Delete removes the entry with the given key from the Secret. // Returns true if there was a matching object to delete. False otherwise. func (s *Secret) Delete(key string) bool { @@ -269,6 +299,29 @@ func (s *Secret) YAML() string { return string(b) } +// Eqals compares two secrets and returns true if they are equal +func (s *Secret) Equals(in *Secret) bool { + + if (s == nil && in == nil) || (s == in) { + return true + } + if (s == nil && in != nil) || (s != nil && in == nil) { + return false + } + if (s.data == nil && in.data != nil) || (s.data != nil && in.data == nil) { + return false + } + a, err := json.Marshal(s.data) + if err != nil { + return false + } + b, err := json.Marshal(in.data) + if err != nil { + return false + } + return string(a) == string(b) +} + // SingleValue converts a secret to a string representing the value extracted. // Returns an error if there are not exactly one results in the secret // object @@ -278,7 +331,16 @@ func (s *Secret) SingleValue() (string, error) { } var ret string for _, v := range s.data { - ret = v + + switch x := v.(type) { + case string: + ret = x + case []byte: + ret = string(x) + default: + res, _ := json.Marshal(x) + ret = string(res) + } } return ret, nil } diff --git a/vault/tree.go b/vault/tree.go index cb3afb8..dba7a03 100644 --- a/vault/tree.go +++ b/vault/tree.go @@ -113,7 +113,7 @@ type secretTree struct { Branches []secretTree Type uint MountVersion uint - Value string + Value interface{} Version uint Deleted bool Destroyed bool @@ -238,7 +238,8 @@ func (t secretTree) convertToSecrets() Secrets { } for _, key := range version.Branches { - thisVersion.Data.Set(key.Basename(), key.Value, false) + //thisVersion.Data.Set(key.Basename(), key.Value, false) + thisVersion.Data.SetAsInterface(key.Basename(), key.Value, false) } thisEntry.Versions = append(thisEntry.Versions, thisVersion) @@ -308,6 +309,8 @@ type TreeOpts struct { GetDeletedVersions bool //Only perform gets. If the target is not a secret, then an error is returned GetOnly bool + //All key values are retrieved as strings + AsStrings bool } func (v *Vault) constructTree(path string, opts TreeOpts) (*secretTree, error) { @@ -479,6 +482,8 @@ type TreeCopyOpts struct { Clear bool //Pad will insert dummy versions that have been truncated by Vault Pad bool + //AppendOnly + AppendOnly bool } func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error { @@ -489,6 +494,40 @@ func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error { } } + if opts.AppendOnly { + var latest *SecretVersion + if len(s.Versions) > 0 && s.Versions[len(s.Versions)-1].State == SecretStateAlive { + latest = &s.Versions[len(s.Versions)-1] + } + + if latest != nil { + updateRequired := false + // get latest version from client + existingSecret := &Secret{ + data: make(map[string]interface{}), + } + + _, err := v.client.Get(dst, &existingSecret.data, &vaultkv.KVGetOpts{Version: 0}) + // if err is nil, then the secret exists + if err == nil { + // compare the latest version of the secret with the existing secret + // if they are the same, then we don't need to do anything + updateRequired = !latest.Data.Equals(existingSecret) + + } else { + updateRequired = true + } + if updateRequired { + _, err = v.Client().Set(dst, latest.Data.data, nil) + if err != nil { + return fmt.Errorf("Could not write secret to path `%s': %s", dst, err) + } + } + + } + return nil + } + var toDelete, toDestroy []uint if opts.Pad && len(s.Versions) > 0 { @@ -503,9 +542,9 @@ func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error { } for _, version := range s.Versions { - var toWrite map[string]string + var toWrite map[string]interface{} if version.State == SecretStateDestroyed { - toWrite = map[string]string{"TO_DESTROY": "TO_DESTROY"} + toWrite = map[string]interface{}{"TO_DESTROY": "TO_DESTROY"} } else { toWrite = version.Data.data } @@ -810,6 +849,7 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) { } s, err := w.vault.Read(EncodePath(path, "", uint64(t.Version))) + //For v1 backends, this is the first non-list Vault access. // If we're unable to get a path that we could list because of permissions, // don't explode. @@ -820,6 +860,10 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) { return nil, err } + if w.opts.AsStrings { + s, err = w.vault.DataAsString(s) + } + if t.Deleted { w.vault.client.Delete(path, &vaultkv.KVDeleteOpts{Versions: []uint{t.Version}}) if err != nil { @@ -838,7 +882,7 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) { ret = append(ret, secretTree{ Name: path + ":" + key, Type: treeTypeKey, - Value: string(s.data[key]), + Value: s.data[key], Version: version, Deleted: t.Deleted, }) diff --git a/vault/vault.go b/vault/vault.go index 1c30bb7..8bb7aec 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -154,25 +154,46 @@ func (v *Vault) Read(path string) (secret *Secret, err error) { } raw = map[string]interface{}{key: val} } - - for k, v := range raw { - if (key != "" && k == key) || key == "" { - if s, ok := v.(string); ok { - secret.data[k] = s - } else { - var b []byte - b, err = json.Marshal(v) - if err != nil { - return + secret.data = raw + /* + for k, v := range raw { + if (key != "" && k == key) || key == "" { + if s, ok := v.(string); ok { + secret.data[k] = s + } else { + var b []byte + b, err = json.Marshal(v) + if err != nil { + return + } + secret.data[k] = string(b) } - secret.data[k] = string(b) } } - } - + */ return } + + +func (v *Vault) DataAsString(in *Secret) (out *Secret, err error) { + out = in + + for k, v := range in.data { + if s, ok := v.(string); ok { + out.data[k] = s + } else { + var b []byte + b, err = json.Marshal(v) + if err != nil { + return + } + out.data[k] = string(b) + } + } + return out, nil +} + // List returns the set of (relative) paths that are directly underneath // the given path. Intermediate path nodes are suffixed with a single "/", // whereas leaf nodes (the secrets themselves) are not.