diff --git a/main.go b/main.go index a942d65..23cb89f 100644 --- a/main.go +++ b/main.go @@ -222,14 +222,16 @@ type Options struct { All bool `cli:"-a, --all"` Deleted bool `cli:"-d, --deleted"` //These do nothing but are kept for backwards-compat - OnlyAlive bool `cli:"-o, --only-alive"` - Shallow bool `cli:"-s, --shallow"` + OnlyAlive bool `cli:"-o, --only-alive"` + Shallow bool `cli:"-s, --shallow"` + NoStringConversion bool `cli:"-n, --no-string-conversion"` } `cli:"export"` Import struct { IgnoreDestroyed bool `cli:"-I, --ignore-destroyed"` IgnoreDeleted bool `cli:"-i, --ignore-deleted"` Shallow bool `cli:"-s, --shallow"` + AppendOnly bool `cli:"-a, --appendonly"` } `cli:"import"` Move struct { @@ -2378,7 +2380,7 @@ redeleting them. r.Dispatch("export", &Help{ Summary: "Export one or more subtrees for migration / backup purposes", - Usage: "safe export [-ad] PATH [PATH ...]", + Usage: "safe export [-adn] PATH [PATH ...]", Type: NonDestructiveCommand, Description: ` Normally, the export will get only the latest version of each secret, and encode it in a format that is backwards- @@ -2387,6 +2389,7 @@ compatible with pre-1.0.0 versions of safe (and newer versions). incompatible with versions of safe prior to v1.0.0 -d (--deleted) will cause safe to undelete, read, and then redelete deleted secrets in order to encode them in the backup. Without this, deleted versions will be ignored. +-n (--no-string-conversion) will use v3 export and does not convert values to strings. This is incompatible with v1/v2 `}, func(command string, args ...string) error { rc.Apply(opt.UseTarget) if len(args) < 1 { @@ -2431,6 +2434,7 @@ backup. Without this, deleted versions will be ignored. FetchAllVersions: opt.Export.All, GetDeletedVersions: opt.Export.Deleted, AllowDeletedSecrets: opt.Export.Deleted, + AsStrings: !opt.Export.NoStringConversion, }) if err != nil { return err @@ -2440,6 +2444,7 @@ backup. Without this, deleted versions will be ignored. } var mustV2Export bool + mustV2Export = opt.Export.All || opt.Export.NoStringConversion //Determine if we can get away with a v1 export for _, s := range secrets { if len(s.Versions) > 1 { @@ -2459,7 +2464,12 @@ backup. Without this, deleted versions will be ignored. } v2Export := func() error { - export := exportFormat{ExportVersion: 2, Data: map[string]exportSecret{}, RequiresVersioning: map[string]bool{}} + + exportVersionNum := uint(2) + if opt.Export.NoStringConversion { + exportVersionNum = 3 + } + export := exportFormat{ExportVersion: exportVersionNum, Data: map[string]exportSecret{}, RequiresVersioning: map[string]bool{}} for _, secret := range secrets { if len(secret.Versions) > 1 { @@ -2477,11 +2487,11 @@ backup. Without this, deleted versions will be ignored. thisVersion := exportVersion{ Deleted: version.State == vault.SecretStateDeleted && opt.Export.Deleted, Destroyed: version.State == vault.SecretStateDestroyed || (version.State == vault.SecretStateDeleted && !opt.Export.Deleted), - Value: map[string]string{}, + Value: map[string]interface{}{}, } for _, key := range version.Data.Keys() { - thisVersion.Value[key] = version.Data.Get(key) + thisVersion.Value[key] = version.Data.GetAsInterface(key) } thisSecret.Versions = append(thisSecret.Versions, thisVersion) @@ -2524,6 +2534,7 @@ backup. Without this, deleted versions will be ignored. rting garbage data and then destroying it (which is originally done to preserve version numbering). -i (--ignore-deleted) will ignore deleted versions from being written during the import. -s (--shallow) will write only the latest version for each secret. +-a (--appendonly) will only write latest alive version if exists. `}, func(command string, args ...string) error { rc.Apply(opt.UseTarget) b, err := ioutil.ReadAll(os.Stdin) @@ -2622,7 +2633,7 @@ rting garbage data and then destroying it (which is originally done to preserve } data := vault.NewSecret() for k, v := range secret.Versions[i].Value { - data.Set(k, v, false) + data.SetAsInterface(k, v, false) } s.Versions = append(s.Versions, vault.SecretVersion{ Number: firstVersion + uint(i), @@ -2632,8 +2643,9 @@ rting garbage data and then destroying it (which is originally done to preserve } err := s.Copy(v, s.Path, vault.TreeCopyOpts{ - Clear: true, - Pad: !(opt.Import.IgnoreDestroyed || opt.Import.Shallow), + Clear: !opt.Import.AppendOnly, + Pad: !(opt.Import.IgnoreDestroyed || opt.Import.Shallow), + AppendOnly: opt.Import.AppendOnly, }) if err != nil { return err @@ -2654,7 +2666,7 @@ rting garbage data and then destroying it (which is originally done to preserve if len(v) == 1 { if meta, isMap := (v[0]).(map[string]interface{}); isMap { version, isFloat64 := meta["export_version"].(float64) - if isFloat64 && version == 2 { + if (isFloat64 && version == 2) || (isFloat64 && version == 3) { fn = v2Import } } @@ -4478,7 +4490,7 @@ type exportSecret struct { } type exportVersion struct { - Deleted bool `json:"deleted,omitempty"` - Destroyed bool `json:"destroyed,omitempty"` - Value map[string]string `json:"value,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Destroyed bool `json:"destroyed,omitempty"` + Value map[string]interface{} `json:"value,omitempty"` } diff --git a/vault/secret.go b/vault/secret.go index 504fd99..ceb19de 100644 --- a/vault/secret.go +++ b/vault/secret.go @@ -18,11 +18,11 @@ import ( // A Secret contains a set of key/value pairs that store anything you // want, including passwords, RSAKey keys, usernames, etc. type Secret struct { - data map[string]string + data map[string]interface{} } func NewSecret() *Secret { - return &Secret{make(map[string]string)} + return &Secret{make(map[string]interface{})} } func (s Secret) MarshalJSON() ([]byte, error) { @@ -41,8 +41,29 @@ func (s *Secret) Has(key string) bool { // Get retrieves the value of the given key, or "" if no such key exists. func (s *Secret) Get(key string) string { - x, _ := s.data[key] - return x + xx, ok := s.data[key] + if !ok { + return "" + } + switch x := xx.(type) { + case string: + return x + case []byte: + return string(x) + default: + res, _ := json.Marshal(x) + return string(res) + } + +} + +func (s *Secret) GetAsInterface(key string) interface{} { + xx, ok := s.data[key] + if !ok { + // return empty interface + return nil + } + return xx } func (s *Secret) Keys() []string { @@ -64,6 +85,15 @@ func (s *Secret) Set(key, value string, skipIfExists bool) error { return nil } +// Set interface +func (s *Secret) SetAsInterface(key string, value interface{}, skipIfExists bool) error { + if s.Has(key) && skipIfExists { + return ansi.Errorf("@R{BUG: Something tried to overwrite the} @C{%s} @R{key, but it already existed, and --no-clobber was specified}", key) + } + s.data[key] = value + return nil +} + // Delete removes the entry with the given key from the Secret. // Returns true if there was a matching object to delete. False otherwise. func (s *Secret) Delete(key string) bool { @@ -269,6 +299,29 @@ func (s *Secret) YAML() string { return string(b) } +// Eqals compares two secrets and returns true if they are equal +func (s *Secret) Equals(in *Secret) bool { + + if (s == nil && in == nil) || (s == in) { + return true + } + if (s == nil && in != nil) || (s != nil && in == nil) { + return false + } + if (s.data == nil && in.data != nil) || (s.data != nil && in.data == nil) { + return false + } + a, err := json.Marshal(s.data) + if err != nil { + return false + } + b, err := json.Marshal(in.data) + if err != nil { + return false + } + return string(a) == string(b) +} + // SingleValue converts a secret to a string representing the value extracted. // Returns an error if there are not exactly one results in the secret // object @@ -278,7 +331,16 @@ func (s *Secret) SingleValue() (string, error) { } var ret string for _, v := range s.data { - ret = v + + switch x := v.(type) { + case string: + ret = x + case []byte: + ret = string(x) + default: + res, _ := json.Marshal(x) + ret = string(res) + } } return ret, nil } diff --git a/vault/tree.go b/vault/tree.go index cb3afb8..dba7a03 100644 --- a/vault/tree.go +++ b/vault/tree.go @@ -113,7 +113,7 @@ type secretTree struct { Branches []secretTree Type uint MountVersion uint - Value string + Value interface{} Version uint Deleted bool Destroyed bool @@ -238,7 +238,8 @@ func (t secretTree) convertToSecrets() Secrets { } for _, key := range version.Branches { - thisVersion.Data.Set(key.Basename(), key.Value, false) + //thisVersion.Data.Set(key.Basename(), key.Value, false) + thisVersion.Data.SetAsInterface(key.Basename(), key.Value, false) } thisEntry.Versions = append(thisEntry.Versions, thisVersion) @@ -308,6 +309,8 @@ type TreeOpts struct { GetDeletedVersions bool //Only perform gets. If the target is not a secret, then an error is returned GetOnly bool + //All key values are retrieved as strings + AsStrings bool } func (v *Vault) constructTree(path string, opts TreeOpts) (*secretTree, error) { @@ -479,6 +482,8 @@ type TreeCopyOpts struct { Clear bool //Pad will insert dummy versions that have been truncated by Vault Pad bool + //AppendOnly + AppendOnly bool } func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error { @@ -489,6 +494,40 @@ func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error { } } + if opts.AppendOnly { + var latest *SecretVersion + if len(s.Versions) > 0 && s.Versions[len(s.Versions)-1].State == SecretStateAlive { + latest = &s.Versions[len(s.Versions)-1] + } + + if latest != nil { + updateRequired := false + // get latest version from client + existingSecret := &Secret{ + data: make(map[string]interface{}), + } + + _, err := v.client.Get(dst, &existingSecret.data, &vaultkv.KVGetOpts{Version: 0}) + // if err is nil, then the secret exists + if err == nil { + // compare the latest version of the secret with the existing secret + // if they are the same, then we don't need to do anything + updateRequired = !latest.Data.Equals(existingSecret) + + } else { + updateRequired = true + } + if updateRequired { + _, err = v.Client().Set(dst, latest.Data.data, nil) + if err != nil { + return fmt.Errorf("Could not write secret to path `%s': %s", dst, err) + } + } + + } + return nil + } + var toDelete, toDestroy []uint if opts.Pad && len(s.Versions) > 0 { @@ -503,9 +542,9 @@ func (s SecretEntry) Copy(v *Vault, dst string, opts TreeCopyOpts) error { } for _, version := range s.Versions { - var toWrite map[string]string + var toWrite map[string]interface{} if version.State == SecretStateDestroyed { - toWrite = map[string]string{"TO_DESTROY": "TO_DESTROY"} + toWrite = map[string]interface{}{"TO_DESTROY": "TO_DESTROY"} } else { toWrite = version.Data.data } @@ -810,6 +849,7 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) { } s, err := w.vault.Read(EncodePath(path, "", uint64(t.Version))) + //For v1 backends, this is the first non-list Vault access. // If we're unable to get a path that we could list because of permissions, // don't explode. @@ -820,6 +860,10 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) { return nil, err } + if w.opts.AsStrings { + s, err = w.vault.DataAsString(s) + } + if t.Deleted { w.vault.client.Delete(path, &vaultkv.KVDeleteOpts{Versions: []uint{t.Version}}) if err != nil { @@ -838,7 +882,7 @@ func (w *treeWorker) workGet(t secretTree) ([]secretTree, error) { ret = append(ret, secretTree{ Name: path + ":" + key, Type: treeTypeKey, - Value: string(s.data[key]), + Value: s.data[key], Version: version, Deleted: t.Deleted, }) diff --git a/vault/vault.go b/vault/vault.go index 1c30bb7..8bb7aec 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -154,25 +154,46 @@ func (v *Vault) Read(path string) (secret *Secret, err error) { } raw = map[string]interface{}{key: val} } - - for k, v := range raw { - if (key != "" && k == key) || key == "" { - if s, ok := v.(string); ok { - secret.data[k] = s - } else { - var b []byte - b, err = json.Marshal(v) - if err != nil { - return + secret.data = raw + /* + for k, v := range raw { + if (key != "" && k == key) || key == "" { + if s, ok := v.(string); ok { + secret.data[k] = s + } else { + var b []byte + b, err = json.Marshal(v) + if err != nil { + return + } + secret.data[k] = string(b) } - secret.data[k] = string(b) } } - } - + */ return } + + +func (v *Vault) DataAsString(in *Secret) (out *Secret, err error) { + out = in + + for k, v := range in.data { + if s, ok := v.(string); ok { + out.data[k] = s + } else { + var b []byte + b, err = json.Marshal(v) + if err != nil { + return + } + out.data[k] = string(b) + } + } + return out, nil +} + // List returns the set of (relative) paths that are directly underneath // the given path. Intermediate path nodes are suffixed with a single "/", // whereas leaf nodes (the secrets themselves) are not.