From 1c62b282ca5ff5ca441dd28cd9685ddc145a3dfa Mon Sep 17 00:00:00 2001 From: William Shoemaker Date: Mon, 12 Oct 2020 23:23:24 -0500 Subject: [PATCH 1/3] add host additional info filters --- server/datastore/inmem/hosts.go | 28 +++++++++++++++ server/datastore/mysql/hosts.go | 59 +++++++++++++++++++++++++++++-- server/kolide/hosts.go | 3 +- server/service/transport_hosts.go | 6 ++++ 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/server/datastore/inmem/hosts.go b/server/datastore/inmem/hosts.go index 6a87993ec98c..7680478e55c5 100644 --- a/server/datastore/inmem/hosts.go +++ b/server/datastore/inmem/hosts.go @@ -1,6 +1,7 @@ package inmem import ( + "encoding/json" "errors" "sort" "strings" @@ -103,6 +104,33 @@ func (d *Datastore) ListHosts(opt kolide.HostListOptions) ([]*kolide.Host, error low, high := d.getLimitOffsetSliceBounds(opt.ListOptions, len(hosts)) hosts = hosts[low:high] + // Filter additional info + if len(opt.AdditionalFilters) > 0 { + fieldsWanted := map[string]interface{}{} + for _, field := range opt.AdditionalFilters { + fieldsWanted[field] = true + } + for i, host := range hosts { + addInfo := map[string]interface{}{} + if err := json.Unmarshal(*host.Additional, &addInfo); err != nil { + return nil, err + } + + for k := range addInfo { + if _, ok := fieldsWanted[k]; !ok { + delete(addInfo, k) + } + } + addInfoJSON := json.RawMessage{} + addInfoJSON, err := json.Marshal(addInfo) + if err != nil { + return nil, err + } + host.Additional = &addInfoJSON + hosts[i] = host + } + } + return hosts, nil } diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index efdcc655761f..5af7ed7ae2b7 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -152,9 +152,62 @@ func (d *Datastore) Host(id uint) (*kolide.Host, error) { } func (d *Datastore) ListHosts(opt kolide.HostListOptions) ([]*kolide.Host, error) { - sql := ` - SELECT * FROM hosts - ` + sql := `SELECT id, + osquery_host_id, + created_at, + updated_at, + detail_update_time, + node_key, + host_name, + uuid, + platform, + osquery_version, + os_version, + build, + platform_like, + code_name, + uptime, + physical_memory, + cpu_type, + cpu_subtype, + cpu_brand, + cpu_physical_cores, + cpu_logical_cores, + hardware_vendor, + hardware_model, + hardware_version, + hardware_serial, + computer_name, + primary_ip_id, + seen_time, + distributed_interval, + logger_tls_period, + config_tls_refresh, + primary_ip, + primary_mac, + label_update_time, + enroll_secret_name, + ` + + // Filter additional info by extracting into a new json object + if len(opt.AdditionalFilters) > 0 { + sql += `JSON_OBJECT( + ` + for _, field := range opt.AdditionalFilters { + sql += fmt.Sprintf(`'%s', JSON_EXTRACT(additional, '$."%s"'), `, field, field) + } + sql = sql[:len(sql)-2] + sql += ` + ) AS additional + ` + } else { + sql += ` + additional + ` + } + + sql += `FROM hosts + ` var params []interface{} switch opt.StatusFilter { case "new": diff --git a/server/kolide/hosts.go b/server/kolide/hosts.go index 8a45721bad6b..30fd2e28890c 100644 --- a/server/kolide/hosts.go +++ b/server/kolide/hosts.go @@ -87,7 +87,8 @@ type HostService interface { type HostListOptions struct { ListOptions - StatusFilter HostStatus + AdditionalFilters []string + StatusFilter HostStatus } type Host struct { diff --git a/server/service/transport_hosts.go b/server/service/transport_hosts.go index 87be34a37a42..6b5e5135e3a8 100644 --- a/server/service/transport_hosts.go +++ b/server/service/transport_hosts.go @@ -3,6 +3,7 @@ package service import ( "context" "net/http" + "strings" "github.com/kolide/fleet/server/kolide" "github.com/pkg/errors" @@ -48,5 +49,10 @@ func decodeListHostsRequest(ctx context.Context, r *http.Request) (interface{}, if err != nil { return nil, err } + + additionalInfoFiltersString := r.URL.Query().Get("additional_info_filters") + if additionalInfoFiltersString != "" { + hopt.AdditionalFilters = strings.Split(additionalInfoFiltersString, ",") + } return listHostsRequest{ListOptions: hopt}, nil } From 271f77bcf0b3916972c4540208dc55edd6e2669a Mon Sep 17 00:00:00 2001 From: William Shoemaker Date: Fri, 13 Nov 2020 12:28:20 -0600 Subject: [PATCH 2/3] use query param in JSON_EXTRACT to prevent sqli --- server/datastore/mysql/hosts.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 5af7ed7ae2b7..973f03222d09 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -189,12 +189,15 @@ func (d *Datastore) ListHosts(opt kolide.HostListOptions) ([]*kolide.Host, error enroll_secret_name, ` - // Filter additional info by extracting into a new json object + var params []interface{} + + // Filter additional info by extracting into a new json object. if len(opt.AdditionalFilters) > 0 { sql += `JSON_OBJECT( ` for _, field := range opt.AdditionalFilters { - sql += fmt.Sprintf(`'%s', JSON_EXTRACT(additional, '$."%s"'), `, field, field) + sql += fmt.Sprintf(`?, JSON_EXTRACT(additional, ?), `) + params = append(params, field, fmt.Sprintf(`$."%s"`, field)) } sql = sql[:len(sql)-2] sql += ` @@ -208,7 +211,6 @@ func (d *Datastore) ListHosts(opt kolide.HostListOptions) ([]*kolide.Host, error sql += `FROM hosts ` - var params []interface{} switch opt.StatusFilter { case "new": sql += "WHERE DATE_ADD(created_at, INTERVAL 1 DAY) >= ?" From 537c15092cc4ad7a21ced5a1af63919bcb194529 Mon Sep 17 00:00:00 2001 From: William Shoemaker Date: Fri, 13 Nov 2020 16:16:07 -0600 Subject: [PATCH 3/3] add testListHostsFilterAdditional --- server/datastore/datastore_hosts_test.go | 42 ++++++++++++++++++++++-- server/datastore/datastore_test.go | 1 + 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/server/datastore/datastore_hosts_test.go b/server/datastore/datastore_hosts_test.go index c49ac1abdd9d..ff7cda4bb0e6 100644 --- a/server/datastore/datastore_hosts_test.go +++ b/server/datastore/datastore_hosts_test.go @@ -5,9 +5,9 @@ import ( "fmt" "sort" "strconv" + "strings" "testing" "time" - "strings" "github.com/WatchBeam/clock" "github.com/kolide/fleet/server/kolide" @@ -156,12 +156,49 @@ func testListHosts(t *testing.T, ds kolide.Datastore) { require.Equal(t, hosts[0].ID, hosts2[0].ID) } +func testListHostsFilterAdditional(t *testing.T, ds kolide.Datastore) { + h, err := ds.NewHost(&kolide.Host{ + DetailUpdateTime: time.Now(), + LabelUpdateTime: time.Now(), + SeenTime: time.Now(), + OsqueryHostID: "foobar", + NodeKey: "nodekey", + UUID: "uuid", + HostName: "foobar.local", + }) + require.Nil(t, err) + + // Add additional + additional := json.RawMessage(`{"field1": "v1", "field2": "v2"}`) + h.Additional = &additional + err = ds.SaveHost(h) + require.Nil(t, err) + + additional = json.RawMessage(`{"field1": "v1", "field2": "v2"}`) + hosts, err := ds.ListHosts(kolide.HostListOptions{}) + require.Nil(t, err) + assert.Equal(t, additional, *hosts[0].Additional) + + hosts, err = ds.ListHosts(kolide.HostListOptions{AdditionalFilters: []string{"field1", "field2"}}) + require.Nil(t, err) + assert.Equal(t, additional, *hosts[0].Additional) + + hosts, err = ds.ListHosts(kolide.HostListOptions{}) + require.Nil(t, err) + assert.Equal(t, additional, *hosts[0].Additional) + + additional = json.RawMessage(`{"field1": "v1", "missing": null}`) + hosts, err = ds.ListHosts(kolide.HostListOptions{AdditionalFilters: []string{"field1", "missing"}}) + require.Nil(t, err) + assert.Equal(t, additional, *hosts[0].Additional) +} + func testListHostsStatus(t *testing.T, ds kolide.Datastore) { for i := 0; i < 10; i++ { _, err := ds.NewHost(&kolide.Host{ DetailUpdateTime: time.Now(), LabelUpdateTime: time.Now(), - SeenTime: time.Now().Add(-time.Duration(i) *time.Minute), + SeenTime: time.Now().Add(-time.Duration(i) * time.Minute), OsqueryHostID: strconv.Itoa(i), NodeKey: fmt.Sprintf("%d", i), UUID: fmt.Sprintf("%d", i), @@ -190,7 +227,6 @@ func testListHostsStatus(t *testing.T, ds kolide.Datastore) { assert.Equal(t, 10, len(hosts)) } - func testEnrollHost(t *testing.T, ds kolide.Datastore) { test.AddAllHostsLabel(t, ds) var hosts []*kolide.Host diff --git a/server/datastore/datastore_test.go b/server/datastore/datastore_test.go index 5552ad4c6fc8..9cb770b47416 100644 --- a/server/datastore/datastore_test.go +++ b/server/datastore/datastore_test.go @@ -44,6 +44,7 @@ var testFunctions = [...]func(*testing.T, kolide.Datastore){ testSaveHosts, testDeleteHost, testListHosts, + testListHostsFilterAdditional, testListHostsStatus, testListHostsInPack, testListPacksForHost,