sync dev to master

.gitignore

@@ -8,6 +8,7 @@
 # Test binary, build with `go test -c`
 *.test
 
+/bin
 .localstack/
 .vscode/
 data/

Dockerfile

@@ -1,5 +1,5 @@
 # build stage
-FROM golang:1.13 as builder
+FROM golang:1.16 as builder
 
 ENV GO111MODULE=on
 
@@ -15,10 +15,10 @@ RUN go mod download
 
 COPY . .
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./bin/api
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./bin/api -mod=mod
 
 # final stage
-FROM alpine:3.9
+FROM alpine:3.13
 COPY --from=builder /app /app
 
 RUN apk update && \

Dockerfile.tests

@@ -1,6 +1,7 @@
-FROM golang:1.13
+FROM golang:1.16
 
 ENV GO111MODULE=on
+ENV GOFLAGS=-mod=mod
 
 WORKDIR /app
 

Makefile

@@ -35,10 +35,10 @@ coverage:
 docker-all: docker-build docker-image
 
 docker-build:
-	@docker run -i --rm -v "$(PWD):/go/src/$(PKG)" -w /go/src/$(PKG) golang:1.10 make build OS=linux ARCH=amd64
+	@docker run -i --rm -v "$(PWD):/go/src/$(PKG)" -w /go/src/$(PKG) golang:1.16 make build OS=linux ARCH=amd64
 
 docker-test:
-	@docker run -i --rm -v "$(PWD):/go/src/$(PKG)" -w /go/src/$(PKG) --network greeny_default golang:1.10 make test
+	@docker run -i --rm -v "$(PWD):/go/src/$(PKG)" -w /go/src/$(PKG) --network phoenix_default golang:1.16 make test
 
 docker-image:
 	@docker build -t $(IMAGE):$(VERSION) .

README.md

@@ -34,4 +34,4 @@ In order to contribute to Phoenix, see the [CONTRIBUTING](CONTRIBUTING.md) file
 If your company or your product is using Phoenix, please let us know by adding yourself to the Phoenix [users](USERS.md) file.
 
 ## License
-Phoenix is licensed under MIT license as found in the [LICENSE](LICENSE.md) file.
+Phoenix is licensed under MIT license as found in the [LICENSE](LICENSE.md) file.

chart/templates/internal-deployment.yaml

@@ -45,3 +45,5 @@ spec:
 {{ toYaml .Values.internal.affinity | indent 8 }}
       tolerations:
 {{ toYaml .Values.internal.tolerations | indent 8 }}      
+      nodeSelector:
+{{ toYaml .Values.internal.nodeSelector | indent 8 }}

chart/templates/public-deployment.yaml

@@ -46,4 +46,6 @@ spec:
       affinity:
 {{ toYaml .Values.public.affinity | indent 8 }}
       tolerations:
-{{ toYaml .Values.internal.tolerations | indent 8 }}  
+{{ toYaml .Values.public.tolerations | indent 8 }}  
+      nodeSelector:
+{{ toYaml .Values.public.nodeSelector | indent 8 }}

chart/templates/worker-deployment.yaml

@@ -34,6 +34,8 @@ spec:
           resources:
 {{ toYaml .Values.worker.resources | indent 12 }}
       affinity:
-{{ toYaml .Values.public.affinity | indent 8 }}
+{{ toYaml .Values.worker.affinity | indent 8 }}
       tolerations:
-{{ toYaml .Values.internal.tolerations | indent 8 }}  
+{{ toYaml .Values.worker.tolerations | indent 8 }}
+      nodeSelector:
+{{ toYaml .Values.worker.nodeSelector | indent 8 }}

chart/values.yaml

@@ -25,6 +25,7 @@ internal:
   data:
     # all the env variables can be found here: https://github.com/rtlnl/phoenix/blob/master/cmd/internal.go#L64
     DB_HOST: "redis-master.phoenix:6379"
+    DB_PASSWORD: ""
     S3_REGION: "us-west-1"
     S3_ENDPOINT: "s3.eu-west-1.amazonaws.com"
     S3_DISABLE_SSL: "false"
@@ -43,6 +44,7 @@ public:
 
   data:
     DB_HOST: "redis-master.phoenix:6379"
+    DB_PASSWORD: ""
     # Other ENV you can set in the configmap
     # You can find all the possible variables here: https://github.com/rtlnl/phoenix/blob/master/cmd/public.go#L102
     # REC_LOGS_TYPE: "kafka"
@@ -65,9 +67,11 @@ worker:
 
   data:
     # all the env variables can be found here: https://github.com/rtlnl/phoenix/blob/master/cmd/worker.go#L79
-    WORKER_BROKER_URL: "edis-master.phoenix:6379"
+    WORKER_BROKER_URL: "redis-master.phoenix:6379"
+    WORKER_PASSWORD: ""
 
   resources: {}
+  nodeSelector: {}
   tolerations: {}
   affinity: {}
 

cmd/internal.go

@@ -26,6 +26,7 @@ var internalCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		addr := viper.GetString(addressInternalFlag)
 		dbHost := viper.GetString(dbHostInternalFlag)
+		dbPassword := viper.GetString(dbPasswordInternalFlag)
 		s3Region := viper.GetString(s3RegionFlag)
 		s3Endpoint := viper.GetString(s3EndpointFlag)
 		s3DisableSSL := viper.GetBool(s3DisableSSLFlag)
@@ -38,7 +39,7 @@ var internalCmd = &cobra.Command{
 		}
 
 		// instantiate Redis client
-		redisClient, err := db.NewRedisClient(dbHost)
+		redisClient, err := db.NewRedisClient(dbHost, db.Password(dbPassword))
 		if err != nil {
 			panic(err)
 		}
@@ -48,7 +49,7 @@ var internalCmd = &cobra.Command{
 		middlewares = append(middlewares, md.DB(redisClient))
 		middlewares = append(middlewares, md.AWSSession(s3Region, s3Endpoint, s3DisableSSL))
 		middlewares = append(middlewares, md.Cors())
-		middlewares = append(middlewares, md.NewWorker(dbHost, workerProducerName, workerQueueName))
+		middlewares = append(middlewares, md.NewWorker(redisClient, workerProducerName, workerQueueName))
 
 		i, err := internal.NewInternalAPI(middlewares...)
 		if err != nil {
@@ -68,13 +69,15 @@ func init() {
 
 	f.String(addressInternalFlag, ":8081", "server address")
 	f.String(dbHostInternalFlag, "127.0.0.1:6379", "database host")
+	f.String(dbPasswordInternalFlag, "", "database password")
 	f.String(s3RegionFlag, "eu-west-1", "s3 region")
 	f.String(s3EndpointFlag, "localhost:4572", "s3 endpoint")
 	f.Bool(s3DisableSSLFlag, true, "disable SSL verification for s3")
 	f.Bool(logDebugFlag, false, "sets log level to debug")
 
 	viper.BindEnv(addressInternalFlag, "ADDRESS_HOST")
 	viper.BindEnv(dbHostInternalFlag, "DB_HOST")
+	viper.BindEnv(dbPasswordInternalFlag, "DB_PASSWORD")
 	viper.BindEnv(s3RegionFlag, "S3_REGION")
 	viper.BindEnv(s3EndpointFlag, "S3_ENDPOINT")
 	viper.BindEnv(s3DisableSSLFlag, "S3_DISABLE_SSL")

cmd/public.go

@@ -39,6 +39,7 @@ APIs for serving the personalized content.`,
 		// read parameters in input
 		addr := viper.GetString(addressPublicFlag)
 		dbHost := viper.GetString(dbHostPublicFlag)
+		dbPassword := viper.GetString(dbPasswordPublicFlag)
 		logType := viper.GetString(recommendationLogsFlag)
 		logDebug := viper.GetBool(logDebugFlag)
 
@@ -49,7 +50,7 @@ APIs for serving the personalized content.`,
 		}
 
 		// instantiate Redis client
-		redisClient, err := db.NewRedisClient(dbHost)
+		redisClient, err := db.NewRedisClient(dbHost, db.Password(dbPassword))
 		if err != nil {
 			panic(err)
 		}
@@ -107,6 +108,7 @@ func init() {
 	// mandatory parameters
 	f.StringP(addressPublicFlag, "a", ":8082", "server address")
 	f.StringP(dbHostPublicFlag, "d", "127.0.0.1:6379", "database host")
+	f.String(dbPasswordPublicFlag, "", "database password")
 	f.Bool(logDebugFlag, false, "sets log level to debug")
 
 	// optional parameters
@@ -121,6 +123,7 @@ func init() {
 
 	viper.BindEnv(addressPublicFlag, "ADDRESS_HOST")
 	viper.BindEnv(dbHostPublicFlag, "DB_HOST")
+	viper.BindEnv(dbPasswordPublicFlag, "DB_PASSWORD")
 	viper.BindEnv(logDebugFlag, "LOG_DEBUG")
 	viper.BindEnv(recommendationLogsFlag, "REC_LOGS_TYPE")
 	viper.BindEnv(recommendationKafkaBrokersFlag, "REC_LOGS_BROKERS")

cmd/root.go

@@ -7,12 +7,15 @@ import (
 )
 
 var (
-	addressPublicFlag   = "address-host-public"
-	dbHostPublicFlag    = "db-host-public"
-	addressInternalFlag = "address-host-internal"
-	dbHostInternalFlag  = "db-host-internal"
-	workerBrokerFlag    = "worker-broker-url"
-	logDebugFlag        = "debug"
+	addressPublicFlag      = "address-host-public"
+	dbHostPublicFlag       = "db-host-public"
+	dbPasswordPublicFlag   = "db-password-public"
+	addressInternalFlag    = "address-host-internal"
+	dbHostInternalFlag     = "db-host-internal"
+	dbPasswordInternalFlag = "db-password-internal"
+	workerBrokerFlag       = "worker-broker-url"
+	workerPasswordFlag     = "worker-password"
+	logDebugFlag           = "debug"
 )
 
 // rootCmd represents the base command when called without any subcommands

cmd/worker.go

@@ -27,22 +27,23 @@ var workerCmd = &cobra.Command{
 	that will be executed one they arrive.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		brokerWorker := viper.GetString(workerBrokerFlag)
+		passwordWorker := viper.GetString(workerPasswordFlag)
 
 		// instantiate Redis client
-		redisClient, err := db.NewRedisClient(brokerWorker)
+		rc, err := db.NewRedisClient(brokerWorker, db.Password(passwordWorker))
 		if err != nil {
 			panic(err)
 		}
 
-		l, err := redisClient.Lock(worker.WorkerLockKey)
-		defer redisClient.Unlock(worker.WorkerLockKey)
+		l, err := rc.Lock(worker.WorkerLockKey)
+		defer rc.Unlock(worker.WorkerLockKey)
 
 		if l == false || err != nil {
 			log.Error().Err(err).Msg("REDIS failed to acquire lock")
 			os.Exit(0)
 		}
 
-		w, err := worker.New(brokerWorker, workerConsumerName, workerQueueName)
+		w, err := worker.New(rc.Client, workerConsumerName, workerQueueName)
 		if err != nil {
 			panic(err)
 		}
@@ -61,7 +62,7 @@ var workerCmd = &cobra.Command{
 		for {
 			select {
 			case <-ticker.C:
-				if err := redisClient.ExtendTTL(worker.WorkerLockKey); err != nil {
+				if err := rc.ExtendTTL(worker.WorkerLockKey); err != nil {
 					log.Error().Msg(err.Error())
 					w.Close()
 					break EXITLOOP
@@ -82,8 +83,10 @@ func init() {
 	f := workerCmd.PersistentFlags()
 
 	f.String(workerBrokerFlag, "127.0.0.1:6379", "broker url for the workers")
+	f.String(workerPasswordFlag, "", "broker password")
 
 	viper.BindEnv(workerBrokerFlag, "WORKER_BROKER_URL")
+	viper.BindEnv(workerPasswordFlag, "WORKER_PASSWORD")
 
 	viper.BindPFlags(f)
 }

docker-compose.yaml

@@ -21,6 +21,7 @@ services:
     restart: always
     command:
       - "redis-server"
+      # - "--requirepass qwerty"
       - "--appendonly yes"
     ports:
       - "6379:6379"

go.mod

@@ -1,50 +1,24 @@
 module github.com/rtlnl/phoenix
 
-go 1.13
+go 1.15
 
 require (
-	github.com/Shopify/sarama v1.24.0
-	github.com/adjust/gocheck v0.0.0-20131111155431-fbc315b36e0e // indirect
-	github.com/adjust/rmq v1.0.0
-	github.com/adjust/uniuri v0.0.0-20130923163420-498743145e60 // indirect
+	github.com/Shopify/sarama v1.29.0
+	github.com/adjust/rmq/v3 v3.0.0
 	github.com/allegro/bigcache v1.2.1
-	github.com/aws/aws-sdk-go v1.25.8
-	github.com/banzaicloud/go-gin-prometheus v0.0.0-20190417120951-df9373ad5327
-	github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b // indirect
+	github.com/aws/aws-sdk-go v1.38.55
+	github.com/banzaicloud/go-gin-prometheus v0.1.0
 	github.com/davecgh/go-spew v1.1.1
-	github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051 // indirect
-	github.com/elastic/go-elasticsearch/v7 v7.4.1
-	github.com/garyburd/redigo v1.6.0 // indirect
-	github.com/gin-contrib/cors v1.3.0
-	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/gin-gonic/gin v1.4.0
-	github.com/go-playground/locales v0.12.1 // indirect
-	github.com/go-playground/universal-translator v0.16.0 // indirect
-	github.com/go-redis/redis v6.15.6+incompatible
-	github.com/google/uuid v1.1.1
-	github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9 // indirect
-	github.com/json-iterator/go v1.1.7
-	github.com/klauspost/cpuid v1.2.1 // indirect
-	github.com/leodido/go-urn v1.1.0 // indirect
-	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
-	github.com/mattn/go-isatty v0.0.8 // indirect
-	github.com/onsi/ginkgo v1.10.0 // indirect
-	github.com/onsi/gomega v1.7.0 // indirect
-	github.com/prometheus/client_golang v1.2.1
-	github.com/rs/zerolog v1.15.0
-	github.com/spf13/cobra v0.0.5
-	github.com/spf13/viper v1.4.0
-	github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect
-	github.com/stretchr/testify v1.4.0
-	github.com/tsenart/vegeta v12.7.0+incompatible
-	github.com/ugorji/go v1.1.7 // indirect
-	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc // indirect
-	golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 // indirect
-	golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect
-	golang.org/x/text v0.3.2 // indirect
-	gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a // indirect
-	gopkg.in/go-playground/validator.v9 v9.29.1
-	gopkg.in/redis.v3 v3.6.4 // indirect
-	gopkg.in/yaml.v2 v2.2.4 // indirect
-	launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect
+	github.com/elastic/go-elasticsearch/v7 v7.13.1
+	github.com/gin-contrib/cors v1.3.1
+	github.com/gin-gonic/gin v1.7.2
+	github.com/go-redis/redis/v7 v7.4.0
+	github.com/google/uuid v1.2.0
+	github.com/json-iterator/go v1.1.11
+	github.com/prometheus/client_golang v1.10.0
+	github.com/rs/zerolog v1.22.0
+	github.com/spf13/cobra v1.1.3
+	github.com/spf13/viper v1.7.1
+	github.com/stretchr/testify v1.7.0
+	gopkg.in/go-playground/validator.v9 v9.31.0
 )