feat: add profile feature for agent, and fix logr's panic (#444)

Signed-off-by: jingchao.djc <jingchao.djc@antfin.com>

Signed-off-by: jingchao.djc <jingchao.djc@antfin.com>

Author: alswl

agent/README.md

@@ -56,4 +56,21 @@ Update the container env [variables](https://github.com/kubernetes/git-sync#para
 
 ### Demo Recording
 
-[![asciicast](https://asciinema.org/a/FWbvVAiSsiI87wQx2TJbRMlxN.svg)](https://asciinema.org/a/FWbvVAiSsiI87wQx2TJbRMlxN)
+[![asciicast](https://asciinema.org/a/FWbvVAiSsiI87wQx2TJbRMlxN.svg)](https://asciinema.org/a/FWbvVAiSsiI87wQx2TJbRMlxN)
+
+
+### Profiling
+
+Using env variables to enable profiling mode, the agent can be started with the following envs:
+
+```bash
+export GITOPS_ENGINE_PROFILE=web
+# optional, default pprofile address is 127.0.0.1:6060
+export GITOPS_ENGINE_PROFILE_HOST=127.0.0.1
+export GITOPS_ENGINE_PROFILE_PORT=6060
+```
+
+And then you can open profile in the browser(or using [pprof](https://github.com/google/pprof) cmd to generate diagrams):
+
+- http://127.0.0.1:6060/debug/pprof/goroutine?debug=2
+- http://127.0.0.1:6060/debug/pprof/mutex?debug=2

agent/main.go

@@ -5,11 +5,13 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
+	"github.com/argoproj/gitops-engine/pkg/utils/text"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"text/tabwriter"
 	"time"
@@ -24,10 +26,15 @@ import (
 	"github.com/argoproj/gitops-engine/pkg/engine"
 	"github.com/argoproj/gitops-engine/pkg/sync"
 	"github.com/argoproj/gitops-engine/pkg/utils/kube"
+
+	_ "net/http/pprof"
 )
 
 const (
 	annotationGCMark = "gitops-agent.argoproj.io/gc-mark"
+	envProfile       = "GITOPS_ENGINE_PROFILE"
+	envProfileHost   = "GITOPS_ENGINE_PROFILE_HOST"
+	envProfilePort   = "GITOPS_ENGINE_PROFILE_PORT"
 )
 
 func main() {
@@ -96,6 +103,19 @@ func (s *settings) parseManifests() ([]*unstructured.Unstructured, string, error
 	return res, string(revision), nil
 }
 
+func StartProfiler(log logr.Logger) {
+	if os.Getenv(envProfile) == "web" {
+		go func() {
+			runtime.SetBlockProfileRate(1)
+			runtime.SetMutexProfileFraction(1)
+			profilePort := text.WithDefault(os.Getenv(envProfilePort), "6060")
+			profileHost := text.WithDefault(os.Getenv(envProfileHost), "127.0.0.1")
+
+			log.Info("pprof", "err", http.ListenAndServe(fmt.Sprintf("%s:%s", profileHost, profilePort), nil))
+		}()
+	}
+}
+
 func newCmd(log logr.Logger) *cobra.Command {
 	var (
 		clientConfig  clientcmd.ClientConfig
@@ -125,6 +145,8 @@ func newCmd(log logr.Logger) *cobra.Command {
 			if namespaced {
 				namespaces = []string{namespace}
 			}
+
+			StartProfiler(log)
 			clusterCache := cache.NewClusterCache(config,
 				cache.SetNamespaces(namespaces),
 				cache.SetLogr(log),

pkg/cache/cluster.go

@@ -730,7 +730,7 @@ func (c *clusterCache) sync() error {
 		if !ok {
 			return err
 		}
-		c.log.Info("warning loading openapi schema: %s", e)
+		c.log.Info("warning loading openapi schema: %s", e.Error())
 	}
 
 	if gvkParser != nil {

pkg/utils/text/text.go

@@ -8,3 +8,11 @@ func FirstNonEmpty(args ...string) string {
 	}
 	return ""
 }
+
+// WithDefault return defaultValue when val is blank
+func WithDefault(val string, defaultValue string) string {
+	if len(val) == 0 {
+		return defaultValue
+	}
+	return val
+}