Add support to route both GRPC and HTTP over the HTTP server (weaveworks#288)

* add support for grpc on the http port

Signed-off-by: Joe Elliott <number101010@gmail.com>

Author: joe-elliott

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/prometheus/exporter-toolkit v0.8.2
 	github.com/sercand/kuberesolver/v4 v4.0.0
 	github.com/sirupsen/logrus v1.6.0
+	github.com/soheilhy/cmux v0.1.5 // indirect
 	github.com/stretchr/testify v1.8.1
 	github.com/uber/jaeger-client-go v2.28.0+incompatible
 	github.com/uber/jaeger-lib v2.2.0+incompatible

go.sum

@@ -668,6 +668,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
+github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
@@ -794,6 +796,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=

server/server.go

@@ -17,6 +17,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/exporter-toolkit/web"
+	"github.com/soheilhy/cmux"
 	"golang.org/x/net/context"
 	"golang.org/x/net/netutil"
 	"google.golang.org/grpc"
@@ -96,6 +97,7 @@ type Config struct {
 	HTTPMiddleware                []middleware.Interface         `yaml:"-"`
 	Router                        *mux.Router                    `yaml:"-"`
 	DoNotAddDefaultHTTPMiddleware bool                           `yaml:"-"`
+	RouteHTTPToGRPC               bool                           `yaml:"-"`
 
 	GPRCServerMaxRecvMsgSize           int           `yaml:"grpc_server_max_recv_msg_size"`
 	GRPCServerMaxSendMsgSize           int           `yaml:"grpc_server_max_send_msg_size"`
@@ -185,6 +187,13 @@ type Server struct {
 	grpcListener net.Listener
 	httpListener net.Listener
 
+	// These fields are used to support grpc over the http server
+	//  if RouteHTTPToGRPC is set. the fields are kept here
+	//  so they can be initialized in New() and started in Run()
+	grpchttpmux        cmux.CMux
+	grpcOnHTTPListener net.Listener
+	GRPCOnHTTPServer   *grpc.Server
+
 	HTTP       *mux.Router
 	HTTPServer *http.Server
 	GRPC       *grpc.Server
@@ -242,6 +251,15 @@ func New(cfg Config) (*Server, error) {
 		httpListener = netutil.LimitListener(httpListener, cfg.HTTPConnLimit)
 	}
 
+	var grpcOnHTTPListener net.Listener
+	var grpchttpmux cmux.CMux
+	if cfg.RouteHTTPToGRPC {
+		grpchttpmux = cmux.New(httpListener)
+
+		httpListener = grpchttpmux.Match(cmux.HTTP1Fast())
+		grpcOnHTTPListener = grpchttpmux.Match(cmux.HTTP2())
+	}
+
 	network = cfg.GRPCListenNetwork
 	if network == "" {
 		network = DefaultNetwork
@@ -384,6 +402,7 @@ func New(cfg Config) (*Server, error) {
 		grpcOptions = append(grpcOptions, grpc.Creds(grpcCreds))
 	}
 	grpcServer := grpc.NewServer(grpcOptions...)
+	grpcOnHttpServer := grpc.NewServer(grpcOptions...)
 
 	// Setup HTTP server
 	var router *mux.Router
@@ -446,17 +465,20 @@ func New(cfg Config) (*Server, error) {
 	}
 
 	return &Server{
-		cfg:          cfg,
-		httpListener: httpListener,
-		grpcListener: grpcListener,
-		handler:      handler,
-
-		HTTP:       router,
-		HTTPServer: httpServer,
-		GRPC:       grpcServer,
-		Log:        log,
-		Registerer: reg,
-		Gatherer:   gatherer,
+		cfg:                cfg,
+		httpListener:       httpListener,
+		grpcListener:       grpcListener,
+		grpcOnHTTPListener: grpcOnHTTPListener,
+		handler:            handler,
+		grpchttpmux:        grpchttpmux,
+
+		HTTP:             router,
+		HTTPServer:       httpServer,
+		GRPC:             grpcServer,
+		GRPCOnHTTPServer: grpcOnHttpServer,
+		Log:              log,
+		Registerer:       reg,
+		Gatherer:         gatherer,
 	}, nil
 }
 
@@ -509,19 +531,37 @@ func (s *Server) Run() error {
 
 	go func() {
 		err := s.GRPC.Serve(s.grpcListener)
-		if err == grpc.ErrServerStopped {
-			err = nil
-		}
-
-		select {
-		case errChan <- err:
-		default:
-		}
+		handleGRPCError(err, errChan)
 	}()
 
+	// grpchttpmux will only be set if grpchttpmux RouteHTTPToGRPC is set
+	if s.grpchttpmux != nil {
+		go func() {
+			err := s.grpchttpmux.Serve()
+			handleGRPCError(err, errChan)
+		}()
+		go func() {
+			err := s.GRPCOnHTTPServer.Serve(s.grpcOnHTTPListener)
+			handleGRPCError(err, errChan)
+		}()
+	}
+
 	return <-errChan
 }
 
+// handleGRPCError consolidates GRPC Server error handling by sending
+// any error to errChan except for grpc.ErrServerStopped which is ignored.
+func handleGRPCError(err error, errChan chan error) {
+	if err == grpc.ErrServerStopped {
+		err = nil
+	}
+
+	select {
+	case errChan <- err:
+	default:
+	}
+}
+
 // HTTPListenAddr exposes `net.Addr` that `Server` is listening to for HTTP connections.
 func (s *Server) HTTPListenAddr() net.Addr {
 	return s.httpListener.Addr()