Golang stdlib impovements

- Support golang.org/toolchain paths
- Use mappings if there is no version identified

Author: simonswine

.pyroscope.yaml

@@ -1,7 +1,7 @@
 source_code:
   mappings:
     - path:
-        - prefix: GOROOT
+        - prefix: $GOROOT/src
       language: go
       source:
        github:

pkg/frontend/vcs/config/config.go

@@ -159,7 +159,7 @@ type FileSpec struct {
 func (c *PyroscopeConfig) FindMapping(file FileSpec) *MappingConfig {
 	// Find the longest matching prefix
 	var bestMatch *MappingConfig
-	var bestMatchLen int = -1
+	var bestMatchLen = -1
 	for _, m := range c.SourceCode.Mappings {
 		if result := m.Match(file); result > bestMatchLen {
 			bestMatch = &m

pkg/frontend/vcs/config/config_test.go

@@ -294,13 +294,13 @@ func TestFindMapping(t *testing.T) {
 				require.NotNil(t, result)
 				require.Len(t, result.Path, 1)
 				assert.Equal(t, tt.expectedPrefix, result.Path[0].Prefix)
-				if tt.expectedSource == "local" {
+				switch tt.expectedSource {
+				case "local":
 					assert.NotNil(t, result.Source.Local)
-				} else if tt.expectedSource == "github" {
+				case "github":
 					assert.NotNil(t, result.Source.GitHub)
 				}
 			}
 		})
 	}
 }
-

pkg/frontend/vcs/source/find_go.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -30,8 +31,8 @@ func (ff FileFinder) findGoFile(ctx context.Context, mappings ...*config.Mapping
 	sp.SetTag("file.path", ff.file.Path)
 	sp.SetTag("file.function_name", ff.file.FunctionName)
 
-	if url, ok := golang.StandardLibraryURL(ff.file.Path); ok {
-		return ff.fetchURL(ctx, url, false)
+	if path, version, ok := golang.IsStandardLibraryPath(ff.file.Path); ok {
+		return ff.fetchGoStdlib(ctx, path, version)
 	}
 
 	if relativePath, ok := golang.VendorRelativePath(ff.file.Path); ok {
@@ -56,6 +57,36 @@ func (ff FileFinder) findGoFile(ctx context.Context, mappings ...*config.Mapping
 	return ff.tryFindGoFile(ctx, 30)
 }
 
+func (ff FileFinder) fetchGoStdlib(ctx context.Context, path string, version string) (*vcsv1.GetFileResponse, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "fetchGoStdlib")
+	defer sp.Finish()
+
+	// if there is no version detected, use the one from .pyroscope.yaml
+	if version == "" {
+		mapping := ff.config.FindMapping(config.FileSpec{Path: "$GOROOT/src"})
+		if mapping != nil {
+			return ff.fetchMappingFile(ctx, mapping, path)
+		}
+	}
+
+	// use master branch as fallback
+	ref := "master"
+	if version != "" {
+		ref = "go" + version
+	}
+
+	content, err := ff.client.GetFile(ctx, client.FileRequest{
+		Owner: "golang",
+		Repo:  "go",
+		Path:  filepath.Join("src", path),
+		Ref:   ref,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return newFileResponse(content.Content, content.URL)
+}
+
 func (ff FileFinder) fetchGoMod(ctx context.Context) (*modfile.File, error) {
 	sp, ctx := opentracing.StartSpanFromContext(ctx, "fetchGoMod")
 	defer sp.Finish()

pkg/frontend/vcs/source/find_java.go

@@ -5,9 +5,10 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/opentracing/opentracing-go"
+
 	vcsv1 "github.com/grafana/pyroscope/api/gen/proto/go/vcs/v1"
 	"github.com/grafana/pyroscope/pkg/frontend/vcs/config"
-	"github.com/opentracing/opentracing-go"
 )
 
 const (

pkg/frontend/vcs/source/golang/golang.go

@@ -1,7 +1,6 @@
 package golang
 
 import (
-	"fmt"
 	"path/filepath"
 	"strings"
 
@@ -14,37 +13,43 @@ const (
 	stdGoRoot  = "$GOROOT/src/"
 )
 
-var stdLibRegex = regexp.MustCompile(`.*?\/go\/.*?(?P<version>(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?).*?\/src\/(?P<path>.*)`)
+var (
+	stdLibRegex        = regexp.MustCompile(`.*?\/go\/.*?(?P<version>(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?).*?\/src\/(?P<path>.*)`)
+	toolchainGoVersion = regexp.MustCompile(`-go(?P<version>[0-9]+\.[0-9]+(?:\.[0-9]+)?(?:rc[0-9]+|beta[0-9]+)?)(?:\.|-)`)
+)
 
-// StandardLibraryURL returns the URL of the standard library package
-// from the given local path if it exists.
-func StandardLibraryURL(path string) (string, bool) {
+// IsStandardLibraryPath returns the cleaned path of the standard library package and a potential version if detected
+func IsStandardLibraryPath(path string) (string, string, bool) {
 	if len(path) == 0 {
-		return "", false
+		return "", "", false
+	}
+
+	// match toolchain go mod paths
+	if modFile, ok := ParseModuleFromPath(path); ok && modFile.Path == "golang.org/toolchain" {
+		// figure out version
+		matches := toolchainGoVersion.FindStringSubmatch(modFile.Version.Version)
+		if len(matches) > 1 {
+			return strings.TrimPrefix(modFile.FilePath, "src/"), matches[1], true
+		}
 	}
 
 	if stdLibRegex.MatchString(path) {
 		matches := stdLibRegex.FindStringSubmatch(path)
 		version := matches[stdLibRegex.SubexpIndex("version")]
 		path = matches[stdLibRegex.SubexpIndex("path")]
-		return fmt.Sprintf(`https://raw.githubusercontent.com/golang/go/go%s/src/%s`, version, path), true
+		return path, version, true
 	}
 
 	path = strings.TrimPrefix(path, stdLocal)
 	path = strings.TrimPrefix(path, stdGoRoot)
 	fileName := filepath.Base(path)
 	packageName := strings.TrimSuffix(path, "/"+fileName)
-	// Todo: Send more metadata from SDK to fetch the correct version of Go std packages.
-	// For this we should use arbitrary k/v metadata in our request so that we don't need to change the API.
-	// I thought about using go.mod go version but it's a min and doesn't guarantee it hasn't been built with a higher version.
-	// Alternatively we could interpret the build system and use the version of the go compiler.
-	ref := "master"
 	isStdVendor := strings.HasPrefix(packageName, vendorPath)
 
 	if _, isStd := StandardPackages[packageName]; !isStdVendor && !isStd {
-		return "", false
+		return "", "", false
 	}
-	return fmt.Sprintf(`https://raw.githubusercontent.com/golang/go/%s/src/%s`, ref, path), true
+	return path, "", true
 }
 
 // VendorRelativePath returns the relative path of the given path

pkg/frontend/vcs/source/golang/golang_test.go

@@ -6,59 +6,74 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestStandardLibraryURL(t *testing.T) {
+func TestIsStandardLibraryPath(t *testing.T) {
 	for _, tt := range []struct {
-		input      string
-		expected   string
-		expectedOk bool
+		input           string
+		expectedPath    string
+		expectedVersion string
+		expectedOk      bool
 	}{
 		{
 			input:      "github.com/grafana/grafana/pkg/frontend/vcs.go",
-			expected:   "",
 			expectedOk: false,
 		},
 		{
-			input:      "/usr/local/go/src/bufio/bufio.go",
-			expected:   "https://raw.githubusercontent.com/golang/go/master/src/bufio/bufio.go",
-			expectedOk: true,
+			input:        "/usr/local/go/src/bufio/bufio.go",
+			expectedPath: "bufio/bufio.go",
+			expectedOk:   true,
 		},
 		{
-			input:      "$GOROOT/src/unicode/utf8/utf8.go",
-			expected:   "https://raw.githubusercontent.com/golang/go/master/src/unicode/utf8/utf8.go",
-			expectedOk: true,
+			input:        "$GOROOT/src/unicode/utf8/utf8.go",
+			expectedPath: "unicode/utf8/utf8.go",
+			expectedOk:   true,
 		},
 		{
-			input:      "fmt/scan.go",
-			expected:   "https://raw.githubusercontent.com/golang/go/master/src/fmt/scan.go",
-			expectedOk: true,
+			input:        "fmt/scan.go",
+			expectedPath: "fmt/scan.go",
+			expectedOk:   true,
 		},
 		{
-			input:      "$GOROOT/src/vendor/golang.org/x/crypto/cryptobyte/asn1.go",
-			expected:   "https://raw.githubusercontent.com/golang/go/master/src/vendor/golang.org/x/crypto/cryptobyte/asn1.go",
-			expectedOk: true,
+			input:        "$GOROOT/src/vendor/golang.org/x/crypto/cryptobyte/asn1.go",
+			expectedPath: "vendor/golang.org/x/crypto/cryptobyte/asn1.go",
+			expectedOk:   true,
 		},
 		{
-			input:      "/usr/local/go/src/vendor/golang.org/x/net/http2/hpack/tables.go",
-			expected:   "https://raw.githubusercontent.com/golang/go/master/src/vendor/golang.org/x/net/http2/hpack/tables.go",
-			expectedOk: true,
+			input:        "/usr/local/go/src/vendor/golang.org/x/net/http2/hpack/tables.go",
+			expectedPath: "vendor/golang.org/x/net/http2/hpack/tables.go",
+			expectedOk:   true,
 		},
 		{
-			input:      "/usr/local/Cellar/go/1.21.3/libexec/src/runtime/netpoll_kqueue.go",
-			expected:   "https://raw.githubusercontent.com/golang/go/go1.21.3/src/runtime/netpoll_kqueue.go",
-			expectedOk: true,
+			input:           "/usr/local/Cellar/go/1.21.3/libexec/src/runtime/netpoll_kqueue.go",
+			expectedPath:    "runtime/netpoll_kqueue.go",
+			expectedVersion: "1.21.3",
+			expectedOk:      true,
 		},
 		{
-			input:      "/opt/hostedtoolcache/go/1.21.6/x64/src/runtime/mgc.go",
-			expected:   "https://raw.githubusercontent.com/golang/go/go1.21.6/src/runtime/mgc.go",
-			expectedOk: true,
+			input:           "/opt/hostedtoolcache/go/1.21.6/x64/src/runtime/mgc.go",
+			expectedPath:    "runtime/mgc.go",
+			expectedVersion: "1.21.6",
+			expectedOk:      true,
+		},
+		{
+			input:           "/Users/pyroscope/.golang/packages/pkg/mod/golang.org/toolchain@v0.0.1-go1.24.6.darwin-arm64/src/runtime/proc.go",
+			expectedPath:    "runtime/proc.go",
+			expectedVersion: "1.24.6",
+			expectedOk:      true,
+		},
+		{
+			input:           "/Users/christian/.golang/packages/pkg/mod/golang.org/toolchain@v0.0.1-go1.25rc1.darwin-arm64/src/runtime/type.go",
+			expectedPath:    "runtime/type.go",
+			expectedVersion: "1.25rc1",
+			expectedOk:      true,
 		},
 	} {
 		t.Run(tt.input, func(t *testing.T) {
-			actual, ok := StandardLibraryURL(tt.input)
+			actualPath, actualVersion, ok := IsStandardLibraryPath(tt.input)
 			if !tt.expectedOk {
 				require.False(t, ok)
 			}
-			require.Equal(t, tt.expected, actual)
+			require.Equal(t, tt.expectedPath, actualPath)
+			require.Equal(t, tt.expectedVersion, actualVersion)
 		})
 	}
 }