event_logs, telemetry_export: do not interrupt insert on context cancel (#57552)

When a request context is cancelled, the [corresponding event database insert can fail on context cancellation](https://sourcegraph.slack.com/archives/C03V51C8K53/p1697072391851359) - but this is highly unlikely to be desirable behaviour, since we probably want to proceed with persisting the event regardless.

This change introduces a new helper in `internal/trace`, `BackgroundContext`, which aliases `CopyContext(context.Background(), from)` with some more docstrings on when to use it and how to use it correctly. We use this new background context in the most inner part of `(TelemetryEventsExportQueueStore).QueueForExport` and `(EventLogStore).BulkInsert` to ensure we don't interrupt database inserts.

Author: bobheadxi

internal/database/BUILD.bazel

@@ -149,6 +149,7 @@ go_library(
         "//internal/trace",
         "//internal/types",
         "//internal/version",
+        "//internal/xcontext",
         "//lib/errors",
         "//lib/pointers",
         "//schema",

internal/database/event_logs.go

@@ -26,6 +26,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/trace"
 	"github.com/sourcegraph/sourcegraph/internal/types"
 	"github.com/sourcegraph/sourcegraph/internal/version"
+	"github.com/sourcegraph/sourcegraph/internal/xcontext"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
 
@@ -45,6 +46,10 @@ type EventLogStore interface {
 	// AggregatedSearchEvents calculates SearchAggregatedEvent for each every unique event type related to search.
 	AggregatedSearchEvents(ctx context.Context, now time.Time) ([]types.SearchAggregatedEvent, error)
 
+	// BulkInsert inserts a set of events.
+	//
+	// It does NOT respect context cancellation, as it is assumed that we never
+	// drop events once we attempt to queue it for export.
 	BulkInsert(ctx context.Context, events []*Event) error
 
 	// CodeIntelligenceCrossRepositoryWAUs returns the WAU (current week) with any (precise or search-based) cross-repository code intelligence event.
@@ -289,8 +294,14 @@ func (l *eventLogStore) BulkInsert(ctx context.Context, events []*Event) error {
 	}
 	close(rowValues)
 
+	// Create a cancel-free context to avoid interrupting the insert when
+	// the parent context is cancelled, and add our own timeout on the insert
+	// to make sure things don't get stuck in an unbounded manner.
+	insertCtx, cancel := context.WithTimeout(xcontext.Detach(ctx), 5*time.Minute)
+	defer cancel()
+
 	return batch.InsertValues(
-		ctx,
+		insertCtx,
 		l.Handle(),
 		"event_logs",
 		batch.MaxNumPostgresParameters,

internal/database/telemetry_export_store.go

@@ -19,6 +19,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/telemetry/sensitivemetadataallowlist"
 	telemetrygatewayv1 "github.com/sourcegraph/sourcegraph/internal/telemetrygateway/v1"
 	"github.com/sourcegraph/sourcegraph/internal/trace"
+	"github.com/sourcegraph/sourcegraph/internal/xcontext"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
 
@@ -40,6 +41,9 @@ type TelemetryEventsExportQueueStore interface {
 	// feature-flagged, such that if the flag is not enabled for the given
 	// context, we do not cache the event for export.
 	//
+	// It does NOT respect context cancellation, as it is assumed that we never
+	// drop events once we attempt to queue it for export.
+	//
 	// 🚨 SECURITY: The implementation strips out sensitive contents from events
 	// that are not in sensitivemetadataallowlist.AllowedEventTypes().
 	QueueForExport(context.Context, []*telemetrygatewayv1.Event) error
@@ -91,7 +95,14 @@ func (s *telemetryEventsExportQueueStore) QueueForExport(ctx context.Context, ev
 		return nil
 	}
 
-	err := batch.InsertValues(ctx,
+	// Create a cancel-free context to avoid interrupting the insert when
+	// the parent context is cancelled, and add our own timeout on the insert
+	// to make sure things don't get stuck in an unbounded manner.
+	insertCtx, cancel := context.WithTimeout(xcontext.Detach(ctx), 5*time.Minute)
+	defer cancel()
+
+	err := batch.InsertValues(
+		insertCtx,
 		s.Handle(),
 		"telemetry_events_export_queue",
 		batch.MaxNumPostgresParameters,