feat: improve structured logging with environment-based formatting and verbosity levels (#534)

# Improved Structured Logging for Better Developer Experience

This PR enhances the logging system in Edge Worker with significant developer experience improvements:

- Added a new `verbose` log level between `info` and `debug` for task outcomes
- Implemented environment detection to automatically set appropriate defaults:
  - Local development: Colorful "fancy" format at `verbose` level
  - Hosted environments: Structured key=value format at `info` level
- Created two formatter types:
  - Fancy formatter with colored icons, worker-prefixed lines, and flow/step paths
  - Simple formatter with structured key=value output optimized for log aggregators
- Added task duration display and retry information with exponential backoff delay calculation
- Added support for the NO_COLOR standard
- Introduced environment variable controls:
  - `EDGE_WORKER_LOG_LEVEL` to set verbosity
  - `EDGE_WORKER_LOG_FORMAT` to choose output style

The documentation has been updated with comprehensive logging configuration details, examples of both output formats, and a clearer explanation of the log level hierarchy.

Author: jumski

.changeset/structured-logging-dx.md

@@ -0,0 +1,12 @@
+---
+'@pgflow/edge-worker': patch
+---
+
+Major developer experience improvements with structured logging:
+
+- Add verbose log level between info and debug for task outcomes
+- Auto-detect local vs hosted environment for log format and level defaults
+- Fancy formatter for local dev with colored icons, worker-prefixed lines, and flow/step paths
+- Simple formatter for production with structured key=value output for log aggregators
+- Display task duration, retry information with exponential backoff delay calculation
+- Support NO_COLOR standard and EDGE_WORKER_LOG_LEVEL/EDGE_WORKER_LOG_FORMAT env vars

pkgs/cli/scripts/assert-pgflow-installed

@@ -19,11 +19,6 @@ if [ ! -d "${SUPABASE_PATH}/migrations" ] || [ -z "$(ls -A ${SUPABASE_PATH}/migr
   error "No SQL migration files found in ${SUPABASE_PATH}/migrations/"
 fi
 
-# Verify environment variables in .env file
-if ! grep -q 'EDGE_WORKER_LOG_LEVEL' "${SUPABASE_PATH}/functions/.env"; then
-  error "EDGE_WORKER_LOG_LEVEL not found in ${SUPABASE_PATH}/functions/.env"
-fi
-
 # Verify per_worker policy in config.toml
 if ! grep -q 'policy = "per_worker"' "${SUPABASE_PATH}/config.toml"; then
   error "policy = \"per_worker\" not found in ${SUPABASE_PATH}/config.toml"

pkgs/cli/src/commands/install/index.ts

@@ -3,7 +3,6 @@ import { intro, log, confirm, cancel, outro } from '@clack/prompts';
 import chalk from 'chalk';
 import { copyMigrations } from './copy-migrations.js';
 import { updateConfigToml } from './update-config-toml.js';
-import { updateEnvFile } from './update-env-file.js';
 import { createEdgeFunction } from './create-edge-function.js';
 import { createFlowsDirectory } from './create-flows-directory.js';
 import { createExampleWorker } from './create-example-worker.js';
@@ -39,7 +38,6 @@ export default (program: Command) => {
         `  • Create ${chalk.cyan('supabase/flows/')} ${chalk.dim('(flow definitions with GreetUser example)')}`,
         `  • Create Control Plane in ${chalk.cyan('supabase/functions/pgflow/')}`,
         `  • Create ${chalk.cyan('supabase/functions/greet-user-worker/')} ${chalk.dim('(example worker)')}`,
-        `  • Configure ${chalk.cyan('supabase/functions/.env')}`,
         '',
         `  ${chalk.green('✓ Safe to re-run - completed steps will be skipped')}`,
       ].join('\n');
@@ -92,15 +90,10 @@ export default (program: Command) => {
         autoConfirm: true,
       });
 
-      const envFile = await updateEnvFile({
-        supabasePath,
-        autoConfirm: true,
-      });
-
       // Step 4: Show completion message
       const outroMessages: string[] = [];
 
-      if (migrations || configUpdate || flowsDirectory || edgeFunction || exampleWorker || envFile) {
+      if (migrations || configUpdate || flowsDirectory || edgeFunction || exampleWorker) {
         outroMessages.push(chalk.green.bold('✓ Installation complete!'));
       } else {
         outroMessages.push(
@@ -114,7 +107,7 @@ export default (program: Command) => {
 
       let stepNumber = 1;
 
-      if (configUpdate || envFile) {
+      if (configUpdate) {
         outroMessages.push(
           `  ${stepNumber}. Restart Supabase: ${chalk.cyan('supabase stop && supabase start')}`
         );

pkgs/cli/src/commands/install/update-env-file.ts

@@ -15,6 +15,12 @@ export type LoggingEnv = Record<string, string | undefined>;
 // ANSI Color Codes (16-color safe palette)
 // ============================================================
 
+/**
+ * Default base delay in seconds for retry exponential backoff.
+ * Matches pgflow SQL default in create_flow/add_step functions.
+ */
+const DEFAULT_BASE_DELAY_SECONDS = 5;
+
 const ANSI = {
   // Colors
   blue: '\x1b[34m',
@@ -117,8 +123,7 @@ class FancyFormatter {
       const retryIcon = colorize('↻', ANSI.yellow, this.colorsEnabled);
       // Exponential backoff: baseDelay * 2^(attempt-1)
       // Example with baseDelay=5: attempt 1 -> 5s, attempt 2 -> 10s, attempt 3 -> 20s
-      // Default 5s matches pgflow SQL default in create_flow/add_step
-      const baseDelay = ctx.baseDelay ?? 5;
+      const baseDelay = ctx.baseDelay ?? DEFAULT_BASE_DELAY_SECONDS;
       const delay = baseDelay * Math.pow(2, ctx.retryAttempt - 1);
       const retryInfo = colorize(`retry ${ctx.retryAttempt}/${ctx.maxRetries} in ${delay}s`, ANSI.yellow, this.colorsEnabled);
       result += `\n${prefix}   ${retryIcon} ${retryInfo}`;
@@ -205,9 +210,8 @@ class SimpleFormatter {
     let result = `[VERBOSE] worker=${ctx.workerName} queue=${ctx.queueName} flow=${ctx.flowSlug} step=${ctx.stepSlug} status=failed error="${error.message}" run_id=${ctx.runId} msg_id=${ctx.msgId} worker_id=${ctx.workerId}`;
 
     // Add retry info if there are retries remaining (see FancyFormatter for detailed comments)
-    // Default 5s matches pgflow SQL default in create_flow/add_step
     if (ctx.retryAttempt !== undefined && ctx.maxRetries !== undefined && ctx.retryAttempt <= ctx.maxRetries) {
-      const baseDelay = ctx.baseDelay ?? 5;
+      const baseDelay = ctx.baseDelay ?? DEFAULT_BASE_DELAY_SECONDS;
       const delay = baseDelay * Math.pow(2, ctx.retryAttempt - 1);
       result += ` retry=${ctx.retryAttempt}/${ctx.maxRetries} retry_delay_s=${delay}`;
     }
@@ -301,7 +305,6 @@ export function createLoggingFactory(env?: LoggingEnv) {
         const levelValue =
           levels[logLevel as keyof typeof levels] ?? levels.info;
         if (levelValue >= levels.debug) {
-          // Use console.log for debug messages since console.debug isn't available in Supabase
           console.debug(
             `worker_id=${sharedWorkerId} module=${module} ${message}`,
             ...args
@@ -312,7 +315,6 @@ export function createLoggingFactory(env?: LoggingEnv) {
         const levelValue =
           levels[logLevel as keyof typeof levels] ?? levels.info;
         if (levelValue >= levels.info) {
-          // Use console.log for info messages since console.info isn't available in Supabase
           console.info(
             `worker_id=${sharedWorkerId} module=${module} ${message}`,
             ...args
@@ -397,7 +399,8 @@ export function createLoggingFactory(env?: LoggingEnv) {
           levels[logLevel as keyof typeof levels] ?? levels.info;
         if (levelValue >= levels.info) {
           const lines = formatter.startupBanner(ctx);
-          lines.forEach(line => console.info(line));
+          // Join lines to reduce blank log entries in Supabase Edge Functions
+          console.info(lines.join('\n'));
         }
       },
 

pkgs/edge-worker/src/platform/logging.ts

@@ -7,22 +7,28 @@ sidebar:
 
 ## Logging
 
-Edge Worker logs various events to the console. You can change the log level
-by setting `EDGE_WORKER_LOG_LEVEL` environment variable:
+Edge Worker automatically configures logging based on your environment:
+- **Local development**: Colorful "fancy" format at `verbose` level
+- **Hosted Supabase**: Structured key=value format at `info` level
+
+You can override the log level by setting `EDGE_WORKER_LOG_LEVEL`:
 
 ```bash
-// supabase/functions/.env
+# supabase/functions/.env
 EDGE_WORKER_LOG_LEVEL=debug
 ```
 
-Available log levels are:
+Available log levels (from least to most verbose):
 
-- `debug`
-- `info`
-- `error`
-- (more will come)
+| Level | Description |
+|-------|-------------|
+| `error` | Critical failures only |
+| `warn` | Deprecation notices, auth failures |
+| `info` | Startup banner, shutdown events |
+| `verbose` | Task outcomes, polling feedback |
+| `debug` | Task started events, detailed identifiers |
 
-By default, Edge Worker's log level is `info`.
+For complete logging configuration including output format options and examples, see [Worker Logging Configuration](/reference/configuration/worker/#logging-configuration).
 
 ## Heartbeats
 

pkgs/website/src/content/docs/deploy/monitor-workers-health.mdx

@@ -117,6 +117,63 @@ How frequently (in milliseconds) to check the database for new tasks during the
 
 Your flow step handlers can access the resolved worker configuration through the context object. For detailed information about available configuration options and usage patterns, see [`workerConfig`](/reference/context/#workerconfig).
 
+## Logging Configuration
+
+Edge Worker automatically configures logging based on your environment, but you can override the defaults with environment variables.
+
+### Environment Variables
+
+#### `EDGE_WORKER_LOG_LEVEL`
+**Type:** `'error' | 'warn' | 'info' | 'verbose' | 'debug'`
+
+**Default:** `'verbose'` (local), `'info'` (hosted)
+
+Controls log verbosity. The hierarchy is: error < warn < info < verbose < debug.
+
+| Level | Description |
+|-------|-------------|
+| error | Critical failures only |
+| warn | Deprecation notices, auth failures |
+| info | Startup banner, shutdown (hosted default) |
+| verbose | Task outcomes, polling feedback (local default) |
+| debug | Task started events, identifiers (run_id, msg_id, worker_id) |
+
+#### `EDGE_WORKER_LOG_FORMAT`
+**Type:** `'fancy' | 'simple'`
+
+**Default:** Auto-detected (`'fancy'` for local, `'simple'` for hosted)
+
+- **fancy**: Colored output with icons, worker-prefixed lines, and human-readable formatting for local development
+- **simple**: Structured `key=value` pairs for log aggregators in production
+
+#### `NO_COLOR`
+**Type:** `string` (any value)
+
+**Default:** `undefined`
+
+Set to any value to disable ANSI colors in fancy mode. Follows the [NO_COLOR standard](https://no-color.org/).
+
+### Log Output Examples
+
+**Fancy format (local development):**
+```
+➜ my-worker [abc123]
+   Queue: my_flow
+   Flows: ✓ myFlow (compiled)
+
+my-worker: ✓ myFlow/fetchData  234ms
+my-worker: ✗ myFlow/sendEmail
+my-worker:   ValidationError: invalid email
+my-worker:   ↻ retry 1/3 in 5s
+```
+
+**Simple format (production):**
+```
+[INFO] worker=my-worker queue=my_flow flow=myFlow status=compiled worker_id=abc123
+[VERBOSE] worker=my-worker flow=myFlow step=fetchData status=completed duration_ms=234
+[VERBOSE] worker=my-worker flow=myFlow step=sendEmail status=failed error="ValidationError: invalid email" retry=1/3 retry_delay_s=5
+```
+
 ## Background Jobs Mode Differences
 
 When using Edge Worker in [Background Jobs Mode](/get-started/faq/#what-are-the-two-edge-worker-modes) (without pgflow orchestration), there are a few key differences: