feat: add verbose log level between info and debug for improved observability (#529)

# Add verbose log level and improve task execution logging

This PR introduces a new `verbose` log level between `info` and `debug` to provide better granularity for operational logging. The change:

- Adds a `verbose` method to the `Logger` interface
- Updates the logging hierarchy to: `error < warn < info < verbose < debug`
- Moves frequent operational logs from `debug` to `verbose` level
- Adds execution duration tracking for step tasks
- Improves log messages for task execution with timing information
- Adds comprehensive unit tests for the new logging level

The verbose level allows us to keep operational visibility without the excessive detail of debug logs, making it easier to monitor production systems.

Author: jumski

pkgs/edge-worker/deno.lock

@@ -18,15 +18,15 @@ export class BatchProcessor<TMessage extends IMessage> {
   }
 
   async processBatch() {
-    this.logger.debug('Polling for new batch of messages...');
+    this.logger.verbose('Polling for new batch of messages...');
     const messageRecords = await this.poller.poll();
 
     if (this.signal.aborted) {
       this.logger.info('Discarding messageRecords because worker is stopping');
       return;
     }
 
-    this.logger.debug(`Starting ${messageRecords.length} messages`);
+    this.logger.verbose(`Starting ${messageRecords.length} tasks`);
 
     const startPromises = messageRecords.map((message) =>
       this.executionController.start(message)

pkgs/edge-worker/src/core/BatchProcessor.ts

@@ -62,13 +62,17 @@ export class StepTaskExecutor<TFlow extends AnyFlow, TContext extends StepTaskHa
         throw new Error(`No step definition found for slug=${stepSlug}`);
       }
 
+      // Measure handler execution duration
+      const startTime = Date.now();
 
       // !!! HANDLER EXECUTION !!!
       const result = await stepDef.handler(this.stepTask.input, this.context);
       // !!! HANDLER EXECUTION !!!
 
-      this.logger.debug(
-        `step task ${this.msgId} completed successfully, marking as complete`
+      const durationMs = Date.now() - startTime;
+
+      this.logger.verbose(
+        `step task ${this.msgId} completed in ${durationMs}ms`
       );
       await this.adapter.completeTask(this.stepTask, result);
 
@@ -92,7 +96,7 @@ export class StepTaskExecutor<TFlow extends AnyFlow, TContext extends StepTaskHa
       // Do not mark as failed - the worker was aborted and stopping,
       // the task will be picked up by another worker later
     } else {
-      this.logger.error(
+      this.logger.verbose(
         `step task ${this.msgId} failed with error: ${error}`
       );
       await this.adapter.failTask(this.stepTask, error);

pkgs/edge-worker/src/flow/StepTaskExecutor.ts

@@ -12,7 +12,8 @@ export function createLoggingFactory() {
   const loggers: Map<string, Logger> = new Map();
 
   // Simple level filtering
-  const levels = { error: 0, warn: 1, info: 2, debug: 3 };
+  // Hierarchy: error < warn < info < verbose < debug
+  const levels = { error: 0, warn: 1, info: 2, verbose: 3, debug: 4 };
 
   /**
    * Creates a new logger for a specific module
@@ -42,6 +43,17 @@ export function createLoggingFactory() {
           );
         }
       },
+      verbose: (message, ...args) => {
+        const levelValue =
+          levels[logLevel as keyof typeof levels] ?? levels.info;
+        if (levelValue >= levels.verbose) {
+          // Use console.log for verbose messages
+          console.log(
+            `worker_id=${sharedWorkerId} module=${module} ${message}`,
+            ...args
+          );
+        }
+      },
       warn: (message, ...args) => {
         const levelValue =
           levels[logLevel as keyof typeof levels] ?? levels.info;

pkgs/edge-worker/src/platform/logging.ts

@@ -5,6 +5,7 @@ import type { Worker } from '../core/Worker.js';
  */
 export interface Logger {
   debug(message: string, ...args: unknown[]): void;
+  verbose(message: string, ...args: unknown[]): void;
   info(message: string, ...args: unknown[]): void;
   warn(message: string, ...args: unknown[]): void;
   error(message: string, ...args: unknown[]): void;

pkgs/edge-worker/src/platform/types.ts

@@ -4,6 +4,7 @@ const noop = () => {};
 
 export const fakeLogger: Logger = {
   debug: noop,
+  verbose: noop,
   info: noop,
   warn: noop,
   error: noop,

pkgs/edge-worker/tests/fakes.ts

@@ -66,6 +66,7 @@ export function startWorker<TFlow extends AnyFlow>(
 
   const consoleLogger = {
     debug: console.log,
+    verbose: console.log,
     info: console.log,
     warn: console.warn,
     error: console.error,

pkgs/edge-worker/tests/integration/_helpers.ts

@@ -18,6 +18,7 @@ const TestCompilationFlow = new Flow<{ value: number }>({ slug: 'test_compilatio
 function createLogger(module: string) {
   return {
     debug: console.log.bind(console, `[${module}]`),
+    verbose: console.log.bind(console, `[${module}]`),
     info: console.log.bind(console, `[${module}]`),
     warn: console.warn.bind(console, `[${module}]`),
     error: console.error.bind(console, `[${module}]`),

pkgs/edge-worker/tests/integration/flow/compilationAtStartup.test.ts

@@ -55,6 +55,7 @@ const createMockFlow = () => TestFlow;
 
 const createLogger = (): Logger => ({
   debug: () => {},
+  verbose: () => {},
   info: () => {},
   error: () => {},
   warn: () => {},
@@ -128,6 +129,7 @@ Deno.test('FlowWorkerLifecycle - logs skip message when ensureCompiledOnStartup
   const logs: string[] = [];
   const testLogger: Logger = {
     debug: () => {},
+    verbose: () => {},
     info: (msg: string) => logs.push(msg),
     error: () => {},
     warn: () => {},
@@ -154,6 +156,7 @@ Deno.test('FlowWorkerLifecycle - does not log skip message when ensureCompiledOn
   const logs: string[] = [];
   const testLogger: Logger = {
     debug: () => {},
+    verbose: () => {},
     info: (msg: string) => logs.push(msg),
     error: () => {},
     warn: () => {},

pkgs/edge-worker/tests/unit/FlowWorkerLifecycle.compilation.test.ts

@@ -185,6 +185,7 @@ Deno.test('FlowWorkerLifecycle - should log appropriate message when transitioni
   const logs: string[] = [];
   const testLogger: Logger = {
     debug: () => {},
+    verbose: () => {},
     info: (msg: string) => logs.push(msg),
     error: () => {},
     warn: () => {},