See measurements problems with CPU usage

Author: Pijukatel

src/crawlee/_autoscaling/autoscaled_pool.py

@@ -61,7 +61,7 @@ class AutoscaledPool:
     _TASK_TIMEOUT: timedelta | None = None
     """Timeout within which the `run_task_function` must complete."""
 
-    _OVERLOADED_BACKOFF_TIME: timedelta = timedelta(seconds=0.5)
+    _OVERLOADED_BACKOFF_TIME: timedelta = timedelta(seconds=1)
     """When overloaded, Autoscaled pool waits this long before rechecking system status."""
 
     def __init__(
@@ -240,7 +240,7 @@ async def _worker_task_orchestrator(self, run: _AutoscaledPoolRun) -> None:
 
                 current_status = self._system_status.get_current_system_info()
                 if not current_status.is_system_idle:
-                    logger.info('Not scheduling new tasks - system is overloaded')
+                    logger.info(f'Not scheduling new tasks - system is overloaded: {current_status}')
                     await asyncio.sleep(self._OVERLOADED_BACKOFF_TIME.total_seconds())
                     logger.info('Release the overloaded backoff')
                 elif self._is_paused:

src/crawlee/events/_local_event_manager.py

@@ -93,8 +93,8 @@ async def __aexit__(
 
     async def _emit_system_info_event(self) -> None:
         """Emit a system info event with the current CPU and memory usage."""
-        cpu_info = await asyncio.to_thread(get_cpu_info)
-        memory_info = await asyncio.to_thread(get_memory_info)
+        cpu_info = get_cpu_info()
+        memory_info = get_memory_info()
 
         event_data = EventSystemInfoData(cpu_info=cpu_info, memory_info=memory_info)
         self.emit(event=Event.SYSTEM_INFO, event_data=event_data)