See more detailed CPu measurements

Author: Pijukatel

src/crawlee/_utils/system.py

@@ -93,8 +93,9 @@ def get_cpu_info() -> CpuInfo:
     It utilizes the `psutil` library. Function `psutil.cpu_percent()` returns a float representing the current
     system-wide CPU utilization as a percentage.
     """
-    logger.debug('Calling get_cpu_info()...')
-    cpu_percent = psutil.cpu_percent(interval=0.1)
+
+    cpu_percent = psutil.cpu_percent()
+    logger.info(f'Calling get_cpu_info()...: {cpu_percent}')
     return CpuInfo(used_ratio=cpu_percent / 100)
 
 

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -12,9 +12,11 @@
 from dataclasses import dataclass
 from datetime import timedelta
 from itertools import product
+from subprocess import run
 from typing import TYPE_CHECKING, Any, Literal, cast
 from unittest.mock import AsyncMock, Mock, call, patch
 
+import psutil
 import pytest
 
 from crawlee import ConcurrencySettings, Glob, service_locator
@@ -1290,6 +1292,13 @@ async def test_timeout_in_handler(sleep_type: str, _) -> None:
     # Test is skipped in older Python versions.
     from asyncio import timeout  # type:ignore[attr-defined] # noqa: PLC0415
 
+    # Debug CPu usage before starting the test
+    run("ps -efaxo comm,user,cmd,pid,%cpu", shell=True)
+
+    run("ps -axo pid,comm,%cpu", shell=True)
+
+
+
     handler_timeout = timedelta(seconds=1)
     max_request_retries = 3
     double_handler_timeout_s = handler_timeout.total_seconds() * 2