Fix instability in CI caused by health check tests (#1584)

Author: TimPansino

tests/agent_features/test_agent_control_health_check.py

@@ -38,7 +38,7 @@ def get_health_file_contents(tmp_path):
         return contents
 
 
-@pytest.fixture(scope="module", autouse=True)
+@pytest.fixture(autouse=True)
 def restore_settings_fixture():
     # Backup settings from before this test file runs
     original_settings = global_settings()
@@ -51,6 +51,10 @@ def restore_settings_fixture():
     original_settings.__dict__.clear()
     original_settings.__dict__.update(backup)
 
+    # Re-initialize the agent to restore the settings
+    _reset_configuration_done()
+    initialize()
+
 
 @pytest.mark.parametrize("file_uri", ["", "file://", "/test/dir", "foo:/test/dir"])
 def test_invalid_file_directory_supplied(monkeypatch, file_uri):
@@ -155,10 +159,18 @@ def test_no_override_on_unhealthy_shutdown(monkeypatch, tmp_path):
 
 
 def test_health_check_running_threads(monkeypatch, tmp_path):
-    running_threads = threading.enumerate()
-    # Only the main thread should be running since not agent control env vars are set
-    assert len(running_threads) == 1
+    # If the Activate-Session thread is still active, give it time to close before we proceed
+    timeout = 30.0
+    while len(threading.enumerate()) != 1 and timeout > 0:
+        time.sleep(0.1)
+        timeout -= 0.1
 
+    # Only the main thread should be running since no agent control env vars are set
+    assert len(threading.enumerate()) == 1, (
+        f"Expected only the main thread to be running before the test starts. Got: {threading.enumerate()}"
+    )
+
+    # Setup expected env vars to run agent control health check
     monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_ENABLED", "True")
     file_path = tmp_path.as_uri()
     monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_HEALTH_DELIVERY_LOCATION", file_path)
@@ -180,6 +192,7 @@ def test_proxy_error_status(monkeypatch, tmp_path):
     file_path = tmp_path.as_uri()
     monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_HEALTH_DELIVERY_LOCATION", file_path)
 
+    # Re-initialize the agent to allow the health check thread to start
     _reset_configuration_done()
     initialize()
 
@@ -209,6 +222,7 @@ def test_multiple_activations_running_threads(monkeypatch, tmp_path):
     file_path = tmp_path.as_uri()
     monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_HEALTH_DELIVERY_LOCATION", file_path)
 
+    # Re-initialize the agent to allow the health check thread to start and assert that it did
     _reset_configuration_done()
     initialize()