Build: disable builds after 25 consecutive failed builds on default version

docs/user/builds.rst

@@ -112,6 +112,25 @@ Read the Docs supports three different mechanisms to cancel a running build:
 
       Take a look at :ref:`build-customization:cancel build based on a condition` section for some examples.
 
+Automatic disabling of builds
+-----------------------------
+
+To reduce resource consumption and improve build queue times for all users,
+Read the Docs will automatically disable builds for projects that have too many consecutive failed builds on their default version.
+
+When a project has **50 consecutive failed builds** on its default version,
+we will disable builds for the project.
+
+This helps ensure that projects with persistent build issues don't consume resources that could be used by active projects.
+
+.. note::
+
+   This only applies to the default version of a project.
+   Builds on other versions (branches, tags, pull requests) are not counted towards this limit.
+
+If your project has been disabled due to consecutive build failures, you'll need to contact support to have your project re-enabled.
+Make sure to fix the underlying issue to avoid being disabled again.
+
 Build resources
 ---------------
 

readthedocs/builds/signals_receivers.py

@@ -4,11 +4,24 @@
 NOTE: Done in a separate file to avoid circular imports.
 """
 
+from itertools import groupby
+
+import structlog
+from django.conf import settings
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 
+from readthedocs.builds.constants import BUILD_STATE_FINISHED
 from readthedocs.builds.models import Build
+from readthedocs.builds.signals import build_complete
+from readthedocs.notifications.models import Notification
 from readthedocs.projects.models import Project
+from readthedocs.projects.notifications import (
+    MESSAGE_PROJECT_BUILDS_DISABLED_DUE_TO_CONSECUTIVE_FAILURES,
+)
+
+
+log = structlog.get_logger(__name__)
 
 
 @receiver(post_save, sender=Build)
@@ -18,3 +31,74 @@ def update_latest_build_for_project(sender, instance, created, **kwargs):
         Project.objects.filter(pk=instance.project_id).update(
             latest_build=instance,
         )
+
+
+@receiver(build_complete, sender=Build)
+def disable_project_on_consecutive_failed_builds(sender, build, **kwargs):
+    """
+    Disable a project if it has too many consecutive failed builds on the default version.
+
+    When a project has more than RTD_BUILDS_MAX_CONSECUTIVE_FAILURES consecutive failed builds on the default version,
+    we attach a notification to the project and disable builds (skip=True).
+    This helps reduce resource consumption from projects that are not being monitored.
+    """
+    # Build is a dict coming from the task, not a Build instance
+    if not isinstance(build, dict):
+        return
+
+    # Only check on failed builds
+    if build.get("success"):
+        return
+
+    project_id = build.get("project")
+    version_slug = build.get("version_slug")
+
+    try:
+        project = Project.objects.get(pk=project_id)
+    except Project.DoesNotExist:
+        return
+
+    # Only check for the default version
+    if version_slug != project.get_default_version():
+        return
+
+    # Skip if the project is already disabled
+    if project.skip:
+        return
+
+    # Count consecutive failed builds on the default version
+    builds = (
+        Build.objects.filter(
+            project=project,
+            version_slug=version_slug,
+            state=BUILD_STATE_FINISHED,
+        )
+        .order_by("-date")
+        .values_list("success", flat=True)[: settings.RTD_BUILDS_MAX_CONSECUTIVE_FAILURES + 1]
+    )
+    for success, group in groupby(builds):
+        consecutive_failed_builds = len(list(group))
+        if not success and consecutive_failed_builds > settings.RTD_BUILDS_MAX_CONSECUTIVE_FAILURES:
+            log.info(
+                "Disabling project due to consecutive failed builds.",
+                project_slug=project.slug,
+                version_slug=version_slug,
+                consecutive_failed_builds=consecutive_failed_builds,
+            )
+
+            # Disable the project
+            project.skip = True
+            project.save()
+
+            # Attach notification to the project
+            Notification.objects.add(
+                message_id=MESSAGE_PROJECT_BUILDS_DISABLED_DUE_TO_CONSECUTIVE_FAILURES,
+                attached_to=project,
+                dismissable=False,
+                format_values={
+                    "consecutive_failed_builds": consecutive_failed_builds,
+                },
+            )
+
+            # If we already detected the threshold, we can stop checking
+            break

readthedocs/builds/tests/test_signals_receivers.py

@@ -0,0 +1,63 @@
+"""Tests for build signal receivers."""
+
+from django.conf import settings
+from django.test import TestCase, override_settings
+from django_dynamic_fixture import get
+
+from readthedocs.builds.constants import BUILD_STATE_FINISHED, LATEST
+from readthedocs.builds.models import Build
+from readthedocs.builds.signals_receivers import (
+    disable_project_on_consecutive_failed_builds,
+)
+from readthedocs.notifications.models import Notification
+from readthedocs.projects.models import Project
+from readthedocs.projects.notifications import (
+    MESSAGE_PROJECT_BUILDS_DISABLED_DUE_TO_CONSECUTIVE_FAILURES,
+)
+
+
+class TestDisableProjectOnConsecutiveFailedBuilds(TestCase):
+    def setUp(self):
+        self.project = get(Project, slug="test-project", skip=False)
+        self.version = self.project.versions.get(slug=LATEST)
+        self.version.active = True
+        self.version.save()
+
+    def _create_builds(self, count, success=False):
+        """Helper to create a series of builds."""
+        builds = []
+        for _ in range(count):
+            build = get(
+                Build,
+                project=self.project,
+                version=self.version,
+                success=success,
+                state=BUILD_STATE_FINISHED,
+            )
+            builds.append(build)
+        return builds
+
+    @override_settings(RTD_BUILDS_MAX_CONSECUTIVE_FAILURES=50)
+    def test_signal_handler_disables_at_threshold(self):
+        """Test that the project is disabled at the failure threshold."""
+        # Create failures at the threshold
+        self._create_builds(settings.RTD_BUILDS_MAX_CONSECUTIVE_FAILURES + 1, success=False)
+
+        build_dict = {
+            "success": False,
+            "project": self.project.pk,
+            "version_slug": self.version.slug,
+        }
+        disable_project_on_consecutive_failed_builds(
+            sender=Build, build=build_dict
+        )
+
+        self.project.refresh_from_db()
+        self.assertTrue(self.project.skip)
+
+        # Verify notification was added
+        notification = Notification.objects.filter(
+            message_id=MESSAGE_PROJECT_BUILDS_DISABLED_DUE_TO_CONSECUTIVE_FAILURES
+        ).first()
+        self.assertIsNotNone(notification)
+        self.assertEqual(notification.attached_to, self.project)

readthedocs/projects/notifications.py

@@ -20,6 +20,9 @@
 MESSAGE_PROJECT_SSH_KEY_WITH_WRITE_ACCESS = "project:ssh-key-with-write-access"
 MESSAGE_PROJECT_DEPRECATED_WEBHOOK = "project:webhooks:deprecated"
 MESSAGE_PROJECT_SEARCH_INDEXING_DISABLED = "project:search:indexing-disabled"
+MESSAGE_PROJECT_BUILDS_DISABLED_DUE_TO_CONSECUTIVE_FAILURES = (
+    "project:builds:disabled-due-to-consecutive-failures"
+)
 
 messages = [
     Message(
@@ -223,5 +226,18 @@
         ),
         type=INFO,
     ),
+    Message(
+        id=MESSAGE_PROJECT_BUILDS_DISABLED_DUE_TO_CONSECUTIVE_FAILURES,
+        header=_("Builds disabled due to consecutive failures"),
+        body=_(
+            textwrap.dedent(
+                """
+                Your project has been automatically disabled because the default version has failed to build {{consecutive_failed_builds}} times in a row.
+                Please fix the build issues and <a href="mailto:{{SUPPORT_EMAIL}}">contact support</a> to re-enable builds.
+                """
+            ).strip(),
+        ),
+        type=WARNING,
+    ),
 ]
 registry.add(messages)

readthedocs/settings/base.py

@@ -176,6 +176,7 @@ def RTD_RESTRICTED_DOMAINS(self):
     RTD_MAX_CONCURRENT_BUILDS = 4
     RTD_BUILDS_MAX_RETRIES = 25
     RTD_BUILDS_RETRY_DELAY = 5 * 60  # seconds
+    RTD_BUILDS_MAX_CONSECUTIVE_FAILURES = 50
     RTD_BUILD_STATUS_API_NAME = "docs/readthedocs"
     RTD_ANALYTICS_DEFAULT_RETENTION_DAYS = 30 * 3
     RTD_AUDITLOGS_DEFAULT_RETENTION_DAYS = 30 * 3