[HealthChecks] Add AzureWebJobsStorage health check (#11471)

* Add AzureWebJobsStorage health check

* Address copilot review

* Add comments to HealthCheckData

* Fix unit tests

* Add Connectivity tag

* Update WebJobs tag

* Rename web_jobs -> webjobs

Author: jviau

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckData.cs

@@ -0,0 +1,142 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using Azure;
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
+{
+    /// <summary>
+    /// A helper for providing data with a health check result.
+    /// </summary>
+    internal partial class HealthCheckData
+    {
+        // exposed to the HealthCheckResult through IReadOnlyDictionary.
+        private readonly Dictionary<string, object> _data = [];
+
+        /// <summary>
+        /// Gets or sets the area of the health check data failure.
+        /// </summary>
+        /// <remarks>
+        /// This is the area that has failed. Such as "configuration", "connectivity", etc.
+        /// </remarks>
+        public string Area
+        {
+            get => GetOrDefault<string>();
+            set => Set(value);
+        }
+
+        /// <summary>
+        /// Gets or sets the configuration section related to the health check data.
+        /// </summary>
+        /// <remarks>
+        /// Useful for when the component being checked is related to a specific configuration section.
+        /// </remarks>
+        public string ConfigurationSection
+        {
+            get => GetOrDefault<string>();
+            set => Set(value);
+        }
+
+        /// <summary>
+        /// Gets or sets the status code related to the health check data.
+        /// For HTTP related related checks, this is the HTTP status code.
+        /// </summary>
+        public int StatusCode
+        {
+            get => GetOrDefault<int>();
+            set => Set(value);
+        }
+
+        /// <summary>
+        /// Gets or sets the error code related to the health check data.
+        /// </summary>
+        /// <remarks>
+        /// For Azure SDK related checks, this is typically the RequestFailedException.ErrorCode value.
+        /// </remarks>
+        public string ErrorCode
+        {
+            get => GetOrDefault<string>();
+            set => Set(value);
+        }
+
+        /// <summary>
+        /// Sets exception details into the health check data.
+        /// </summary>
+        /// <param name="ex">The exception to set details from.</param>
+        /// <remarks>
+        /// This will set various properties based on the type of exception.
+        /// </remarks>
+        public void SetExceptionDetails(Exception ex)
+        {
+            ArgumentNullException.ThrowIfNull(ex);
+            if (ex is AggregateException aggregate)
+            {
+                // Azure SDK will retry a few times in some cases, leading to multiple inner exceptions.
+                // We only care about the last one.
+                ex = aggregate.InnerExceptions.Last();
+            }
+
+            if (ex is TimeoutException)
+            {
+                ErrorCode = "Timeout";
+            }
+            else if (ex is OperationCanceledException)
+            {
+                ErrorCode = "OperationCanceled";
+            }
+            else if (ex is RequestFailedException rfe)
+            {
+                StatusCode = rfe.Status;
+                ErrorCode = rfe.ErrorCode;
+            }
+        }
+
+        private void Set<T>(T value, [CallerMemberName] string key = null)
+        {
+            _data[key] = value;
+        }
+
+        private T GetOrDefault<T>([CallerMemberName] string key = null, T defaultValue = default)
+        {
+            if (_data.TryGetValue(key, out var value) && value is T typedValue)
+            {
+                return typedValue;
+            }
+
+            return defaultValue;
+        }
+    }
+
+    // Partial class down here to separate IReadOnlyDictionary implementation details.
+    internal partial class HealthCheckData : IReadOnlyDictionary<string, object>
+    {
+        IEnumerable<string> IReadOnlyDictionary<string, object>.Keys
+            => _data.Keys;
+
+        IEnumerable<object> IReadOnlyDictionary<string, object>.Values
+            => _data.Values;
+
+        int IReadOnlyCollection<KeyValuePair<string, object>>.Count
+            => _data.Count;
+
+        object IReadOnlyDictionary<string, object>.this[string key]
+            => _data[key];
+
+        bool IReadOnlyDictionary<string, object>.ContainsKey(string key)
+            => _data.ContainsKey(key);
+
+        IEnumerator<KeyValuePair<string, object>> IEnumerable<KeyValuePair<string, object>>.GetEnumerator()
+            => _data.GetEnumerator();
+
+        IEnumerator IEnumerable.GetEnumerator()
+            => _data.GetEnumerator();
+
+        bool IReadOnlyDictionary<string, object>.TryGetValue(string key, out object value)
+            => _data.TryGetValue(key, out value);
+    }
+}

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs

@@ -1,10 +1,11 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.Extensions.Logging;
 
@@ -27,6 +28,7 @@ public static IHealthChecksBuilder AddWebJobsScriptHealthChecks(this IHealthChec
             builder
                 .AddWebHostHealthCheck()
                 .AddScriptHostHealthCheck()
+                .AddWebJobsStorageHealthCheck()
                 .AddTelemetryPublisher(HealthCheckTags.Liveness, HealthCheckTags.Readiness)
                 .UseDynamicHealthCheckService();
             return builder;
@@ -126,6 +128,24 @@ public static IHealthChecksBuilder AddScriptHostHealthCheck(this IHealthChecksBu
             return builder;
         }
 
+        /// <summary>
+        /// Adds a health check for the WebJobs storage account.
+        /// </summary>
+        /// <param name="builder">The builder to register health checks with.</param>
+        /// <returns>The original builder, for call chaining.</returns>
+        public static IHealthChecksBuilder AddWebJobsStorageHealthCheck(this IHealthChecksBuilder builder)
+        {
+            ArgumentNullException.ThrowIfNull(builder);
+
+            // Ensure singleton as this health check refreshes in the background.
+            builder.Services.TryAddSingleton<WebJobsStorageHealthCheck>();
+            builder.AddCheck<WebJobsStorageHealthCheck>(
+                HealthCheckNames.WebJobsStorage,
+                tags: [HealthCheckTags.Configuration, HealthCheckTags.Connectivity, HealthCheckTags.WebJobsStorage],
+                timeout: TimeSpan.FromSeconds(10));
+            return builder;
+        }
+
         /// <summary>
         /// Filters a health report to include only specified entries.
         /// </summary>

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckNames.cs

@@ -19,5 +19,10 @@ internal static class HealthCheckNames
         /// The 'azure.functions.script_host.lifecycle' check monitors the lifecycle of the script host.
         /// </summary>
         public const string ScriptHostLifeCycle = Prefix + "script_host.lifecycle";
+
+        /// <summary>
+        /// The 'azure.functions.webjobs.storage' check monitors connectivity to the WebJobs storage account.
+        /// </summary>
+        public const string WebJobsStorage = Prefix + "webjobs.storage";
     }
 }

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
@@ -8,30 +8,44 @@ namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
     /// </summary>
     internal static class HealthCheckTags
     {
-        private const string Prefix = "azure.functions.";
+        private const string FuncPrefix = "azure.functions.";
+        private const string WebJobsPrefix = FuncPrefix + "webjobs.";
 
         /// <summary>
         /// The 'azure.functions.liveness' tag is used for liveness checks in the Azure Functions host.
         /// </summary>
         /// <remarks>
         /// Liveness checks are used to determine if the host is alive and responsive.
         /// </remarks>
-        public const string Liveness = Prefix + "liveness";
+        public const string Liveness = FuncPrefix + "liveness";
 
         /// <summary>
         /// The 'azure.functions.readiness' tag is used for readiness checks in the Azure Functions host.
         /// </summary>
         /// <remarks>
         /// Readiness checks are used to determine if the host is ready to process requests.
         /// </remarks>
-        public const string Readiness = Prefix + "readiness";
+        public const string Readiness = FuncPrefix + "readiness";
 
         /// <summary>
         /// The 'azure.functions.configuration' tag is used for configuration-related health checks in the Azure Functions host.
         /// </summary>
         /// <remarks>
         /// These are typically customer configuration related, such as configuring AzureWebJobsStorage access.
         /// </remarks>
-        public const string Configuration = Prefix + "configuration";
+        public const string Configuration = FuncPrefix + "configuration";
+
+        /// <summary>
+        /// The 'azure.functions.connectivity' tag is used for connectivity-related health checks in the Azure Functions host.
+        /// </summary>
+        /// <remarks>
+        /// These are typically related to connectivity to external services, such as Azure Storage.
+        /// </remarks>
+        public const string Connectivity = FuncPrefix + "connectivity";
+
+        /// <summary>
+        /// The "azure.functions.webjobs.storage" tag is used for health checks related to the WebJobs storage account.
+        /// </summary>
+        public const string WebJobsStorage = WebJobsPrefix + "storage";
     }
 }