[release/6.x] Port curlm to 6.x (#7103, #7102, #7219, #7213) (#7293)

Co-authored-by: Eddy Ashton <ashton.eddy@gmail.com>
Co-authored-by: Amaury Chamayou <amaury@xargs.fr>
Co-authored-by: Amaury Chamayou <amchamay@microsoft.com>
Co-authored-by: Eddy Ashton <edashton@microsoft.com>

Author: 3 people

CHANGELOG.md

@@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - Validate endorsement metadata (tcb version and chip id) against attestation (#7240)
+- Curl multi based fetching of quote endorsements and snapshots
+
+### Fixed
+
+- Fixed quote endorsements retry logic
 
 ### Removed
 

CMakeLists.txt

@@ -617,6 +617,8 @@ if(COMPILE_TARGET STREQUAL "snp")
            ccf_kv.snp
            nghttp2
            ${CMAKE_THREAD_LIBS_INIT}
+           curl
+           uv
            ccf_pal.snp
   )
 
@@ -670,6 +672,8 @@ elseif(COMPILE_TARGET STREQUAL "virtual")
            ccf_kv.host
            nghttp2
            ${CMAKE_THREAD_LIBS_INIT}
+           curl
+           uv
            ccf_pal.host
   )
 
@@ -1515,6 +1519,17 @@ if(BUILD_TESTS)
       NAME historical_query_cache_test
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_cache.py
     )
+
+    add_test_bin(
+      curl_test ${CMAKE_CURRENT_SOURCE_DIR}/src/http/test/curl_test.cpp
+    )
+    target_link_libraries(curl_test PRIVATE curl uv http_parser.host)
+
+    if(LONG_TESTS)
+      add_e2e_test(
+        NAME e2e_curl PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_curl.py
+      )
+    endif()
   endif()
 endif()
 

cmake/common.cmake

@@ -45,7 +45,9 @@ endfunction()
 function(add_test_bin name)
   add_executable(${name} ${CCF_DIR}/src/enclave/thread_local.cpp ${ARGN})
   target_compile_options(${name} PRIVATE ${COMPILE_LIBCXX})
-  target_include_directories(${name} PRIVATE src ${CCFCRYPTO_INC})
+  target_include_directories(
+    ${name} PRIVATE src ${CCFCRYPTO_INC} ${CCF_DIR}/3rdparty/test
+  )
   enable_coverage(${name})
   target_link_libraries(${name} PRIVATE ${LINK_LIBCXX} ccfcrypto.host)
   add_san(${name})

include/ccf/pal/attestation_sev_snp.h

@@ -4,6 +4,7 @@
 
 #include "ccf/ds/enum_formatter.h"
 #include "ccf/ds/json.h"
+#include "ccf/ds/unit_strings.h"
 #include "ccf/pal/attestation_sev_snp_endorsements.h"
 #include "ccf/pal/measurement.h"
 #include "ccf/pal/report_data.h"
@@ -459,6 +460,8 @@ pRb21iI1NlNCfOGUPIhVpWECAwEAAQ==
     auto reported_tcb = fmt::format("{:0x}", *(uint64_t*)(&quote.reported_tcb));
 
     constexpr size_t default_max_retries_count = 10;
+    static const ds::SizeString default_max_client_response_size =
+      ds::SizeString("100mb");
 
     if (endorsements_servers.empty())
     {
@@ -467,22 +470,30 @@ pRb21iI1NlNCfOGUPIhVpWECAwEAAQ==
         default_azure_endorsements_endpoint,
         chip_id_hex,
         reported_tcb,
-        default_max_retries_count));
+        default_max_retries_count,
+        default_max_client_response_size));
       return config;
     }
 
     for (auto const& server : endorsements_servers)
     {
       size_t max_retries_count =
         server.max_retries_count.value_or(default_max_retries_count);
+      size_t max_client_response_size =
+        server.max_client_response_size.value_or(
+          default_max_client_response_size);
       switch (server.type)
       {
         case EndorsementsEndpointType::Azure:
         {
           auto loc =
             get_endpoint_loc(server, default_azure_endorsements_endpoint);
           config.servers.emplace_back(make_azure_endorsements_server(
-            loc, chip_id_hex, reported_tcb, max_retries_count));
+            loc,
+            chip_id_hex,
+            reported_tcb,
+            max_retries_count,
+            max_client_response_size));
           break;
         }
         case EndorsementsEndpointType::AMD:
@@ -532,15 +543,20 @@ pRb21iI1NlNCfOGUPIhVpWECAwEAAQ==
             snp,
             microcode,
             product,
-            max_retries_count));
+            max_retries_count,
+            max_client_response_size));
           break;
         }
         case EndorsementsEndpointType::THIM:
         {
           auto loc =
             get_endpoint_loc(server, default_thim_endorsements_endpoint);
           config.servers.emplace_back(make_thim_endorsements_server(
-            loc, chip_id_hex, reported_tcb, max_retries_count));
+            loc,
+            chip_id_hex,
+            reported_tcb,
+            max_retries_count,
+            max_client_response_size));
           break;
         }
         default:

include/ccf/pal/attestation_sev_snp_endorsements.h

@@ -76,13 +76,14 @@ namespace ccf::pal::snp
     EndorsementsEndpointType type = Azure;
     std::optional<std::string> url = std::nullopt;
     std::optional<size_t> max_retries_count = std::nullopt;
+    std::optional<ccf::ds::SizeString> max_client_response_size = std::nullopt;
 
     bool operator==(const EndorsementsServer&) const = default;
   };
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(EndorsementsServer);
   DECLARE_JSON_REQUIRED_FIELDS(EndorsementsServer);
   DECLARE_JSON_OPTIONAL_FIELDS(
-    EndorsementsServer, type, url, max_retries_count);
+    EndorsementsServer, type, url, max_retries_count, max_client_response_size);
   using EndorsementsServers = std::vector<EndorsementsServer>;
 
   struct HostPort
@@ -99,17 +100,20 @@ namespace ccf::pal::snp
     const HostPort& endpoint,
     const std::string& chip_id_hex,
     const std::string& reported_tcb,
-    size_t max_retries_count)
+    size_t max_retries_count,
+    size_t max_client_response_size)
   {
     std::map<std::string, std::string> params;
     params["api-version"] = "2020-10-15-preview";
     EndorsementEndpointsConfiguration::EndpointInfo info{
       endpoint.host,
       endpoint.port,
       fmt::format("/SevSnpVM/certificates/{}/{}", chip_id_hex, reported_tcb),
-      params};
+      params,
+    };
 
     info.max_retries_count = max_retries_count;
+    info.max_client_response_size = max_client_response_size;
 
     return {info};
   }
@@ -127,7 +131,8 @@ namespace ccf::pal::snp
     const std::string& snp,
     const std::string& microcode,
     const ProductName& product_name,
-    size_t max_retries_count)
+    size_t max_retries_count,
+    size_t max_client_response_size)
   {
     std::map<std::string, std::string> params;
     params["blSPL"] = boot_loader;
@@ -144,12 +149,14 @@ namespace ccf::pal::snp
       true // DER
     };
     leaf.max_retries_count = max_retries_count;
+    leaf.max_client_response_size = max_client_response_size;
     EndorsementEndpointsConfiguration::EndpointInfo chain{
       endpoint.host,
       endpoint.port,
       fmt::format("/vcek/v1/{}/cert_chain", to_string(product_name)),
       {}};
     chain.max_retries_count = max_retries_count;
+    chain.max_client_response_size = max_client_response_size;
 
     server.push_back(leaf);
     server.push_back(chain);
@@ -164,7 +171,8 @@ namespace ccf::pal::snp
     const HostPort& endpoint,
     const std::string& chip_id_hex,
     const std::string& reported_tcb,
-    size_t max_retries_count)
+    size_t max_retries_count,
+    size_t max_client_response_size)
   {
     std::map<std::string, std::string> params;
     params["tcbVersion"] = reported_tcb;
@@ -180,6 +188,7 @@ namespace ccf::pal::snp
       false // No TLS
     };
     info.max_retries_count = max_retries_count;
+    info.max_client_response_size = max_client_response_size;
 
     return {info};
   }

include/ccf/service/tables/jsengine.h

@@ -14,7 +14,7 @@ namespace ccf
     {
       static constexpr size_t max_heap_bytes = 100 * 1024 * 1024;
       static constexpr size_t max_stack_bytes = 1024 * 1024;
-      static constexpr uint64_t max_execution_time_ms = 1000;
+      static constexpr uint64_t max_execution_time_ms = 5000;
       static constexpr bool log_exception_details = false;
       static constexpr bool return_exception_details = false;
       static constexpr size_t max_cached_interpreters = 10;

src/common/configuration.h

@@ -16,6 +16,7 @@
 #include "common/enclave_interface_types.h"
 #include "consensus/consensus_types.h"
 #include "ds/oversized.h"
+#include "http/curl.h"
 #include "service/tables/config.h"
 
 #include <optional>
@@ -39,6 +40,8 @@ struct EnclaveConfig
   ringbuffer::Offsets* from_enclave_buffer_offsets;
 
   oversized::WriterConfig writer_config = {};
+
+  std::shared_ptr<ccf::curl::CurlmLibuvContext> curlm_libuv_context_instance;
 };
 
 static constexpr auto node_to_node_interface_name = "node_to_node_interface";

src/enclave/main.cpp

@@ -81,6 +81,14 @@ extern "C"
     auto ringbuffer_logger = new_logger.get();
     ccf::logger::config::loggers().push_back(std::move(new_logger));
 
+    if (ec.curlm_libuv_context_instance == nullptr)
+    {
+      LOG_FAIL_FMT("CurlmLibuvContext instance is null");
+      return CreateNodeStatus::EnclaveInitFailed;
+    }
+    ccf::curl::CurlmLibuvContextSingleton::set_instance(
+      ec.curlm_libuv_context_instance);
+
     {
       auto ccf_version_string = std::string(ccf::ccf_version);
       if (ccf_version_string.size() > enclave_version_size)

src/host/main.cpp

@@ -537,6 +537,11 @@ int main(int argc, char** argv) // NOLINT(bugprone-exception-escape)
     rpc_udp->behaviour.register_udp_message_handlers(
       buffer_processor.get_dispatcher());
 
+    // Initialise the curlm singleton
+    curl_global_init(CURL_GLOBAL_DEFAULT);
+    auto curl_libuv_context =
+      ccf::curl::CurlmLibuvContextSingleton(uv_default_loop());
+
     ResolvedAddresses resolved_rpc_addresses;
     for (auto& [name, interface] : config.network.rpc_interfaces)
     {
@@ -602,6 +607,9 @@ int main(int argc, char** argv) // NOLINT(bugprone-exception-escape)
 
     enclave_config.writer_config = writer_config;
 
+    enclave_config.curlm_libuv_context_instance =
+      curl_libuv_context.get_context();
+
     ccf::StartupConfig startup_config(config);
 
     if (startup_config.attestation.snp_security_policy_file.has_value())
@@ -1010,25 +1018,38 @@ int main(int argc, char** argv) // NOLINT(bugprone-exception-escape)
 
   process_launcher.stop();
 
-  // Continue running the loop long enough for the on_close
-  // callbacks to be despatched, so as to avoid memory being
-  // leaked by handles. Capped out of abundance of caution.
-  constexpr size_t max_iterations = 1000;
-  size_t close_iterations = max_iterations;
-  while ((uv_loop_alive(uv_default_loop()) != 0) && (close_iterations > 0))
+  constexpr size_t max_close_iterations = 1000;
+  size_t close_iterations = max_close_iterations;
+  int loop_close_rc = 0;
+  while (close_iterations > 0)
   {
+    loop_close_rc = uv_loop_close(uv_default_loop());
+    if (loop_close_rc != UV_EBUSY)
+    {
+      break;
+    }
     uv_run(uv_default_loop(), UV_RUN_NOWAIT);
-    close_iterations--;
+    --close_iterations;
+    std::this_thread::sleep_for(10ms);
   }
   LOG_INFO_FMT(
-    "Ran an extra {} cleanup iteration(s)", max_iterations - close_iterations);
-
-  auto loop_close_rc = uv_loop_close(uv_default_loop());
+    "Ran an extra {} cleanup iteration(s)",
+    max_close_iterations - close_iterations);
   if (loop_close_rc != 0)
   {
     LOG_FAIL_FMT(
       "Failed to close uv loop cleanly: {}", uv_err_name(loop_close_rc));
+    // walk loop to diagnose unclosed handles
+    auto cb = [](uv_handle_t* handle, void* arg) {
+      (void)arg;
+      LOG_FAIL_FMT(
+        "Leaked handle: type={}, ptr={}",
+        uv_handle_type_name(handle->type),
+        fmt::ptr(handle));
+    };
+    uv_walk(uv_default_loop(), cb, nullptr);
   }
+  curl_global_cleanup();
   ccf::crypto::openssl_sha256_shutdown();
 
   return loop_close_rc;

src/host/proxy.h

@@ -25,6 +25,7 @@ namespace asynchost
     {
       raw = new T(std::forward<Args>(args)...);
     }
+    close_ptr(T* that) : raw(that) {}
 
     ~close_ptr()
     {
@@ -56,6 +57,7 @@ namespace asynchost
     proxy_ptr(const proxy_ptr<T>& that) : internal(that.internal) {}
     proxy_ptr(proxy_ptr<T>&& that) : internal(std::move(that.internal)) {}
     proxy_ptr(std::nullptr_t that) : internal(that) {}
+    proxy_ptr(T* that) : internal(std::make_shared<close_ptr<T>>(that)) {}
 
     template <typename... Args>
     proxy_ptr(Args&&... args) :