From 657ab3760228ec2b16f1e23b4c7664be123077c1 Mon Sep 17 00:00:00 2001 From: hhvrc Date: Thu, 14 Nov 2024 15:08:35 +0100 Subject: [PATCH 01/28] Initial commit --- include/http/HTTPClient.h | 89 +++++++++++++++++++++++++++++++ include/http/HTTPRequestManager.h | 7 +-- include/http/HTTPResponse.h | 53 ++++++++++++++++++ include/util/PartitionUtils.h | 2 +- src/OtaUpdateManager.cpp | 22 ++++---- src/http/HTTPRequestManager.cpp | 24 +++------ src/util/ParitionUtils.cpp | 8 +-- 7 files changed, 171 insertions(+), 34 deletions(-) create mode 100644 include/http/HTTPClient.h create mode 100644 include/http/HTTPResponse.h diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h new file mode 100644 index 00000000..b87c7714 --- /dev/null +++ b/include/http/HTTPClient.h @@ -0,0 +1,89 @@ +#pragma once + +#include "Common.h" +#include "http/HTTPResponse.h" + +#include +#include + +#include +#include + +namespace OpenShock::HTTP { + class HTTPClient { + public: + HTTPClient(const char* url, int timeout_ms = 10'000) + { + esp_http_client_config_t cfg = { + .url = url, + .user_agent = OpenShock::Constants::FW_USERAGENT, + .timeout_ms = timeout_ms, + .disable_auto_redirect = true, + .is_async = true, + .use_global_ca_store = true, + }; + handle = esp_http_client_init(&cfg); + } + ~HTTPClient() + { + if (handle != nullptr) { + esp_http_client_cleanup(handle); + } + } + HTTPClient(const HTTPClient&) = delete; + HTTPClient(HTTPClient&& other) + { + handle = other.handle; + other.handle = nullptr; + } + + constexpr bool IsClosed() const { return handle == nullptr; } + + bool SetHeader(const char* key, const char* value) + { + if (handle == nullptr) return false; + + return esp_http_client_set_header(handle, key, value) == ESP_OK; + } + bool RemoveHeader(const char* key) + { + if (handle == nullptr) return false; + + return esp_http_client_delete_header(handle, key) == ESP_OK; + } + + HTTPResponse Send(const char* url, esp_http_client_method_t method, int contentLength = 0) + { + esp_err_t err = ESP_FAIL; + if (handle == nullptr) { + return HTTPResponse(err); + } + + err = esp_http_client_set_method(handle, method); + if (err != ESP_OK) return HTTPResponse(err); + + err = esp_http_client_set_url(handle, url); + if (err != ESP_OK) return HTTPResponse(err); + + err = esp_http_client_open(handle, contentLength); + if (err != ESP_OK) return HTTPResponse(err); + + std::int64_t respContentLength = esp_http_client_fetch_headers(handle); + if (respContentLength < 0) return HTTPResponse(ESP_FAIL); + + bool isChunked = false; + if (respContentLength == 0) { + isChunked = esp_http_client_is_chunked_response(handle); + } + + return HTTPResponse(handle, respContentLength, isChunked); + } + + HTTPResponse Get(const char* url) { return Send(url, HTTP_METHOD_GET); } + + HTTPClient& operator=(const HTTPClient&) = delete; + + private: + esp_http_client_handle_t handle; + }; +} // namespace OpenShock::HTTP diff --git a/include/http/HTTPRequestManager.h b/include/http/HTTPRequestManager.h index 26402f0d..3823706b 100644 --- a/include/http/HTTPRequestManager.h +++ b/include/http/HTTPRequestManager.h @@ -34,11 +34,12 @@ namespace OpenShock::HTTP { using GotContentLengthCallback = std::function; using DownloadCallback = std::function; - Response Download(std::string_view url, const std::map& headers, GotContentLengthCallback contentLengthCallback, DownloadCallback downloadCallback, const std::vector& acceptedCodes = {200}, uint32_t timeoutMs = 10'000); - Response GetString(std::string_view url, const std::map& headers, const std::vector& acceptedCodes = {200}, uint32_t timeoutMs = 10'000); + Response Download(const char* url, const std::map& headers, GotContentLengthCallback contentLengthCallback, DownloadCallback downloadCallback, const std::vector& acceptedCodes = {200}, int timeoutMs = 10'000); + Response GetString(const char* url, const std::map& headers, const std::vector& acceptedCodes = {200}, int timeoutMs = 10'000); template - Response GetJSON(std::string_view url, const std::map& headers, JsonParser jsonParser, const std::vector& acceptedCodes = {200}, uint32_t timeoutMs = 10'000) { + Response GetJSON(const char* url, const std::map& headers, JsonParser jsonParser, const std::vector& acceptedCodes = {200}, int timeoutMs = 10'000) + { auto response = GetString(url, headers, acceptedCodes, timeoutMs); if (response.result != RequestResult::Success) { return {response.result, response.code, {}}; diff --git a/include/http/HTTPResponse.h b/include/http/HTTPResponse.h new file mode 100644 index 00000000..d6a91c28 --- /dev/null +++ b/include/http/HTTPResponse.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include + +#include +#include + +namespace OpenShock::HTTP { + class HTTPClient; + class HTTPResponse { + friend class HTTPClient; + + HTTPResponse(esp_err_t error) + : m_handle(nullptr) + , m_error(error) + { + } + HTTPResponse(esp_http_client_handle_t handle, std::int64_t contentLength, bool isChunked) + : m_handle(handle) + , m_contentLength(contentLength) + , m_isChunked(isChunked) + { + if (m_handle == nullptr) { + m_error = ESP_FAIL; + return; + } + } + + public: + HTTPResponse(const HTTPResponse&) = delete; + HTTPResponse(HTTPResponse&& other) + { + m_handle = other.m_handle; + other.m_handle = nullptr; + } + + constexpr bool IsValid() const { return m_handle != nullptr; } + constexpr esp_err_t GetError() const { return m_handle == nullptr ? m_error : ESP_OK; } + + HTTPResponse& operator=(const HTTPResponse&) = delete; + + private: + esp_http_client_handle_t m_handle; + union { + struct { // Only valid if m_handle is not nullptr. + std::int64_t m_contentLength; + bool m_isChunked; + }; + esp_err_t m_error; // If m_handle is nullptr, this is the error code. + }; + }; +} // namespace OpenShock::HTTP diff --git a/include/util/PartitionUtils.h b/include/util/PartitionUtils.h index 22669020..0745894c 100644 --- a/include/util/PartitionUtils.h +++ b/include/util/PartitionUtils.h @@ -8,5 +8,5 @@ namespace OpenShock { bool TryGetPartitionHash(const esp_partition_t* partition, char (&hash)[65]); - bool FlashPartitionFromUrl(const esp_partition_t* partition, std::string_view remoteUrl, const uint8_t (&remoteHash)[32], std::function progressCallback = nullptr); + bool FlashPartitionFromUrl(const esp_partition_t* partition, const char* remoteUrl, const uint8_t (&remoteHash)[32], std::function progressCallback = nullptr); } diff --git a/src/OtaUpdateManager.cpp b/src/OtaUpdateManager.cpp index 2e8b5680..5e3de77e 100644 --- a/src/OtaUpdateManager.cpp +++ b/src/OtaUpdateManager.cpp @@ -159,7 +159,7 @@ static bool _sendFailureMessage(std::string_view message, bool fatal = false) return true; } -static bool _flashAppPartition(const esp_partition_t* partition, std::string_view remoteUrl, const uint8_t (&remoteHash)[32]) +static bool _flashAppPartition(const esp_partition_t* partition, const char* remoteUrl, const uint8_t (&remoteHash)[32]) { OS_LOGD(TAG, "Flashing app partition"); @@ -195,7 +195,7 @@ static bool _flashAppPartition(const esp_partition_t* partition, std::string_vie return true; } -static bool _flashFilesystemPartition(const esp_partition_t* parition, std::string_view remoteUrl, const uint8_t (&remoteHash)[32]) +static bool _flashFilesystemPartition(const esp_partition_t* parition, const char* remoteUrl, const uint8_t (&remoteHash)[32]) { if (!_sendProgressMessage(Serialization::Types::OtaUpdateProgressTask::PreparingForUpdate, 0.0f)) { return false; @@ -396,8 +396,8 @@ static void _otaUpdateTask(void* arg) esp_task_wdt_init(15, true); // Flash app and filesystem partitions. - if (!_flashFilesystemPartition(filesystemPartition, release.filesystemBinaryUrl, release.filesystemBinaryHash)) continue; - if (!_flashAppPartition(appPartition, release.appBinaryUrl, release.appBinaryHash)) continue; + if (!_flashFilesystemPartition(filesystemPartition, release.filesystemBinaryUrl.c_str(), release.filesystemBinaryHash)) continue; + if (!_flashAppPartition(appPartition, release.appBinaryUrl.c_str(), release.appBinaryHash)) continue; // Set OTA boot type in config. if (!Config::SetOtaUpdateStep(OpenShock::OtaUpdateStep::Updated)) { @@ -422,7 +422,7 @@ static void _otaUpdateTask(void* arg) esp_restart(); } -static bool _tryGetStringList(std::string_view url, std::vector& list) +static bool _tryGetStringList(const char* url, std::vector& list) { auto response = OpenShock::HTTP::GetString( url, @@ -525,16 +525,16 @@ bool OtaUpdateManager::Init() bool OtaUpdateManager::TryGetFirmwareVersion(OtaUpdateChannel channel, OpenShock::SemVer& version) { - std::string_view channelIndexUrl; + const char* channelIndexUrl; switch (channel) { case OtaUpdateChannel::Stable: - channelIndexUrl = OPENSHOCK_FW_CDN_STABLE_URL ""sv; + channelIndexUrl = OPENSHOCK_FW_CDN_STABLE_URL; break; case OtaUpdateChannel::Beta: - channelIndexUrl = OPENSHOCK_FW_CDN_BETA_URL ""sv; + channelIndexUrl = OPENSHOCK_FW_CDN_BETA_URL; break; case OtaUpdateChannel::Develop: - channelIndexUrl = OPENSHOCK_FW_CDN_DEVELOP_URL ""sv; + channelIndexUrl = OPENSHOCK_FW_CDN_DEVELOP_URL; break; default: OS_LOGE(TAG, "Unknown channel: %u", channel); @@ -573,7 +573,7 @@ bool OtaUpdateManager::TryGetFirmwareBoards(const OpenShock::SemVer& version, st OS_LOGD(TAG, "Fetching firmware boards from %s", channelIndexUrl.c_str()); - if (!_tryGetStringList(channelIndexUrl, boards)) { + if (!_tryGetStringList(channelIndexUrl.c_str(), boards)) { OS_LOGE(TAG, "Failed to fetch firmware boards"); return false; } @@ -614,7 +614,7 @@ bool OtaUpdateManager::TryGetFirmwareRelease(const OpenShock::SemVer& version, F // Fetch hashes. auto sha256HashesResponse = OpenShock::HTTP::GetString( - sha256HashesUrl, + sha256HashesUrl.c_str(), { {"Accept", "text/plain"} }, diff --git a/src/http/HTTPRequestManager.cpp b/src/http/HTTPRequestManager.cpp index 9ed9d704..4c5f03e1 100644 --- a/src/http/HTTPRequestManager.cpp +++ b/src/http/HTTPRequestManager.cpp @@ -3,13 +3,12 @@ const char* const TAG = "HTTPRequestManager"; #include "Common.h" +#include "http/HTTPClient.h" #include "Logging.h" #include "SimpleMutex.h" #include "Time.h" #include "util/StringUtils.h" -#include - #include #include #include @@ -193,11 +192,6 @@ std::shared_ptr _getRateLimiter(std::string_view url) return it->second; } -void _setupClient(HTTPClient& client) -{ - client.setUserAgent(OpenShock::Constants::FW_USERAGENT); -} - struct StreamReaderResult { HTTP::RequestResult result; std::size_t nWritten; @@ -338,7 +332,7 @@ void _alignChunk(uint8_t* buffer, std::size_t& bufferCursor, std::size_t payload } } -StreamReaderResult _readStreamDataChunked(HTTPClient& client, WiFiClient* stream, HTTP::DownloadCallback downloadCallback, int64_t begin, uint32_t timeoutMs) +StreamReaderResult _readStreamDataChunked(HTTPClient& client, WiFiClient* stream, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) { std::size_t totalWritten = 0; HTTP::RequestResult result = HTTP::RequestResult::Success; @@ -419,7 +413,7 @@ StreamReaderResult _readStreamDataChunked(HTTPClient& client, WiFiClient* stream return {result, totalWritten}; } -StreamReaderResult _readStreamData(HTTPClient& client, WiFiClient* stream, std::size_t contentLength, HTTP::DownloadCallback downloadCallback, int64_t begin, uint32_t timeoutMs) +StreamReaderResult _readStreamData(HTTPClient& client, WiFiClient* stream, std::size_t contentLength, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) { std::size_t nWritten = 0; HTTP::RequestResult result = HTTP::RequestResult::Success; @@ -466,13 +460,13 @@ StreamReaderResult _readStreamData(HTTPClient& client, WiFiClient* stream, std:: HTTP::Response _doGetStream( HTTPClient& client, - std::string_view url, + const char* url, const std::map& headers, const std::vector& acceptedCodes, std::shared_ptr rateLimiter, HTTP::GotContentLengthCallback contentLengthCallback, HTTP::DownloadCallback downloadCallback, - uint32_t timeoutMs + int timeoutMs ) { int64_t begin = OpenShock::millis(); @@ -560,8 +554,7 @@ HTTP::Response _doGetStream( return {result.result, responseCode, result.nWritten}; } -HTTP::Response - HTTP::Download(std::string_view url, const std::map& headers, HTTP::GotContentLengthCallback contentLengthCallback, HTTP::DownloadCallback downloadCallback, const std::vector& acceptedCodes, uint32_t timeoutMs) +HTTP::Response HTTP::Download(const char* url, const std::map& headers, HTTP::GotContentLengthCallback contentLengthCallback, HTTP::DownloadCallback downloadCallback, const std::vector& acceptedCodes, int timeoutMs) { std::shared_ptr rateLimiter = _getRateLimiter(url); if (rateLimiter == nullptr) { @@ -572,13 +565,12 @@ HTTP::Response return {RequestResult::RateLimited, 0, 0}; } - HTTPClient client; - _setupClient(client); + HTTP::HTTPClient client(url); return _doGetStream(client, url, headers, acceptedCodes, rateLimiter, contentLengthCallback, downloadCallback, timeoutMs); } -HTTP::Response HTTP::GetString(std::string_view url, const std::map& headers, const std::vector& acceptedCodes, uint32_t timeoutMs) +HTTP::Response HTTP::GetString(const char* url, const std::map& headers, const std::vector& acceptedCodes, int timeoutMs) { std::string result; diff --git a/src/util/ParitionUtils.cpp b/src/util/ParitionUtils.cpp index 3764879a..c1997252 100644 --- a/src/util/ParitionUtils.cpp +++ b/src/util/ParitionUtils.cpp @@ -8,7 +8,8 @@ const char* const TAG = "PartitionUtils"; #include "Time.h" #include "util/HexUtils.h" -bool OpenShock::TryGetPartitionHash(const esp_partition_t* partition, char (&hash)[65]) { +bool OpenShock::TryGetPartitionHash(const esp_partition_t* partition, char (&hash)[65]) +{ uint8_t buffer[32]; esp_err_t err = esp_partition_get_sha256(partition, buffer); if (err != ESP_OK) { @@ -22,7 +23,8 @@ bool OpenShock::TryGetPartitionHash(const esp_partition_t* partition, char (&has return true; } -bool OpenShock::FlashPartitionFromUrl(const esp_partition_t* partition, std::string_view remoteUrl, const uint8_t (&remoteHash)[32], std::function progressCallback) { +bool OpenShock::FlashPartitionFromUrl(const esp_partition_t* partition, const char* remoteUrl, const uint8_t (&remoteHash)[32], std::function progressCallback) +{ OpenShock::SHA256 sha256; if (!sha256.begin()) { OS_LOGE(TAG, "Failed to initialize SHA256 hash"); @@ -31,7 +33,7 @@ bool OpenShock::FlashPartitionFromUrl(const esp_partition_t* partition, std::str std::size_t contentLength = 0; std::size_t contentWritten = 0; - int64_t lastProgress = 0; + int64_t lastProgress = 0; auto sizeValidator = [partition, &contentLength, progressCallback, &lastProgress](std::size_t size) -> bool { if (size > partition->size) { From 8ceac767a402cb667569ade6aad93416736684dc Mon Sep 17 00:00:00 2001 From: hhvrc Date: Tue, 4 Feb 2025 11:52:25 +0100 Subject: [PATCH 02/28] Use helpers --- include/Common.h | 1 + include/http/HTTPClient.h | 6 +++--- include/http/HTTPResponse.h | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/Common.h b/include/Common.h index 665acee7..e850d89a 100644 --- a/include/Common.h +++ b/include/Common.h @@ -3,6 +3,7 @@ #include #include +#define DISABLE_DEFAULT(TypeName) TypeName() = delete #define DISABLE_COPY(TypeName) \ TypeName(const TypeName&) = delete; \ TypeName& operator=(const TypeName&) = delete diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h index b87c7714..8c0c1c0a 100644 --- a/include/http/HTTPClient.h +++ b/include/http/HTTPClient.h @@ -11,6 +11,9 @@ namespace OpenShock::HTTP { class HTTPClient { + DISABLE_DEFAULT(HTTPClient); + DISABLE_COPY(HTTPClient); + public: HTTPClient(const char* url, int timeout_ms = 10'000) { @@ -30,7 +33,6 @@ namespace OpenShock::HTTP { esp_http_client_cleanup(handle); } } - HTTPClient(const HTTPClient&) = delete; HTTPClient(HTTPClient&& other) { handle = other.handle; @@ -81,8 +83,6 @@ namespace OpenShock::HTTP { HTTPResponse Get(const char* url) { return Send(url, HTTP_METHOD_GET); } - HTTPClient& operator=(const HTTPClient&) = delete; - private: esp_http_client_handle_t handle; }; diff --git a/include/http/HTTPResponse.h b/include/http/HTTPResponse.h index d6a91c28..5436c69a 100644 --- a/include/http/HTTPResponse.h +++ b/include/http/HTTPResponse.h @@ -9,6 +9,9 @@ namespace OpenShock::HTTP { class HTTPClient; class HTTPResponse { + DISABLE_DEFAULT(HTTPResponse); + DISABLE_COPY(HTTPResponse); + friend class HTTPClient; HTTPResponse(esp_err_t error) @@ -28,7 +31,6 @@ namespace OpenShock::HTTP { } public: - HTTPResponse(const HTTPResponse&) = delete; HTTPResponse(HTTPResponse&& other) { m_handle = other.m_handle; @@ -38,8 +40,6 @@ namespace OpenShock::HTTP { constexpr bool IsValid() const { return m_handle != nullptr; } constexpr esp_err_t GetError() const { return m_handle == nullptr ? m_error : ESP_OK; } - HTTPResponse& operator=(const HTTPResponse&) = delete; - private: esp_http_client_handle_t m_handle; union { From 9dcc1629c64dbe58057ce48bae8de9af8c6033f8 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 2 Dec 2025 14:57:20 +0100 Subject: [PATCH 03/28] Some more stuff --- include/http/HTTPClient.h | 14 ++++++++++++++ include/http/HTTPResponse.h | 2 ++ src/http/HTTPRequestManager.cpp | 22 +++++++++++++--------- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h index 8c0c1c0a..886f6882 100644 --- a/include/http/HTTPClient.h +++ b/include/http/HTTPClient.h @@ -22,6 +22,8 @@ namespace OpenShock::HTTP { .user_agent = OpenShock::Constants::FW_USERAGENT, .timeout_ms = timeout_ms, .disable_auto_redirect = true, + .event_handler = HTTPClient::EventHandler, + .user_data = reinterpret_cast(this), .is_async = true, .use_global_ca_store = true, }; @@ -84,6 +86,18 @@ namespace OpenShock::HTTP { HTTPResponse Get(const char* url) { return Send(url, HTTP_METHOD_GET); } private: + static esp_err_t EventHandler(esp_http_client_event_t* evt) + { + HTTPClient* client = reinterpret_cast(evt->user_data); + + if (evt->event_id == HTTP_EVENT_ON_HEADER) { + // evt->header_key; + // evt->header_value; + } + + return ESP_OK; + } + esp_http_client_handle_t handle; }; } // namespace OpenShock::HTTP diff --git a/include/http/HTTPResponse.h b/include/http/HTTPResponse.h index 5436c69a..0199eebf 100644 --- a/include/http/HTTPResponse.h +++ b/include/http/HTTPResponse.h @@ -40,6 +40,8 @@ namespace OpenShock::HTTP { constexpr bool IsValid() const { return m_handle != nullptr; } constexpr esp_err_t GetError() const { return m_handle == nullptr ? m_error : ESP_OK; } + int ResponseCode() const { return esp_http_client_get_status_code(m_handle); } + private: esp_http_client_handle_t m_handle; union { diff --git a/src/http/HTTPRequestManager.cpp b/src/http/HTTPRequestManager.cpp index e4baa368..0a389e29 100644 --- a/src/http/HTTPRequestManager.cpp +++ b/src/http/HTTPRequestManager.cpp @@ -11,6 +11,8 @@ const char* const TAG = "HTTPRequestManager"; #include "util/HexUtils.h" #include "util/StringUtils.h" +#include "WiFiClient.h" + #include #include #include @@ -219,7 +221,7 @@ void _alignChunk(uint8_t* buffer, std::size_t& bufferCursor, std::size_t payload } } -StreamReaderResult _readStreamDataChunked(HTTPClient& client, WiFiClient* stream, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) +StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient& client, WiFiClient* stream, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) { std::size_t totalWritten = 0; HTTP::RequestResult result = HTTP::RequestResult::Success; @@ -302,7 +304,7 @@ StreamReaderResult _readStreamDataChunked(HTTPClient& client, WiFiClient* stream return {result, totalWritten}; } -StreamReaderResult _readStreamData(HTTPClient& client, WiFiClient* stream, std::size_t contentLength, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) +StreamReaderResult _readStreamData(HTTP::HTTPClient& client, WiFiClient* stream, std::size_t contentLength, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) { std::size_t nWritten = 0; HTTP::RequestResult result = HTTP::RequestResult::Success; @@ -348,7 +350,7 @@ StreamReaderResult _readStreamData(HTTPClient& client, WiFiClient* stream, std:: } HTTP::Response _doGetStream( - HTTPClient& client, + HTTP::HTTPClient& client, const char* url, const std::map& headers, tcb::span acceptedCodes, @@ -359,17 +361,19 @@ HTTP::Response _doGetStream( ) { int64_t begin = OpenShock::millis(); - if (!client.begin(OpenShock::StringToArduinoString(url))) { - OS_LOGE(TAG, "Failed to begin HTTP request"); - return {HTTP::RequestResult::RequestFailed, 0, 0}; - } for (auto& header : headers) { - client.addHeader(header.first, header.second); + client.SetHeader(header.first, header.second); } - int responseCode = client.GET(); + auto response = client.Get(url); + if (!response.IsValid()) { + esp_err_t err = response.GetError(); + OS_LOGE(TAG, "Failed to begin HTTP request"); + return {HTTP::RequestResult::RequestFailed, 0, 0}; + } + auto responseCode = response.ResponseCode(); if (responseCode == HTTP_CODE_REQUEST_TIMEOUT || begin + timeoutMs < OpenShock::millis()) { OS_LOGW(TAG, "Request timed out"); return {HTTP::RequestResult::TimedOut, responseCode, 0}; From eb734c7f78b3830d741eb44cfa8620aa729ae648 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Wed, 3 Dec 2025 17:00:01 +0100 Subject: [PATCH 04/28] Push WIP --- include/http/HTTPClient.h | 103 ------------------------------ include/http/HTTPRequestManager.h | 8 +-- include/http/HTTPResponse.h | 55 ---------------- include/http/JsonAPI.h | 4 +- src/http/HTTPRequestManager.cpp | 48 ++++++++++---- src/http/JsonAPI.cpp | 16 ++--- 6 files changed, 48 insertions(+), 186 deletions(-) delete mode 100644 include/http/HTTPClient.h delete mode 100644 include/http/HTTPResponse.h diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h deleted file mode 100644 index 886f6882..00000000 --- a/include/http/HTTPClient.h +++ /dev/null @@ -1,103 +0,0 @@ -#pragma once - -#include "Common.h" -#include "http/HTTPResponse.h" - -#include -#include - -#include -#include - -namespace OpenShock::HTTP { - class HTTPClient { - DISABLE_DEFAULT(HTTPClient); - DISABLE_COPY(HTTPClient); - - public: - HTTPClient(const char* url, int timeout_ms = 10'000) - { - esp_http_client_config_t cfg = { - .url = url, - .user_agent = OpenShock::Constants::FW_USERAGENT, - .timeout_ms = timeout_ms, - .disable_auto_redirect = true, - .event_handler = HTTPClient::EventHandler, - .user_data = reinterpret_cast(this), - .is_async = true, - .use_global_ca_store = true, - }; - handle = esp_http_client_init(&cfg); - } - ~HTTPClient() - { - if (handle != nullptr) { - esp_http_client_cleanup(handle); - } - } - HTTPClient(HTTPClient&& other) - { - handle = other.handle; - other.handle = nullptr; - } - - constexpr bool IsClosed() const { return handle == nullptr; } - - bool SetHeader(const char* key, const char* value) - { - if (handle == nullptr) return false; - - return esp_http_client_set_header(handle, key, value) == ESP_OK; - } - bool RemoveHeader(const char* key) - { - if (handle == nullptr) return false; - - return esp_http_client_delete_header(handle, key) == ESP_OK; - } - - HTTPResponse Send(const char* url, esp_http_client_method_t method, int contentLength = 0) - { - esp_err_t err = ESP_FAIL; - if (handle == nullptr) { - return HTTPResponse(err); - } - - err = esp_http_client_set_method(handle, method); - if (err != ESP_OK) return HTTPResponse(err); - - err = esp_http_client_set_url(handle, url); - if (err != ESP_OK) return HTTPResponse(err); - - err = esp_http_client_open(handle, contentLength); - if (err != ESP_OK) return HTTPResponse(err); - - std::int64_t respContentLength = esp_http_client_fetch_headers(handle); - if (respContentLength < 0) return HTTPResponse(ESP_FAIL); - - bool isChunked = false; - if (respContentLength == 0) { - isChunked = esp_http_client_is_chunked_response(handle); - } - - return HTTPResponse(handle, respContentLength, isChunked); - } - - HTTPResponse Get(const char* url) { return Send(url, HTTP_METHOD_GET); } - - private: - static esp_err_t EventHandler(esp_http_client_event_t* evt) - { - HTTPClient* client = reinterpret_cast(evt->user_data); - - if (evt->event_id == HTTP_EVENT_ON_HEADER) { - // evt->header_key; - // evt->header_value; - } - - return ESP_OK; - } - - esp_http_client_handle_t handle; - }; -} // namespace OpenShock::HTTP diff --git a/include/http/HTTPRequestManager.h b/include/http/HTTPRequestManager.h index 200fa2f1..59621a37 100644 --- a/include/http/HTTPRequestManager.h +++ b/include/http/HTTPRequestManager.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include @@ -61,11 +59,11 @@ namespace OpenShock::HTTP { using GotContentLengthCallback = std::function; using DownloadCallback = std::function; - Response Download(const char* url, const std::map& headers, GotContentLengthCallback contentLengthCallback, DownloadCallback downloadCallback, tcb::span acceptedCodes, uint32_t timeoutMs = 10'000); - Response GetString(const char* url, const std::map& headers, tcb::span acceptedCodes, uint32_t timeoutMs = 10'000); + Response Download(const char* url, const std::map& headers, GotContentLengthCallback contentLengthCallback, DownloadCallback downloadCallback, tcb::span acceptedCodes, uint32_t timeoutMs = 10'000); + Response GetString(const char* url, const std::map& headers, tcb::span acceptedCodes, uint32_t timeoutMs = 10'000); template - Response GetJSON(const char* url, const std::map& headers, JsonParser jsonParser, tcb::span acceptedCodes, uint32_t timeoutMs = 10'000) + Response GetJSON(const char* url, const std::map& headers, JsonParser jsonParser, tcb::span acceptedCodes, uint32_t timeoutMs = 10'000) { auto response = GetString(url, headers, acceptedCodes, timeoutMs); if (response.result != RequestResult::Success) { diff --git a/include/http/HTTPResponse.h b/include/http/HTTPResponse.h deleted file mode 100644 index 0199eebf..00000000 --- a/include/http/HTTPResponse.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#include -#include - -#include -#include - -namespace OpenShock::HTTP { - class HTTPClient; - class HTTPResponse { - DISABLE_DEFAULT(HTTPResponse); - DISABLE_COPY(HTTPResponse); - - friend class HTTPClient; - - HTTPResponse(esp_err_t error) - : m_handle(nullptr) - , m_error(error) - { - } - HTTPResponse(esp_http_client_handle_t handle, std::int64_t contentLength, bool isChunked) - : m_handle(handle) - , m_contentLength(contentLength) - , m_isChunked(isChunked) - { - if (m_handle == nullptr) { - m_error = ESP_FAIL; - return; - } - } - - public: - HTTPResponse(HTTPResponse&& other) - { - m_handle = other.m_handle; - other.m_handle = nullptr; - } - - constexpr bool IsValid() const { return m_handle != nullptr; } - constexpr esp_err_t GetError() const { return m_handle == nullptr ? m_error : ESP_OK; } - - int ResponseCode() const { return esp_http_client_get_status_code(m_handle); } - - private: - esp_http_client_handle_t m_handle; - union { - struct { // Only valid if m_handle is not nullptr. - std::int64_t m_contentLength; - bool m_isChunked; - }; - esp_err_t m_error; // If m_handle is nullptr, this is the error code. - }; - }; -} // namespace OpenShock::HTTP diff --git a/include/http/JsonAPI.h b/include/http/JsonAPI.h index fc735bb4..c713a2e9 100644 --- a/include/http/JsonAPI.h +++ b/include/http/JsonAPI.h @@ -14,10 +14,10 @@ namespace OpenShock::HTTP::JsonAPI { /// @brief Gets the hub info for the given hub token. Valid response codes: 200, 401 /// @param hubToken /// @return - HTTP::Response GetHubInfo(std::string_view hubToken); + HTTP::Response GetHubInfo(std::string hubToken); /// @brief Requests a Live Control Gateway to connect to. Valid response codes: 200, 401 /// @param hubToken /// @return - HTTP::Response AssignLcg(std::string_view hubToken); + HTTP::Response AssignLcg(std::string hubToken); } // namespace OpenShock::HTTP::JsonAPI diff --git a/src/http/HTTPRequestManager.cpp b/src/http/HTTPRequestManager.cpp index 0a389e29..4d0c6bb7 100644 --- a/src/http/HTTPRequestManager.cpp +++ b/src/http/HTTPRequestManager.cpp @@ -4,14 +4,13 @@ const char* const TAG = "HTTPRequestManager"; #include "Common.h" #include "Core.h" -#include "http/HTTPClient.h" #include "Logging.h" #include "RateLimiter.h" #include "SimpleMutex.h" #include "util/HexUtils.h" #include "util/StringUtils.h" -#include "WiFiClient.h" +#include #include #include @@ -350,9 +349,9 @@ StreamReaderResult _readStreamData(HTTP::HTTPClient& client, WiFiClient* stream, } HTTP::Response _doGetStream( - HTTP::HTTPClient& client, + esp_http_client_handle_t client, const char* url, - const std::map& headers, + const std::map& headers, tcb::span acceptedCodes, std::shared_ptr rateLimiter, HTTP::GotContentLengthCallback contentLengthCallback, @@ -360,19 +359,25 @@ HTTP::Response _doGetStream( int timeoutMs ) { + esp_err_t err; + int64_t begin = OpenShock::millis(); for (auto& header : headers) { - client.SetHeader(header.first, header.second); + err = esp_http_client_set_header(client, header.first.c_str(), header.second.c_str()); + if (err != ESP_OK) { + // TODO: Handle error + } + return {HTTP::RequestResult::RequestFailed, 0, 0}; } - auto response = client.Get(url); - if (!response.IsValid()) { - esp_err_t err = response.GetError(); + err = esp_http_client_open(client, 0); + if (err != ESP_OK) { OS_LOGE(TAG, "Failed to begin HTTP request"); return {HTTP::RequestResult::RequestFailed, 0, 0}; } + err = esp_http_ auto responseCode = response.ResponseCode(); if (responseCode == HTTP_CODE_REQUEST_TIMEOUT || begin + timeoutMs < OpenShock::millis()) { OS_LOGW(TAG, "Request timed out"); @@ -383,7 +388,7 @@ HTTP::Response _doGetStream( // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After // Get "Retry-After" header - String retryAfterStr = client.header("Retry-After"); + std::string retryAfterStr = client.header("Retry-After"); // Try to parse it as an integer (delay-seconds) long retryAfter = 0; @@ -444,7 +449,7 @@ HTTP::Response _doGetStream( return {result.result, responseCode, result.nWritten}; } -HTTP::Response HTTP::Download(const char* url, const std::map& headers, HTTP::GotContentLengthCallback contentLengthCallback, HTTP::DownloadCallback downloadCallback, tcb::span acceptedCodes, uint32_t timeoutMs) +HTTP::Response HTTP::Download(const char* url, const std::map& headers, HTTP::GotContentLengthCallback contentLengthCallback, HTTP::DownloadCallback downloadCallback, tcb::span acceptedCodes, uint32_t timeoutMs) { std::shared_ptr rateLimiter = _getRateLimiter(url); if (rateLimiter == nullptr) { @@ -455,12 +460,29 @@ HTTP::Response HTTP::Download(const char* url, const std::map(this), + .is_async = true, + .use_global_ca_store = true, + }; + esp_http_client_handle_t client = esp_http_client_init(&cfg); + + auto result = _doGetStream(client, url, headers, acceptedCodes, rateLimiter, contentLengthCallback, downloadCallback, timeoutMs); + + esp_err_t err = esp_http_client_cleanup(client); + if (err != ESP_OK) { + // TODO: Handle error + } - return _doGetStream(client, url, headers, acceptedCodes, rateLimiter, contentLengthCallback, downloadCallback, timeoutMs); + return result; } -HTTP::Response HTTP::GetString(const char* url, const std::map& headers, tcb::span acceptedCodes, uint32_t timeoutMs) +HTTP::Response HTTP::GetString(const char* url, const std::map& headers, tcb::span acceptedCodes, uint32_t timeoutMs) { std::string result; diff --git a/src/http/JsonAPI.cpp b/src/http/JsonAPI.cpp index a83a4c09..cab68ee9 100644 --- a/src/http/JsonAPI.cpp +++ b/src/http/JsonAPI.cpp @@ -26,7 +26,7 @@ HTTP::Response HTTP::JsonAPI::LinkA ); } -HTTP::Response HTTP::JsonAPI::GetHubInfo(std::string_view hubToken) +HTTP::Response HTTP::JsonAPI::GetHubInfo(std::string hubToken) { std::string domain; if (!Config::GetBackendDomain(domain)) { @@ -39,15 +39,15 @@ HTTP::Response HTTP::JsonAPI::GetHubInf return HTTP::GetJSON( uri, { - { "Accept", "application/json"}, - {"DeviceToken", OpenShock::StringToArduinoString(hubToken)} - }, + { "Accept", "application/json"}, + {"DeviceToken", std::move(hubToken)} + }, Serialization::JsonAPI::ParseHubInfoJsonResponse, std::array {200} ); } -HTTP::Response HTTP::JsonAPI::AssignLcg(std::string_view hubToken) +HTTP::Response HTTP::JsonAPI::AssignLcg(std::string hubToken) { std::string domain; if (!Config::GetBackendDomain(domain)) { @@ -60,9 +60,9 @@ HTTP::Response HTTP::JsonAPI::AssignL return HTTP::GetJSON( uri, { - { "Accept", "application/json"}, - {"DeviceToken", OpenShock::StringToArduinoString(hubToken)} - }, + { "Accept", "application/json"}, + {"DeviceToken", std::move(hubToken)} + }, Serialization::JsonAPI::ParseAssignLcgJsonResponse, std::array {200} ); From c342d31e58867b5008c18783a6da6fa8d8c55d0d Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Thu, 4 Dec 2025 03:21:48 +0100 Subject: [PATCH 05/28] More work --- include/http/HTTPClient.h | 99 +++++++++++++++ include/http/HTTPRequestManager.h | 87 ------------- include/http/JsonAPI.h | 8 +- include/http/RateLimiters.h | 9 ++ include/serialization/JsonAPI.h | 10 +- include/util/DomainUtils.h | 7 ++ src/GatewayConnectionManager.cpp | 13 +- src/http/HTTPClient.cpp | 161 ++++++++++++++++++++++++ src/http/HTTPRequestManager.cpp | 197 +++++------------------------- src/http/JsonAPI.cpp | 70 ++++++----- src/http/RateLimiters.cpp | 47 +++++++ src/serialization/JsonAPI.cpp | 10 +- src/util/DomainUtils.cpp | 37 ++++++ 13 files changed, 451 insertions(+), 304 deletions(-) create mode 100644 include/http/HTTPClient.h delete mode 100644 include/http/HTTPRequestManager.h create mode 100644 include/http/RateLimiters.h create mode 100644 include/util/DomainUtils.h create mode 100644 src/http/HTTPClient.cpp create mode 100644 src/http/RateLimiters.cpp create mode 100644 src/util/DomainUtils.cpp diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h new file mode 100644 index 00000000..6a047c4b --- /dev/null +++ b/include/http/HTTPClient.h @@ -0,0 +1,99 @@ +#pragma once + +#include "Common.h" +#include "RateLimiter.h" + +#include + +#include + +#include +#include +#include +#include + +namespace OpenShock::HTTP { + enum class DownloadResult : uint8_t { + Closed, // Connection closed + Success, // Request completed successfully + TimedOut, // Request timed out + ParseFailed, // Request completed, but JSON parsing failed + Cancelled, // Request was cancelled + }; + + template + struct [[nodiscard]] Response { + DownloadResult result; + esp_err_t error; + T data; + }; + + template + using JsonParser = std::function; + using GotContentLengthCallback = std::function; + using DownloadCallback = std::function; + + class HTTPClient { + DISABLE_COPY(HTTPClient); + DISABLE_MOVE(HTTPClient); + + public: + HTTPClient(uint32_t timeoutMs = 10'000, const char* useragent = OpenShock::Constants::FW_USERAGENT); + ~HTTPClient(); + + inline esp_err_t SetHeader(const char* key, const char* value) { + return esp_http_client_set_header(m_handle, key, value); + } + esp_err_t SetHeaders(const std::map& headers); + + esp_err_t Get(const char* url); + + inline int ResponseLength() const { + return m_responseLength; + } + inline int StatusCode() const { + return m_statusCode; + } + + Response ReadResponseStream(DownloadCallback downloadCallback); + Response ReadResponseString(); + template + inline Response ReadResponseJSON(JsonParser jsonParser) + { + auto response = ReadResponseString(); + if (response.result != DownloadResult::Success) { + return {response.result, response.error, {}}; + } + + cJSON* json = cJSON_ParseWithLength(response.data.c_str(), response.data.length()); + if (json == nullptr) { + return {DownloadResult::ParseFailed, ESP_OK, {}}; + } + + T data; + if (!jsonParser(json, data)) { + return {DownloadResult::ParseFailed, ESP_OK, {}}; + } + + cJSON_Delete(json); + + return {response.result, ESP_OK, std::move(data)}; + } + + esp_err_t Close(); + private: + esp_err_t Start(esp_http_client_method_t method, const char* url, int writeLen); + + static esp_err_t EventHandler(esp_http_client_event_t* evt); + esp_err_t HandleHeader(std::string_view key, std::string_view value); + + esp_http_client_handle_t m_handle; + std::shared_ptr m_ratelimiter; + bool m_connected; + int m_responseLength; + int m_statusCode; + + GotContentLengthCallback m_cbGotContentLength; + DownloadCallback m_cbDownload; + }; +} // namespace OpenShock::HTTP diff --git a/include/http/HTTPRequestManager.h b/include/http/HTTPRequestManager.h deleted file mode 100644 index 59621a37..00000000 --- a/include/http/HTTPRequestManager.h +++ /dev/null @@ -1,87 +0,0 @@ -#pragma once - -#include - -#include -#include -#include - -#include "span.h" - -namespace OpenShock::HTTP { - enum class RequestResult : uint8_t { - InternalError, // Internal error - InvalidURL, // Invalid URL - RequestFailed, // Failed to start request - TimedOut, // Request timed out - RateLimited, // Rate limited (can be both local and global) - CodeRejected, // Request completed, but response code was not OK - ParseFailed, // Request completed, but JSON parsing failed - Cancelled, // Request was cancelled - Success, // Request completed successfully - }; - - template - struct [[nodiscard]] Response { - RequestResult result; - int code; - T data; - - inline const char* ResultToString() const - { - switch (result) { - case RequestResult::InternalError: - return "Internal error"; - case RequestResult::InvalidURL: - return "Requested url was invalid"; - case RequestResult::RequestFailed: - return "Request failed"; - case RequestResult::TimedOut: - return "Request timed out"; - case RequestResult::RateLimited: - return "Client was ratelimited"; - case RequestResult::CodeRejected: - return "Unexpected response code"; - case RequestResult::ParseFailed: - return "Parsing the response failed"; - case RequestResult::Cancelled: - return "Request was cancelled"; - case RequestResult::Success: - return "Success"; - default: - return "Unknown reason"; - } - } - }; - - template - using JsonParser = std::function; - using GotContentLengthCallback = std::function; - using DownloadCallback = std::function; - - Response Download(const char* url, const std::map& headers, GotContentLengthCallback contentLengthCallback, DownloadCallback downloadCallback, tcb::span acceptedCodes, uint32_t timeoutMs = 10'000); - Response GetString(const char* url, const std::map& headers, tcb::span acceptedCodes, uint32_t timeoutMs = 10'000); - - template - Response GetJSON(const char* url, const std::map& headers, JsonParser jsonParser, tcb::span acceptedCodes, uint32_t timeoutMs = 10'000) - { - auto response = GetString(url, headers, acceptedCodes, timeoutMs); - if (response.result != RequestResult::Success) { - return {response.result, response.code, {}}; - } - - cJSON* json = cJSON_ParseWithLength(response.data.c_str(), response.data.length()); - if (json == nullptr) { - return {RequestResult::ParseFailed, response.code, {}}; - } - - T data; - if (!jsonParser(response.code, json, data)) { - return {RequestResult::ParseFailed, response.code, {}}; - } - - cJSON_Delete(json); - - return {response.result, response.code, std::move(data)}; - } -} // namespace OpenShock::HTTP diff --git a/include/http/JsonAPI.h b/include/http/JsonAPI.h index c713a2e9..64887f83 100644 --- a/include/http/JsonAPI.h +++ b/include/http/JsonAPI.h @@ -1,6 +1,6 @@ #pragma once -#include "http/HTTPRequestManager.h" +#include "http/HTTPClient.h" #include "serialization/JsonAPI.h" #include @@ -9,15 +9,15 @@ namespace OpenShock::HTTP::JsonAPI { /// @brief Links the hub to the account with the given account link code, returns the hub token. Valid response codes: 200, 404 /// @param hubToken /// @return - HTTP::Response LinkAccount(std::string_view accountLinkCode); + HTTP::Response LinkAccount(HTTP::HTTPClient& client, std::string_view accountLinkCode); /// @brief Gets the hub info for the given hub token. Valid response codes: 200, 401 /// @param hubToken /// @return - HTTP::Response GetHubInfo(std::string hubToken); + HTTP::Response GetHubInfo(HTTP::HTTPClient& client, const char* hubToken); /// @brief Requests a Live Control Gateway to connect to. Valid response codes: 200, 401 /// @param hubToken /// @return - HTTP::Response AssignLcg(std::string hubToken); + HTTP::Response AssignLcg(HTTP::HTTPClient& client, const char* hubToken); } // namespace OpenShock::HTTP::JsonAPI diff --git a/include/http/RateLimiters.h b/include/http/RateLimiters.h new file mode 100644 index 00000000..0e1991d7 --- /dev/null +++ b/include/http/RateLimiters.h @@ -0,0 +1,9 @@ +#pragma once + +#include "RateLimiter.h" + +#include + +namespace OpenShock::HTTP::RateLimiters { + std::shared_ptr GetRateLimiter(std::string_view url); +} // namespace OpenShock::HTTP diff --git a/include/serialization/JsonAPI.h b/include/serialization/JsonAPI.h index 322e8c8b..a4a17a6c 100644 --- a/include/serialization/JsonAPI.h +++ b/include/serialization/JsonAPI.h @@ -41,9 +41,9 @@ namespace OpenShock::Serialization::JsonAPI { std::string country; }; - bool ParseLcgInstanceDetailsJsonResponse(int code, const cJSON* root, LcgInstanceDetailsResponse& out); - bool ParseBackendVersionJsonResponse(int code, const cJSON* root, BackendVersionResponse& out); - bool ParseAccountLinkJsonResponse(int code, const cJSON* root, AccountLinkResponse& out); - bool ParseHubInfoJsonResponse(int code, const cJSON* root, HubInfoResponse& out); - bool ParseAssignLcgJsonResponse(int code, const cJSON* root, AssignLcgResponse& out); + bool ParseLcgInstanceDetailsJsonResponse(const cJSON* root, LcgInstanceDetailsResponse& out); + bool ParseBackendVersionJsonResponse(const cJSON* root, BackendVersionResponse& out); + bool ParseAccountLinkJsonResponse(const cJSON* root, AccountLinkResponse& out); + bool ParseHubInfoJsonResponse(const cJSON* root, HubInfoResponse& out); + bool ParseAssignLcgJsonResponse(const cJSON* root, AssignLcgResponse& out); } // namespace OpenShock::Serialization::JsonAPI diff --git a/include/util/DomainUtils.h b/include/util/DomainUtils.h new file mode 100644 index 00000000..99ad7248 --- /dev/null +++ b/include/util/DomainUtils.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +namespace OpenShock::DomainUtils { + std::string_view GetDomainFromUrl(std::string_view url); +} // namespace OpenShock::DomainUtils diff --git a/src/GatewayConnectionManager.cpp b/src/GatewayConnectionManager.cpp index 3204373c..c09d8bfb 100644 --- a/src/GatewayConnectionManager.cpp +++ b/src/GatewayConnectionManager.cpp @@ -105,7 +105,8 @@ AccountLinkResultCode GatewayConnectionManager::Link(std::string_view linkCode) return AccountLinkResultCode::InvalidCode; } - auto response = HTTP::JsonAPI::LinkAccount(linkCode); + HTTP::HTTPClient client; + auto response = HTTP::JsonAPI::LinkAccount(client, linkCode); if (response.code == 404) { return AccountLinkResultCode::InvalidCode; @@ -166,7 +167,7 @@ bool GatewayConnectionManager::SendMessageBIN(tcb::span data) return s_wsClient->sendMessageBIN(data); } -bool FetchHubInfo(std::string authToken) +bool FetchHubInfo(const char* authToken) { // TODO: this function is very slow, should be optimized! if ((s_flags & FLAG_HAS_IP) == 0) { @@ -177,7 +178,8 @@ bool FetchHubInfo(std::string authToken) return false; } - auto response = HTTP::JsonAPI::GetHubInfo(std::move(authToken)); + HTTP::HTTPClient client; + auto response = HTTP::JsonAPI::GetHubInfo(client, authToken); if (response.code == 401) { OS_LOGD(TAG, "Auth token is invalid, waiting 5 minutes before checking again"); @@ -241,7 +243,8 @@ bool StartConnectingToLCG() return false; } - auto response = HTTP::JsonAPI::AssignLcg(std::move(authToken)); + HTTP::HTTPClient client; + auto response = HTTP::JsonAPI::AssignLcg(client, authToken.c_str()); if (response.code == 401) { OS_LOGD(TAG, "Auth token is invalid, waiting 5 minutes before retrying"); @@ -283,7 +286,7 @@ void GatewayConnectionManager::Update() } // Fetch hub info - if (!FetchHubInfo(std::move(authToken))) { + if (!FetchHubInfo(authToken.c_str())) { return; } diff --git a/src/http/HTTPClient.cpp b/src/http/HTTPClient.cpp new file mode 100644 index 00000000..c6868aae --- /dev/null +++ b/src/http/HTTPClient.cpp @@ -0,0 +1,161 @@ +#include "http/HTTPClient.h" + +const char* const TAG = "HTTPClient"; + +#include "http/RateLimiters.h" +#include "Logging.h" +#include "util/DomainUtils.h" + +const std::size_t HTTP_BUFFER_SIZE = 4096LLU; +const int HTTP_DOWNLOAD_SIZE_LIMIT = 200 * 1024 * 1024; // 200 MB + +using namespace OpenShock; + +HTTP::HTTPClient::HTTPClient(uint32_t timeoutMs, const char* useragent) +{ + esp_http_client_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + + cfg.timeout_ms = static_cast(std::min(timeoutMs, INT32_MAX)); + cfg.disable_auto_redirect = true; + cfg.event_handler = HTTPClient::EventHandler; + cfg.transport_type = HTTP_TRANSPORT_OVER_SSL; + cfg.user_data = reinterpret_cast(this); + cfg.is_async = false; + cfg.use_global_ca_store = true; + #warning This still uses SSL, upgrade to TLS! (latest ESP-IDF) + + m_handle = esp_http_client_init(&cfg); +} + +HTTP::HTTPClient::~HTTPClient() +{ + esp_http_client_cleanup(m_handle); +} + +esp_err_t HTTP::HTTPClient::SetHeaders(const std::map& headers) { + esp_err_t err; + + for (auto& header : headers) { + err = SetHeader(header.first.c_str(), header.second.c_str()); + + if (err != ESP_OK) return err; + } + + return ESP_OK; +} + +esp_err_t HTTP::HTTPClient::Get(const char* url) { + esp_err_t err; + + err = Start(HTTP_METHOD_GET, url, 0); + if (err != ESP_OK) return err; + + m_responseLength = esp_http_client_fetch_headers(m_handle); + if (m_responseLength == ESP_FAIL) { + return ESP_FAIL; + } + + m_statusCode = esp_http_client_get_status_code(m_handle); + + return ESP_OK; +} + +HTTP::Response HTTP::HTTPClient::ReadResponseStream(DownloadCallback downloadCallback) { + if (!m_connected) { + return {DownloadResult::Closed, ESP_FAIL, 0}; + } + + std::size_t nWritten; + + return {DownloadResult::Success, ESP_OK, nWritten}; +} + +HTTP::Response HTTP::HTTPClient::ReadResponseString() { + std::string result; + if (m_responseLength > 0) { + result.reserve(m_responseLength); + } + + auto writer = [&result](std::size_t offset, const uint8_t* data, std::size_t len) { + result.append(reinterpret_cast(data), len); + return true; + }; + + auto response = ReadResponseStream(writer); + if (response.result != DownloadResult::Success) { + return {response.result, response.error, {}}; + } + + return {response.result, response.error, result}; +} + +esp_err_t HTTP::HTTPClient::Close() { + return esp_http_client_close(m_handle); +} + +esp_err_t HTTP::HTTPClient::Start(esp_http_client_method_t method, const char* url, int writeLen) { + esp_err_t err; + + m_ratelimiter = HTTP::RateLimiters::GetRateLimiter(url); + if (m_ratelimiter == nullptr) { + OS_LOGW(TAG, "Invalid URL!"); + return ESP_FAIL; + } + if (!m_ratelimiter->tryRequest()) { + OS_LOGW(TAG, "Hit ratelimit, refusing to send request!"); + return ESP_FAIL; + } + + err = esp_http_client_set_url(m_handle, url); + if (err != ESP_OK) return err; + + err = esp_http_client_set_method(m_handle, HTTP_METHOD_GET); + if (err != ESP_OK) return err; + + return esp_http_client_open(m_handle, writeLen); +} + +esp_err_t HTTP::HTTPClient::EventHandler(esp_http_client_event_t* evt) { + HTTPClient* client = reinterpret_cast(evt->user_data); + + switch (evt->event_id) + { + case HTTP_EVENT_ERROR: + OS_LOGE(TAG, "Got error event"); + break; + case HTTP_EVENT_ON_CONNECTED: + client->m_connected = true; + OS_LOGI(TAG, "Got connected event"); + break; + case HTTP_EVENT_HEADERS_SENT: + OS_LOGI(TAG, "Got headers_sent event"); + break; + case HTTP_EVENT_ON_HEADER: + OS_LOGI(TAG, "Got header_received event: %s - %s", evt->header_key, evt->header_value); + return client->HandleHeader(evt->header_key, evt->header_value); + case HTTP_EVENT_ON_DATA: + OS_LOGI(TAG, "Got on_data event"); + break; + case HTTP_EVENT_ON_FINISH: + OS_LOGI(TAG, "Got on_finish event"); + break; + case HTTP_EVENT_DISCONNECTED: + client->m_connected = false; + OS_LOGI(TAG, "Got disconnected event"); + break; + default: + OS_LOGE(TAG, "Got unknown event"); + break; + } + + return ESP_OK; +} + +esp_err_t HTTP::HTTPClient::HandleHeader(std::string_view key, std::string_view value) { + if (key == "Retry-After") { + // TODO: Set block on m_ratelimiter + } + + return ESP_OK; +} diff --git a/src/http/HTTPRequestManager.cpp b/src/http/HTTPRequestManager.cpp index 4d0c6bb7..1394823f 100644 --- a/src/http/HTTPRequestManager.cpp +++ b/src/http/HTTPRequestManager.cpp @@ -1,17 +1,16 @@ -#include "http/HTTPRequestManager.h" +/* const char* const TAG = "HTTPRequestManager"; #include "Common.h" #include "Core.h" #include "Logging.h" +#include "http/HTTPClient.h" #include "RateLimiter.h" #include "SimpleMutex.h" #include "util/HexUtils.h" #include "util/StringUtils.h" -#include - #include #include #include @@ -23,87 +22,10 @@ using namespace std::string_view_literals; const std::size_t HTTP_BUFFER_SIZE = 4096LLU; const int HTTP_DOWNLOAD_SIZE_LIMIT = 200 * 1024 * 1024; // 200 MB -static OpenShock::SimpleMutex s_rateLimitsMutex = {}; -static std::unordered_map> s_rateLimits = {}; - using namespace OpenShock; -std::string_view _getDomain(std::string_view url) -{ - if (url.empty()) { - return {}; - } - - // Remove the protocol eg. "https://api.example.com:443/path" -> "api.example.com:443/path" - auto seperator = url.find("://"); - if (seperator != std::string_view::npos) { - url.substr(seperator + 3); - } - - // Remove the path eg. "api.example.com:443/path" -> "api.example.com:443" - seperator = url.find('/'); - if (seperator != std::string_view::npos) { - url = url.substr(0, seperator); - } - - // Remove the port eg. "api.example.com:443" -> "api.example.com" - seperator = url.rfind(':'); - if (seperator != std::string_view::npos) { - url = url.substr(0, seperator); - } - - // Remove all subdomains eg. "api.example.com" -> "example.com" - seperator = url.rfind('.'); - if (seperator == std::string_view::npos) { - return url; // E.g. "localhost" - } - seperator = url.rfind('.', seperator - 1); - if (seperator != std::string_view::npos) { - url = url.substr(seperator + 1); - } - - return url; -} - -std::shared_ptr _rateLimiterFactory(std::string_view domain) -{ - auto rateLimit = std::make_shared(); - - // Add default limits - rateLimit->addLimit(1000, 5); // 5 per second - rateLimit->addLimit(10 * 1000, 10); // 10 per 10 seconds - - // per-domain limits - if (domain == OPENSHOCK_API_DOMAIN) { - rateLimit->addLimit(60 * 1000, 12); // 12 per minute - rateLimit->addLimit(60 * 60 * 1000, 120); // 120 per hour - } - - return rateLimit; -} - -std::shared_ptr _getRateLimiter(std::string_view url) -{ - auto domain = std::string(_getDomain(url)); - if (domain.empty()) { - return nullptr; - } - - s_rateLimitsMutex.lock(portMAX_DELAY); - - auto it = s_rateLimits.find(domain); - if (it == s_rateLimits.end()) { - s_rateLimits.emplace(domain, _rateLimiterFactory(domain)); - it = s_rateLimits.find(domain); - } - - s_rateLimitsMutex.unlock(); - - return it->second; -} - struct StreamReaderResult { - HTTP::RequestResult result; + HTTP::DownloadResult result; std::size_t nWritten; }; @@ -220,15 +142,15 @@ void _alignChunk(uint8_t* buffer, std::size_t& bufferCursor, std::size_t payload } } -StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient& client, WiFiClient* stream, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) +StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient client, WiFiClient* stream, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) { std::size_t totalWritten = 0; - HTTP::RequestResult result = HTTP::RequestResult::Success; + HTTP::DownloadResult result = HTTP::DownloadResult::Success; uint8_t* buffer = static_cast(malloc(HTTP_BUFFER_SIZE)); if (buffer == nullptr) { OS_LOGE(TAG, "Out of memory"); - return {HTTP::RequestResult::RequestFailed, 0}; + return {HTTP::DownloadResult::RequestFailed, 0}; } ParserState state = ParserState::NeedMoreData; @@ -237,7 +159,7 @@ StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient& client, WiFiClient* while (client.connected() && state != ParserState::Invalid) { if (begin + timeoutMs < OpenShock::millis()) { OS_LOGW(TAG, "Request timed out"); - result = HTTP::RequestResult::TimedOut; + result = HTTP::DownloadResult::TimedOut; break; } @@ -250,7 +172,7 @@ StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient& client, WiFiClient* std::size_t bytesRead = stream->readBytes(buffer + bufferCursor, HTTP_BUFFER_SIZE - bufferCursor); if (bytesRead == 0) { OS_LOGW(TAG, "No bytes read"); - result = HTTP::RequestResult::RequestFailed; + result = HTTP::DownloadResult::RequestFailed; break; } @@ -260,7 +182,7 @@ StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient& client, WiFiClient* state = _parseChunk(buffer, bufferCursor, payloadPos, payloadSize); if (state == ParserState::Invalid) { OS_LOGE(TAG, "Failed to parse chunk"); - result = HTTP::RequestResult::RequestFailed; + result = HTTP::DownloadResult::RequestFailed; state = ParserState::Invalid; // Mark to exit both loops break; } @@ -269,7 +191,7 @@ StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient& client, WiFiClient* if (state == ParserState::NeedMoreData) { if (bufferCursor == HTTP_BUFFER_SIZE) { OS_LOGE(TAG, "Chunk too large"); - result = HTTP::RequestResult::RequestFailed; + result = HTTP::DownloadResult::RequestFailed; state = ParserState::Invalid; // Mark to exit both loops } break; // If chunk size good, this only exits one loop @@ -282,7 +204,7 @@ StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient& client, WiFiClient* } if (!downloadCallback(totalWritten, buffer + payloadPos, payloadSize)) { - result = HTTP::RequestResult::Cancelled; + result = HTTP::DownloadResult::Cancelled; state = ParserState::Invalid; // Mark to exit both loops break; } @@ -303,17 +225,17 @@ StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient& client, WiFiClient* return {result, totalWritten}; } -StreamReaderResult _readStreamData(HTTP::HTTPClient& client, WiFiClient* stream, std::size_t contentLength, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) +StreamReaderResult _readStreamData(HTTP::HTTPClient client, WiFiClient* stream, std::size_t contentLength, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) { std::size_t nWritten = 0; - HTTP::RequestResult result = HTTP::RequestResult::Success; + HTTP::DownloadResult result = HTTP::DownloadResult::Success; uint8_t* buffer = static_cast(malloc(HTTP_BUFFER_SIZE)); while (client.connected() && nWritten < contentLength) { if (begin + timeoutMs < OpenShock::millis()) { OS_LOGW(TAG, "Request timed out"); - result = HTTP::RequestResult::TimedOut; + result = HTTP::DownloadResult::TimedOut; break; } @@ -328,13 +250,13 @@ StreamReaderResult _readStreamData(HTTP::HTTPClient& client, WiFiClient* stream, std::size_t bytesRead = stream->readBytes(buffer, bytesToRead); if (bytesRead == 0) { OS_LOGW(TAG, "No bytes read"); - result = HTTP::RequestResult::RequestFailed; + result = HTTP::DownloadResult::RequestFailed; break; } if (!downloadCallback(nWritten, buffer, bytesRead)) { OS_LOGW(TAG, "Request cancelled by callback"); - result = HTTP::RequestResult::Cancelled; + result = HTTP::DownloadResult::Cancelled; break; } @@ -349,11 +271,9 @@ StreamReaderResult _readStreamData(HTTP::HTTPClient& client, WiFiClient* stream, } HTTP::Response _doGetStream( - esp_http_client_handle_t client, + HTTP::HTTPClient& client, const char* url, - const std::map& headers, tcb::span acceptedCodes, - std::shared_ptr rateLimiter, HTTP::GotContentLengthCallback contentLengthCallback, HTTP::DownloadCallback downloadCallback, int timeoutMs @@ -363,25 +283,17 @@ HTTP::Response _doGetStream( int64_t begin = OpenShock::millis(); - for (auto& header : headers) { - err = esp_http_client_set_header(client, header.first.c_str(), header.second.c_str()); - if (err != ESP_OK) { - // TODO: Handle error - } - return {HTTP::RequestResult::RequestFailed, 0, 0}; - } - - err = esp_http_client_open(client, 0); + err = client.Get(url); if (err != ESP_OK) { OS_LOGE(TAG, "Failed to begin HTTP request"); - return {HTTP::RequestResult::RequestFailed, 0, 0}; + return {HTTP::DownloadResult::RequestFailed, 0, 0}; } - err = esp_http_ + err = esp_http_res auto responseCode = response.ResponseCode(); if (responseCode == HTTP_CODE_REQUEST_TIMEOUT || begin + timeoutMs < OpenShock::millis()) { OS_LOGW(TAG, "Request timed out"); - return {HTTP::RequestResult::TimedOut, responseCode, 0}; + return {HTTP::DownloadResult::TimedOut, responseCode, 0}; } if (responseCode == HTTP_CODE_TOO_MANY_REQUESTS) { @@ -404,7 +316,7 @@ HTTP::Response _doGetStream( // Apply the block-for time rateLimiter->blockFor(retryAfter * 1000); - return {HTTP::RequestResult::RateLimited, responseCode, 0}; + return {HTTP::DownloadResult::RateLimited, responseCode, 0}; } if (responseCode == 418) { @@ -413,30 +325,30 @@ HTTP::Response _doGetStream( if (std::find(acceptedCodes.begin(), acceptedCodes.end(), responseCode) == acceptedCodes.end()) { OS_LOGD(TAG, "Received unexpected response code %d", responseCode); - return {HTTP::RequestResult::CodeRejected, responseCode, 0}; + return {HTTP::DownloadResult::CodeRejected, responseCode, 0}; } int contentLength = client.getSize(); if (contentLength == 0) { - return {HTTP::RequestResult::Success, responseCode, 0}; + return {HTTP::DownloadResult::Success, responseCode, 0}; } if (contentLength > 0) { if (contentLength > HTTP_DOWNLOAD_SIZE_LIMIT) { OS_LOGE(TAG, "Content-Length too large"); - return {HTTP::RequestResult::RequestFailed, responseCode, 0}; + return {HTTP::DownloadResult::RequestFailed, responseCode, 0}; } if (!contentLengthCallback(contentLength)) { OS_LOGW(TAG, "Request cancelled by callback"); - return {HTTP::RequestResult::Cancelled, responseCode, 0}; + return {HTTP::DownloadResult::Cancelled, responseCode, 0}; } } WiFiClient* stream = client.getStreamPtr(); if (stream == nullptr) { OS_LOGE(TAG, "Failed to get stream"); - return {HTTP::RequestResult::RequestFailed, 0, 0}; + return {HTTP::DownloadResult::RequestFailed, 0, 0}; } StreamReaderResult result; @@ -448,57 +360,4 @@ HTTP::Response _doGetStream( return {result.result, responseCode, result.nWritten}; } - -HTTP::Response HTTP::Download(const char* url, const std::map& headers, HTTP::GotContentLengthCallback contentLengthCallback, HTTP::DownloadCallback downloadCallback, tcb::span acceptedCodes, uint32_t timeoutMs) -{ - std::shared_ptr rateLimiter = _getRateLimiter(url); - if (rateLimiter == nullptr) { - return {RequestResult::InvalidURL, 0, 0}; - } - - if (!rateLimiter->tryRequest()) { - return {RequestResult::RateLimited, 0, 0}; - } - - esp_http_client_config_t cfg = { - .url = url, - .user_agent = OpenShock::Constants::FW_USERAGENT, - .timeout_ms = 10'000, - .disable_auto_redirect = true, - .event_handler = HTTPClient::EventHandler, - .user_data = reinterpret_cast(this), - .is_async = true, - .use_global_ca_store = true, - }; - esp_http_client_handle_t client = esp_http_client_init(&cfg); - - auto result = _doGetStream(client, url, headers, acceptedCodes, rateLimiter, contentLengthCallback, downloadCallback, timeoutMs); - - esp_err_t err = esp_http_client_cleanup(client); - if (err != ESP_OK) { - // TODO: Handle error - } - - return result; -} - -HTTP::Response HTTP::GetString(const char* url, const std::map& headers, tcb::span acceptedCodes, uint32_t timeoutMs) -{ - std::string result; - - auto allocator = [&result](std::size_t contentLength) { - result.reserve(contentLength); - return true; - }; - auto writer = [&result](std::size_t offset, const uint8_t* data, std::size_t len) { - result.append(reinterpret_cast(data), len); - return true; - }; - - auto response = Download(url, headers, allocator, writer, acceptedCodes, timeoutMs); - if (response.result != RequestResult::Success) { - return {response.result, response.code, {}}; - } - - return {response.result, response.code, result}; -} +*/ diff --git a/src/http/JsonAPI.cpp b/src/http/JsonAPI.cpp index cab68ee9..d0270a08 100644 --- a/src/http/JsonAPI.cpp +++ b/src/http/JsonAPI.cpp @@ -6,7 +6,7 @@ using namespace OpenShock; -HTTP::Response HTTP::JsonAPI::LinkAccount(std::string_view accountLinkCode) +HTTP::Response HTTP::JsonAPI::LinkAccount(HTTP::HTTPClient& client, std::string_view accountLinkCode) { std::string domain; if (!Config::GetBackendDomain(domain)) { @@ -16,17 +16,21 @@ HTTP::Response HTTP::JsonAPI::LinkA char uri[OPENSHOCK_URI_BUFFER_SIZE]; sprintf(uri, "https://%s/1/device/pair/%.*s", domain.c_str(), accountLinkCode.length(), accountLinkCode.data()); - return HTTP::GetJSON( - uri, - { - {"Accept", "application/json"} - }, - Serialization::JsonAPI::ParseAccountLinkJsonResponse, - std::array {200} - ); + client.SetHeader("Accept", "application/json"); + + esp_err_t err = client.Get(uri); + if (err != ESP_OK) { + return {HTTP::RequestResult::InternalError, 0, {}}; + } + + if (client.StatusCode() != 200) { + return {HTTP::RequestResult::InternalError, 0, {}}; + } + + return client.ReadResponseJSON(Serialization::JsonAPI::ParseAccountLinkJsonResponse); } -HTTP::Response HTTP::JsonAPI::GetHubInfo(std::string hubToken) +HTTP::Response HTTP::JsonAPI::GetHubInfo(HTTP::HTTPClient& client, const char* hubToken) { std::string domain; if (!Config::GetBackendDomain(domain)) { @@ -36,18 +40,22 @@ HTTP::Response HTTP::JsonAPI::GetHubInf char uri[OPENSHOCK_URI_BUFFER_SIZE]; sprintf(uri, "https://%s/1/device/self", domain.c_str()); - return HTTP::GetJSON( - uri, - { - { "Accept", "application/json"}, - {"DeviceToken", std::move(hubToken)} - }, - Serialization::JsonAPI::ParseHubInfoJsonResponse, - std::array {200} - ); + client.SetHeader("Accept", "application/json"); + client.SetHeader("DeviceToken", hubToken); + + esp_err_t err = client.Get(uri); + if (err != ESP_OK) { + return {HTTP::RequestResult::InternalError, 0, {}}; + } + + if (client.StatusCode() != 200) { + return {HTTP::RequestResult::InternalError, 0, {}}; + } + + return client.ReadResponseJSON(Serialization::JsonAPI::ParseHubInfoJsonResponse); } -HTTP::Response HTTP::JsonAPI::AssignLcg(std::string hubToken) +HTTP::Response HTTP::JsonAPI::AssignLcg(HTTP::HTTPClient& client, const char* hubToken) { std::string domain; if (!Config::GetBackendDomain(domain)) { @@ -57,13 +65,17 @@ HTTP::Response HTTP::JsonAPI::AssignL char uri[OPENSHOCK_URI_BUFFER_SIZE]; sprintf(uri, "https://%s/2/device/assignLCG?version=2", domain.c_str()); - return HTTP::GetJSON( - uri, - { - { "Accept", "application/json"}, - {"DeviceToken", std::move(hubToken)} - }, - Serialization::JsonAPI::ParseAssignLcgJsonResponse, - std::array {200} - ); + client.SetHeader("Accept", "application/json"); + client.SetHeader("DeviceToken", hubToken); + + esp_err_t err = client.Get(uri); + if (err != ESP_OK) { + return {HTTP::RequestResult::InternalError, 0, {}}; + } + + if (client.StatusCode() != 200) { + return {HTTP::RequestResult::InternalError, 0, {}}; + } + + return client.ReadResponseJSON(Serialization::JsonAPI::ParseAssignLcgJsonResponse); } diff --git a/src/http/RateLimiters.cpp b/src/http/RateLimiters.cpp new file mode 100644 index 00000000..c19647bc --- /dev/null +++ b/src/http/RateLimiters.cpp @@ -0,0 +1,47 @@ +#include "http/RateLimiters.h" + +#include "SimpleMutex.h" +#include "util/DomainUtils.h" + +#include +#include + +static OpenShock::SimpleMutex s_rateLimitsMutex = {}; +static std::unordered_map> s_rateLimits = {}; + +using namespace OpenShock; + +std::shared_ptr _rateLimiterFactory(std::string_view domain) +{ + auto rateLimit = std::make_shared(); + + // Add default limits + rateLimit->addLimit(1000, 5); // 5 per second + rateLimit->addLimit(10 * 1000, 10); // 10 per 10 seconds + + // per-domain limits + if (domain == OPENSHOCK_API_DOMAIN) { + rateLimit->addLimit(60 * 1000, 12); // 12 per minute + rateLimit->addLimit(60 * 60 * 1000, 120); // 120 per hour + } + + return rateLimit; +} + +std::shared_ptr HTTP::RateLimiters::GetRateLimiter(std::string_view url) +{ + auto domain = std::string(DomainUtils::GetDomainFromUrl(url)); + if (domain.empty()) { + return nullptr; + } + + OpenShock::ScopedLock lock__(&s_rateLimitsMutex); + + auto it = s_rateLimits.find(domain); + if (it == s_rateLimits.end()) { + s_rateLimits.emplace(domain, _rateLimiterFactory(domain)); + it = s_rateLimits.find(domain); + } + + return it->second; +} diff --git a/src/serialization/JsonAPI.cpp b/src/serialization/JsonAPI.cpp index 1a412ae2..7de08563 100644 --- a/src/serialization/JsonAPI.cpp +++ b/src/serialization/JsonAPI.cpp @@ -8,7 +8,7 @@ const char* const TAG = "JsonAPI"; using namespace OpenShock::Serialization; -bool JsonAPI::ParseLcgInstanceDetailsJsonResponse(int code, const cJSON* root, JsonAPI::LcgInstanceDetailsResponse& out) +bool JsonAPI::ParseLcgInstanceDetailsJsonResponse(const cJSON* root, JsonAPI::LcgInstanceDetailsResponse& out) { (void)code; @@ -57,7 +57,7 @@ bool JsonAPI::ParseLcgInstanceDetailsJsonResponse(int code, const cJSON* root, J return true; } -bool JsonAPI::ParseBackendVersionJsonResponse(int code, const cJSON* root, JsonAPI::BackendVersionResponse& out) +bool JsonAPI::ParseBackendVersionJsonResponse(const cJSON* root, JsonAPI::BackendVersionResponse& out) { (void)code; @@ -99,7 +99,7 @@ bool JsonAPI::ParseBackendVersionJsonResponse(int code, const cJSON* root, JsonA return true; } -bool JsonAPI::ParseAccountLinkJsonResponse(int code, const cJSON* root, JsonAPI::AccountLinkResponse& out) +bool JsonAPI::ParseAccountLinkJsonResponse(const cJSON* root, JsonAPI::AccountLinkResponse& out) { (void)code; @@ -120,7 +120,7 @@ bool JsonAPI::ParseAccountLinkJsonResponse(int code, const cJSON* root, JsonAPI: return true; } -bool JsonAPI::ParseHubInfoJsonResponse(int code, const cJSON* root, JsonAPI::HubInfoResponse& out) +bool JsonAPI::ParseHubInfoJsonResponse(const cJSON* root, JsonAPI::HubInfoResponse& out) { (void)code; @@ -213,7 +213,7 @@ bool JsonAPI::ParseHubInfoJsonResponse(int code, const cJSON* root, JsonAPI::Hub return true; } -bool JsonAPI::ParseAssignLcgJsonResponse(int code, const cJSON* root, JsonAPI::AssignLcgResponse& out) +bool JsonAPI::ParseAssignLcgJsonResponse(const cJSON* root, JsonAPI::AssignLcgResponse& out) { (void)code; diff --git a/src/util/DomainUtils.cpp b/src/util/DomainUtils.cpp new file mode 100644 index 00000000..cc57e720 --- /dev/null +++ b/src/util/DomainUtils.cpp @@ -0,0 +1,37 @@ +#include "util/DomainUtils.h" + +std::string_view OpenShock::DomainUtils::GetDomainFromUrl(std::string_view url) { + if (url.empty()) { + return {}; + } + + // Remove the protocol eg. "https://api.example.com:443/path" -> "api.example.com:443/path" + auto seperator = url.find("://"); + if (seperator != std::string_view::npos) { + url.substr(seperator + 3); + } + + // Remove the path eg. "api.example.com:443/path" -> "api.example.com:443" + seperator = url.find('/'); + if (seperator != std::string_view::npos) { + url = url.substr(0, seperator); + } + + // Remove the port eg. "api.example.com:443" -> "api.example.com" + seperator = url.rfind(':'); + if (seperator != std::string_view::npos) { + url = url.substr(0, seperator); + } + + // Remove all subdomains eg. "api.example.com" -> "example.com" + seperator = url.rfind('.'); + if (seperator == std::string_view::npos) { + return url; // E.g. "localhost" + } + seperator = url.rfind('.', seperator - 1); + if (seperator != std::string_view::npos) { + url = url.substr(seperator + 1); + } + + return url; +} From 9604c39c51b6bd12fa627e4b35089d47ad02f4d1 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Thu, 4 Dec 2025 03:50:29 +0100 Subject: [PATCH 06/28] More work --- include/OtaUpdateManager.h | 7 +-- include/http/HTTPClient.h | 16 +----- include/http/Response.h | 14 +++++ include/http/ResponseResult.h | 13 +++++ src/GatewayConnectionManager.cpp | 12 ++--- src/OtaUpdateManager.cpp | 87 +++++++++++++++++++------------- src/http/HTTPClient.cpp | 6 +-- src/serialization/JsonAPI.cpp | 10 ---- 8 files changed, 94 insertions(+), 71 deletions(-) create mode 100644 include/http/Response.h create mode 100644 include/http/ResponseResult.h diff --git a/include/OtaUpdateManager.h b/include/OtaUpdateManager.h index 2d319488..606ce120 100644 --- a/include/OtaUpdateManager.h +++ b/include/OtaUpdateManager.h @@ -1,6 +1,7 @@ #pragma once #include "FirmwareBootType.h" +#include "http/HTTPClient.h" #include "OtaUpdateChannel.h" #include "SemVer.h" @@ -19,9 +20,9 @@ namespace OpenShock::OtaUpdateManager { uint8_t filesystemBinaryHash[32]; }; - bool TryGetFirmwareVersion(OtaUpdateChannel channel, OpenShock::SemVer& version); - bool TryGetFirmwareBoards(const OpenShock::SemVer& version, std::vector& boards); - bool TryGetFirmwareRelease(const OpenShock::SemVer& version, FirmwareRelease& release); + bool TryGetFirmwareVersion(HTTP::HTTPClient& client, OtaUpdateChannel channel, OpenShock::SemVer& version); + bool TryGetFirmwareBoards(HTTP::HTTPClient& client, const OpenShock::SemVer& version, std::vector& boards); + bool TryGetFirmwareRelease(HTTP::HTTPClient& client, const OpenShock::SemVer& version, FirmwareRelease& release); bool TryStartFirmwareUpdate(const OpenShock::SemVer& version); diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h index 6a047c4b..832c3571 100644 --- a/include/http/HTTPClient.h +++ b/include/http/HTTPClient.h @@ -2,6 +2,7 @@ #include "Common.h" #include "RateLimiter.h" +#include "http/Response.h" #include @@ -13,21 +14,6 @@ #include namespace OpenShock::HTTP { - enum class DownloadResult : uint8_t { - Closed, // Connection closed - Success, // Request completed successfully - TimedOut, // Request timed out - ParseFailed, // Request completed, but JSON parsing failed - Cancelled, // Request was cancelled - }; - - template - struct [[nodiscard]] Response { - DownloadResult result; - esp_err_t error; - T data; - }; - template using JsonParser = std::function; using GotContentLengthCallback = std::function; diff --git a/include/http/Response.h b/include/http/Response.h new file mode 100644 index 00000000..88abf656 --- /dev/null +++ b/include/http/Response.h @@ -0,0 +1,14 @@ +#pragma once + +#include "http/ResponseResult.h" + +#include + +namespace OpenShock::HTTP { + template + struct [[nodiscard]] Response { + ResponseResult result; + esp_err_t error; + T data; + }; +} // namespace OpenShock::HTTP diff --git a/include/http/ResponseResult.h b/include/http/ResponseResult.h new file mode 100644 index 00000000..ad31cd41 --- /dev/null +++ b/include/http/ResponseResult.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace OpenShock::HTTP { + enum class ResponseResult : uint8_t { + Closed, // Connection closed + Success, // Request completed successfully + TimedOut, // Request timed out + ParseFailed, // Request completed, but JSON parsing failed + Cancelled, // Request was cancelled + }; +} // namespace OpenShock::HTTP diff --git a/src/GatewayConnectionManager.cpp b/src/GatewayConnectionManager.cpp index c09d8bfb..5289372d 100644 --- a/src/GatewayConnectionManager.cpp +++ b/src/GatewayConnectionManager.cpp @@ -112,11 +112,11 @@ AccountLinkResultCode GatewayConnectionManager::Link(std::string_view linkCode) return AccountLinkResultCode::InvalidCode; } - if (response.result == HTTP::RequestResult::RateLimited) { + if (response.result == HTTP::ResponseResult::RateLimited) { OS_LOGW(TAG, "Account Link request got ratelimited"); return AccountLinkResultCode::RateLimited; } - if (response.result != HTTP::RequestResult::Success) { + if (response.result != HTTP::ResponseResult::Success) { OS_LOGE(TAG, "Error while getting auth token: %s %d", response.ResultToString(), response.code); return AccountLinkResultCode::InternalError; @@ -187,10 +187,10 @@ bool FetchHubInfo(const char* authToken) return false; } - if (response.result == HTTP::RequestResult::RateLimited) { + if (response.result == HTTP::ResponseResult::RateLimited) { return false; // Just return false, don't spam the console with errors } - if (response.result != HTTP::RequestResult::Success) { + if (response.result != HTTP::ResponseResult::Success) { OS_LOGE(TAG, "Error while fetching hub info: %s %d", response.ResultToString(), response.code); return false; } @@ -252,10 +252,10 @@ bool StartConnectingToLCG() return false; } - if (response.result == HTTP::RequestResult::RateLimited) { + if (response.result == HTTP::ResponseResult::RateLimited) { return false; // Just return false, don't spam the console with errors } - if (response.result != HTTP::RequestResult::Success) { + if (response.result != HTTP::ResponseResult::Success) { OS_LOGE(TAG, "Error while fetching LCG endpoint: %s %d", response.ResultToString(), response.code); return false; } diff --git a/src/OtaUpdateManager.cpp b/src/OtaUpdateManager.cpp index 11039644..36da3558 100644 --- a/src/OtaUpdateManager.cpp +++ b/src/OtaUpdateManager.cpp @@ -8,7 +8,7 @@ const char* const TAG = "OtaUpdateManager"; #include "Core.h" #include "GatewayConnectionManager.h" #include "Hashing.h" -#include "http/HTTPRequestManager.h" +#include "http/HTTPClient.h" #include "Logging.h" #include "SemVer.h" #include "serialization/WSGateway.h" @@ -324,7 +324,7 @@ static void otaum_updatetask(void* arg) OS_LOGD(TAG, "Checking for updates"); // Fetch current version. - if (!OtaUpdateManager::TryGetFirmwareVersion(config.updateChannel, version)) { + if (!OtaUpdateManager::TryGetFirmwareVersion(client, config.updateChannel, version)) { OS_LOGE(TAG, "Failed to fetch firmware version"); continue; } @@ -361,7 +361,7 @@ static void otaum_updatetask(void* arg) // Fetch current release. OtaUpdateManager::FirmwareRelease release; - if (!OtaUpdateManager::TryGetFirmwareRelease(version, release)) { + if (!OtaUpdateManager::TryGetFirmwareRelease(client, version, release)) { OS_LOGE(TAG, "Failed to fetch firmware release"); // TODO: Send error message to server _sendFailureMessage("Failed to fetch firmware release"sv); continue; @@ -422,16 +422,23 @@ static void otaum_updatetask(void* arg) esp_restart(); } -static bool _tryGetStringList(const char* url, std::vector& list) +static bool _tryGetStringList(HTTP::HTTPClient& client, const char* url, std::vector& list) { - auto response = OpenShock::HTTP::GetString( - url, - { - {"Accept", "text/plain"} - }, - std::array {200, 304} - ); - if (response.result != OpenShock::HTTP::RequestResult::Success) { + ; + esp_err_t err = client.Get(url); + if (err != ESP_OK) { + OS_LOGE(TAG, "Failed to fetch list"); + return false; + } + + int statusCode = client.StatusCode(); + if (statusCode != 200 && statusCode != 304) { + OS_LOGE(TAG, "Failed to fetch list"); + return false; + } + + auto response = client.ReadResponseString(); + if (response.result != HTTP::ResponseResult::Success) { OS_LOGE(TAG, "Failed to fetch list: %s [%u] %s", response.ResultToString(), response.code, response.data.c_str()); return false; } @@ -523,7 +530,7 @@ bool OtaUpdateManager::Init() return true; } -bool OtaUpdateManager::TryGetFirmwareVersion(OtaUpdateChannel channel, OpenShock::SemVer& version) +bool OtaUpdateManager::TryGetFirmwareVersion(HTTP::HTTPClient& client, OtaUpdateChannel channel, OpenShock::SemVer& version) { const char* channelIndexUrl; switch (channel) { @@ -543,14 +550,20 @@ bool OtaUpdateManager::TryGetFirmwareVersion(OtaUpdateChannel channel, OpenShock OS_LOGD(TAG, "Fetching firmware version from %s", channelIndexUrl); - auto response = OpenShock::HTTP::GetString( - channelIndexUrl, - { - {"Accept", "text/plain"} - }, - std::array {200, 304} - ); - if (response.result != OpenShock::HTTP::RequestResult::Success) { + esp_err_t err = client.Get(channelIndexUrl); + if (err != ESP_OK) { + OS_LOGE(TAG, "Failed to fetch firmware version"); + return false; + } + + int statusCode = client.StatusCode(); + if (statusCode != 200 && statusCode != 304) { + OS_LOGE(TAG, "Failed to fetch firmware version"); + return false; + } + + auto response = client.ReadResponseString(); + if (response.result != HTTP::ResponseResult::Success) { OS_LOGE(TAG, "Failed to fetch firmware version: %s [%u] %s", response.ResultToString(), response.code, response.data.c_str()); return false; } @@ -563,7 +576,7 @@ bool OtaUpdateManager::TryGetFirmwareVersion(OtaUpdateChannel channel, OpenShock return true; } -bool OtaUpdateManager::TryGetFirmwareBoards(const OpenShock::SemVer& version, std::vector& boards) +bool OtaUpdateManager::TryGetFirmwareBoards(HTTP::HTTPClient& client, const OpenShock::SemVer& version, std::vector& boards) { std::string channelIndexUrl; if (!FormatToString(channelIndexUrl, OPENSHOCK_FW_CDN_BOARDS_INDEX_URL_FORMAT, version.toString().c_str())) { // TODO: This is abusing the SemVer::toString() method causing alot of string copies, fix this @@ -573,7 +586,7 @@ bool OtaUpdateManager::TryGetFirmwareBoards(const OpenShock::SemVer& version, st OS_LOGD(TAG, "Fetching firmware boards from %s", channelIndexUrl.c_str()); - if (!_tryGetStringList(channelIndexUrl.c_str(), boards)) { + if (!_tryGetStringList(client, channelIndexUrl.c_str(), boards)) { OS_LOGE(TAG, "Failed to fetch firmware boards"); return false; } @@ -591,7 +604,7 @@ static bool _tryParseIntoHash(std::string_view hash, uint8_t (&hashBytes)[32]) return true; } -bool OtaUpdateManager::TryGetFirmwareRelease(const OpenShock::SemVer& version, FirmwareRelease& release) +bool OtaUpdateManager::TryGetFirmwareRelease(HTTP::HTTPClient& client, const OpenShock::SemVer& version, FirmwareRelease& release) { auto versionStr = version.toString(); // TODO: This is abusing the SemVer::toString() method causing alot of string copies, fix this @@ -613,19 +626,25 @@ bool OtaUpdateManager::TryGetFirmwareRelease(const OpenShock::SemVer& version, F } // Fetch hashes. - auto sha256HashesResponse = OpenShock::HTTP::GetString( - sha256HashesUrl.c_str(), - { - {"Accept", "text/plain"} - }, - std::array {200, 304} - ); - if (sha256HashesResponse.result != OpenShock::HTTP::RequestResult::Success) { - OS_LOGE(TAG, "Failed to fetch hashes: %s [%u] %s", sha256HashesResponse.ResultToString(), sha256HashesResponse.code, sha256HashesResponse.data.c_str()); + esp_err_t err = client.Get(sha256HashesUrl.c_str()); + if (err != ESP_OK) { + OS_LOGE(TAG, "Failed to fetch hashes"); + return false; + } + + int statusCode = client.StatusCode(); + if (statusCode != 200 && statusCode != 304) { + OS_LOGE(TAG, "Failed to fetch hashes"); + return false; + } + + auto response = client.ReadResponseString(); + if (response.result != HTTP::ResponseResult::Success) { + OS_LOGE(TAG, "Failed to fetch hashes: %s [%u] %s", response.ResultToString(), response.code, response.data.c_str()); return false; } - auto hashesLines = OpenShock::StringSplitNewLines(sha256HashesResponse.data); + auto hashesLines = OpenShock::StringSplitNewLines(response.data); // Parse hashes. bool foundAppHash = false, foundFilesystemHash = false; diff --git a/src/http/HTTPClient.cpp b/src/http/HTTPClient.cpp index c6868aae..879b71c1 100644 --- a/src/http/HTTPClient.cpp +++ b/src/http/HTTPClient.cpp @@ -63,12 +63,12 @@ esp_err_t HTTP::HTTPClient::Get(const char* url) { HTTP::Response HTTP::HTTPClient::ReadResponseStream(DownloadCallback downloadCallback) { if (!m_connected) { - return {DownloadResult::Closed, ESP_FAIL, 0}; + return {ResponseResult::Closed, ESP_FAIL, 0}; } std::size_t nWritten; - return {DownloadResult::Success, ESP_OK, nWritten}; + return {ResponseResult::Success, ESP_OK, nWritten}; } HTTP::Response HTTP::HTTPClient::ReadResponseString() { @@ -83,7 +83,7 @@ HTTP::Response HTTP::HTTPClient::ReadResponseString() { }; auto response = ReadResponseStream(writer); - if (response.result != DownloadResult::Success) { + if (response.result != ResponseResult::Success) { return {response.result, response.error, {}}; } diff --git a/src/serialization/JsonAPI.cpp b/src/serialization/JsonAPI.cpp index 7de08563..215a4cac 100644 --- a/src/serialization/JsonAPI.cpp +++ b/src/serialization/JsonAPI.cpp @@ -10,8 +10,6 @@ using namespace OpenShock::Serialization; bool JsonAPI::ParseLcgInstanceDetailsJsonResponse(const cJSON* root, JsonAPI::LcgInstanceDetailsResponse& out) { - (void)code; - if (cJSON_IsObject(root) == 0) { ESP_LOGJSONE("not an object", root); return false; @@ -59,8 +57,6 @@ bool JsonAPI::ParseLcgInstanceDetailsJsonResponse(const cJSON* root, JsonAPI::Lc } bool JsonAPI::ParseBackendVersionJsonResponse(const cJSON* root, JsonAPI::BackendVersionResponse& out) { - (void)code; - if (cJSON_IsObject(root) == 0) { ESP_LOGJSONE("not an object", root); return false; @@ -101,8 +97,6 @@ bool JsonAPI::ParseBackendVersionJsonResponse(const cJSON* root, JsonAPI::Backen bool JsonAPI::ParseAccountLinkJsonResponse(const cJSON* root, JsonAPI::AccountLinkResponse& out) { - (void)code; - if (cJSON_IsObject(root) == 0) { ESP_LOGJSONE("not an object", root); return false; @@ -122,8 +116,6 @@ bool JsonAPI::ParseAccountLinkJsonResponse(const cJSON* root, JsonAPI::AccountLi } bool JsonAPI::ParseHubInfoJsonResponse(const cJSON* root, JsonAPI::HubInfoResponse& out) { - (void)code; - if (cJSON_IsObject(root) == 0) { ESP_LOGJSONE("not an object", root); return false; @@ -215,8 +207,6 @@ bool JsonAPI::ParseHubInfoJsonResponse(const cJSON* root, JsonAPI::HubInfoRespon } bool JsonAPI::ParseAssignLcgJsonResponse(const cJSON* root, JsonAPI::AssignLcgResponse& out) { - (void)code; - if (cJSON_IsObject(root) == 0) { ESP_LOGJSONE("not an object", root); return false; From d7774cca5cdd2fdcf464069415484c350b6f7473 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Thu, 4 Dec 2025 15:18:50 +0100 Subject: [PATCH 07/28] More work on custom http client --- include/http/DownloadCallback.h | 8 ++ include/http/GotContentLengthCallback.h | 8 ++ include/http/HTTPClient.h | 69 ++--------- include/http/HTTPClientState.h | 55 +++++++++ include/http/HTTPResponse.h | 22 ++++ include/http/JsonParserFn.h | 11 ++ include/http/Response.h | 14 --- include/http/ResponseResult.h | 13 -- src/http/HTTPClient.cpp | 124 +++---------------- src/http/HTTPClientState.cpp | 154 ++++++++++++++++++++++++ src/util/IPAddressUtils.cpp | 2 +- 11 files changed, 283 insertions(+), 197 deletions(-) create mode 100644 include/http/DownloadCallback.h create mode 100644 include/http/GotContentLengthCallback.h create mode 100644 include/http/HTTPClientState.h create mode 100644 include/http/HTTPResponse.h create mode 100644 include/http/JsonParserFn.h delete mode 100644 include/http/Response.h delete mode 100644 include/http/ResponseResult.h create mode 100644 src/http/HTTPClientState.cpp diff --git a/include/http/DownloadCallback.h b/include/http/DownloadCallback.h new file mode 100644 index 00000000..7066d189 --- /dev/null +++ b/include/http/DownloadCallback.h @@ -0,0 +1,8 @@ +#pragma once + +#include +#include + +namespace OpenShock::HTTP { + using DownloadCallback = std::function; +} diff --git a/include/http/GotContentLengthCallback.h b/include/http/GotContentLengthCallback.h new file mode 100644 index 00000000..ab4ac6d5 --- /dev/null +++ b/include/http/GotContentLengthCallback.h @@ -0,0 +1,8 @@ +#pragma once + +#include +#include + +namespace OpenShock::HTTP { + using GotContentLengthCallback = std::function; +} diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h index 832c3571..91c23974 100644 --- a/include/http/HTTPClient.h +++ b/include/http/HTTPClient.h @@ -1,85 +1,36 @@ #pragma once #include "Common.h" -#include "RateLimiter.h" -#include "http/Response.h" - -#include +#include "http/HTTPClientState.h" +#include "http/HTTPResponse.h" #include -#include -#include #include #include +#include namespace OpenShock::HTTP { - template - using JsonParser = std::function; - using GotContentLengthCallback = std::function; - using DownloadCallback = std::function; - class HTTPClient { DISABLE_COPY(HTTPClient); DISABLE_MOVE(HTTPClient); public: - HTTPClient(uint32_t timeoutMs = 10'000, const char* useragent = OpenShock::Constants::FW_USERAGENT); - ~HTTPClient(); - - inline esp_err_t SetHeader(const char* key, const char* value) { - return esp_http_client_set_header(m_handle, key, value); + HTTPClient(uint32_t timeoutMs = 10'000) + : m_state(std::make_shared()) + { } - esp_err_t SetHeaders(const std::map& headers); - - esp_err_t Get(const char* url); - inline int ResponseLength() const { - return m_responseLength; - } - inline int StatusCode() const { - return m_statusCode; + inline esp_err_t SetHeader(const char* key, const char* value) { + return m_state->SetHeader(key, value); } - Response ReadResponseStream(DownloadCallback downloadCallback); - Response ReadResponseString(); - template - inline Response ReadResponseJSON(JsonParser jsonParser) - { - auto response = ReadResponseString(); - if (response.result != DownloadResult::Success) { - return {response.result, response.error, {}}; - } - - cJSON* json = cJSON_ParseWithLength(response.data.c_str(), response.data.length()); - if (json == nullptr) { - return {DownloadResult::ParseFailed, ESP_OK, {}}; - } - - T data; - if (!jsonParser(json, data)) { - return {DownloadResult::ParseFailed, ESP_OK, {}}; - } - - cJSON_Delete(json); - - return {response.result, ESP_OK, std::move(data)}; - } + HTTPResponse Get(const char* url); esp_err_t Close(); private: esp_err_t Start(esp_http_client_method_t method, const char* url, int writeLen); - static esp_err_t EventHandler(esp_http_client_event_t* evt); - esp_err_t HandleHeader(std::string_view key, std::string_view value); - - esp_http_client_handle_t m_handle; - std::shared_ptr m_ratelimiter; - bool m_connected; - int m_responseLength; - int m_statusCode; - - GotContentLengthCallback m_cbGotContentLength; - DownloadCallback m_cbDownload; + std::shared_ptr m_state; }; } // namespace OpenShock::HTTP diff --git a/include/http/HTTPClientState.h b/include/http/HTTPClientState.h new file mode 100644 index 00000000..86600506 --- /dev/null +++ b/include/http/HTTPClientState.h @@ -0,0 +1,55 @@ +#pragma once + +#include "Common.h" +#include "http/DownloadCallback.h" + +#include + +#include +#include +#include + +namespace OpenShock::HTTP { + class HTTPClientState { + DISABLE_COPY(HTTPClientState); + DISABLE_MOVE(HTTPClientState); + public: + HTTPClientState(uint32_t timeoutMs); + ~HTTPClientState(); + + inline esp_err_t SetHeader(const char* key, const char* value) { + return esp_http_client_set_header(m_handle, key, value); + } + + struct [[nodiscard]] StartRequestResult { + esp_err_t err; + bool isChunked; + uint32_t nAvailable; + }; + + StartRequestResult StartRequest(esp_http_client_method_t method, const char* url, int writeLen); + + enum class ReadResultCode { + Success = 0, + ConnectionClosed = 1, + NetworkError = 2, + SizeLimitExceeded = 3, + Aborted = 4, + }; + + struct [[nodiscard]] ReadResult { + ReadResultCode result; + std::size_t nRead; + }; + + // High-throughput streaming logic + ReadResult ReadStreamImpl(DownloadCallback cb); + private: + static esp_err_t EventHandler(esp_http_client_event_t* evt); + esp_err_t EventHeaderHandler(std::string_view key, std::string_view value); + + esp_http_client_handle_t m_handle; + bool m_reading; + std::vector> m_headers; + }; +} // namespace OpenShock::HTTP diff --git a/include/http/HTTPResponse.h b/include/http/HTTPResponse.h new file mode 100644 index 00000000..10bcbac1 --- /dev/null +++ b/include/http/HTTPResponse.h @@ -0,0 +1,22 @@ +#pragma once + +#include "Common.h" +#include "http/HTTPClientState.h" + +#include + +namespace OpenShock::HTTP { + class HTTPClient; + class HTTPResponse { + DISABLE_COPY(HTTPResponse); + DISABLE_MOVE(HTTPResponse); + + friend HTTPClient; + + HTTPResponse(std::shared_ptr state); + public: + + private: + std::weak_ptr m_state; + }; +} // namespace OpenShock::HTTP diff --git a/include/http/JsonParserFn.h b/include/http/JsonParserFn.h new file mode 100644 index 00000000..ba9a10f1 --- /dev/null +++ b/include/http/JsonParserFn.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +#include +#include + +namespace OpenShock::HTTP { + template + using JsonParserFn = std::function; +} diff --git a/include/http/Response.h b/include/http/Response.h deleted file mode 100644 index 88abf656..00000000 --- a/include/http/Response.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "http/ResponseResult.h" - -#include - -namespace OpenShock::HTTP { - template - struct [[nodiscard]] Response { - ResponseResult result; - esp_err_t error; - T data; - }; -} // namespace OpenShock::HTTP diff --git a/include/http/ResponseResult.h b/include/http/ResponseResult.h deleted file mode 100644 index ad31cd41..00000000 --- a/include/http/ResponseResult.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include - -namespace OpenShock::HTTP { - enum class ResponseResult : uint8_t { - Closed, // Connection closed - Success, // Request completed successfully - TimedOut, // Request timed out - ParseFailed, // Request completed, but JSON parsing failed - Cancelled, // Request was cancelled - }; -} // namespace OpenShock::HTTP diff --git a/src/http/HTTPClient.cpp b/src/http/HTTPClient.cpp index 879b71c1..11c243db 100644 --- a/src/http/HTTPClient.cpp +++ b/src/http/HTTPClient.cpp @@ -2,55 +2,19 @@ const char* const TAG = "HTTPClient"; +#include "Convert.h" #include "http/RateLimiters.h" #include "Logging.h" #include "util/DomainUtils.h" -const std::size_t HTTP_BUFFER_SIZE = 4096LLU; -const int HTTP_DOWNLOAD_SIZE_LIMIT = 200 * 1024 * 1024; // 200 MB - using namespace OpenShock; -HTTP::HTTPClient::HTTPClient(uint32_t timeoutMs, const char* useragent) -{ - esp_http_client_config_t cfg; - memset(&cfg, 0, sizeof(cfg)); - - cfg.timeout_ms = static_cast(std::min(timeoutMs, INT32_MAX)); - cfg.disable_auto_redirect = true; - cfg.event_handler = HTTPClient::EventHandler; - cfg.transport_type = HTTP_TRANSPORT_OVER_SSL; - cfg.user_data = reinterpret_cast(this); - cfg.is_async = false; - cfg.use_global_ca_store = true; - #warning This still uses SSL, upgrade to TLS! (latest ESP-IDF) - - m_handle = esp_http_client_init(&cfg); -} - -HTTP::HTTPClient::~HTTPClient() -{ - esp_http_client_cleanup(m_handle); -} - -esp_err_t HTTP::HTTPClient::SetHeaders(const std::map& headers) { - esp_err_t err; - - for (auto& header : headers) { - err = SetHeader(header.first.c_str(), header.second.c_str()); - - if (err != ESP_OK) return err; +HTTP::HTTPResponse HTTP::HTTPClient::Get(const char* url) { + esp_err_t err = m_state->StartRequest(HTTP_METHOD_GET, url, 0); + if (err != ESP_OK) { + // TODO: Do something } - return ESP_OK; -} - -esp_err_t HTTP::HTTPClient::Get(const char* url) { - esp_err_t err; - - err = Start(HTTP_METHOD_GET, url, 0); - if (err != ESP_OK) return err; - m_responseLength = esp_http_client_fetch_headers(m_handle); if (m_responseLength == ESP_FAIL) { return ESP_FAIL; @@ -61,35 +25,6 @@ esp_err_t HTTP::HTTPClient::Get(const char* url) { return ESP_OK; } -HTTP::Response HTTP::HTTPClient::ReadResponseStream(DownloadCallback downloadCallback) { - if (!m_connected) { - return {ResponseResult::Closed, ESP_FAIL, 0}; - } - - std::size_t nWritten; - - return {ResponseResult::Success, ESP_OK, nWritten}; -} - -HTTP::Response HTTP::HTTPClient::ReadResponseString() { - std::string result; - if (m_responseLength > 0) { - result.reserve(m_responseLength); - } - - auto writer = [&result](std::size_t offset, const uint8_t* data, std::size_t len) { - result.append(reinterpret_cast(data), len); - return true; - }; - - auto response = ReadResponseStream(writer); - if (response.result != ResponseResult::Success) { - return {response.result, response.error, {}}; - } - - return {response.result, response.error, result}; -} - esp_err_t HTTP::HTTPClient::Close() { return esp_http_client_close(m_handle); } @@ -107,54 +42,23 @@ esp_err_t HTTP::HTTPClient::Start(esp_http_client_method_t method, const char* u return ESP_FAIL; } - err = esp_http_client_set_url(m_handle, url); - if (err != ESP_OK) return err; - - err = esp_http_client_set_method(m_handle, HTTP_METHOD_GET); - if (err != ESP_OK) return err; + return m_state return esp_http_client_open(m_handle, writeLen); } esp_err_t HTTP::HTTPClient::EventHandler(esp_http_client_event_t* evt) { - HTTPClient* client = reinterpret_cast(evt->user_data); - - switch (evt->event_id) - { - case HTTP_EVENT_ERROR: - OS_LOGE(TAG, "Got error event"); - break; - case HTTP_EVENT_ON_CONNECTED: - client->m_connected = true; - OS_LOGI(TAG, "Got connected event"); - break; - case HTTP_EVENT_HEADERS_SENT: - OS_LOGI(TAG, "Got headers_sent event"); - break; - case HTTP_EVENT_ON_HEADER: - OS_LOGI(TAG, "Got header_received event: %s - %s", evt->header_key, evt->header_value); - return client->HandleHeader(evt->header_key, evt->header_value); - case HTTP_EVENT_ON_DATA: - OS_LOGI(TAG, "Got on_data event"); - break; - case HTTP_EVENT_ON_FINISH: - OS_LOGI(TAG, "Got on_finish event"); - break; - case HTTP_EVENT_DISCONNECTED: - client->m_connected = false; - OS_LOGI(TAG, "Got disconnected event"); - break; - default: - OS_LOGE(TAG, "Got unknown event"); - break; - } - - return ESP_OK; } esp_err_t HTTP::HTTPClient::HandleHeader(std::string_view key, std::string_view value) { - if (key == "Retry-After") { - // TODO: Set block on m_ratelimiter + if (key == "Retry-After" && m_ratelimiter != nullptr) { + uint32_t seconds; + if (!Convert::ToUint32(value, seconds) || seconds <= 0) { + seconds = 15; + } + + OS_LOGI(TAG, "Retry-After: %d seconds, applying delay to rate limiter", seconds); + m_ratelimiter->blockFor(seconds * 1000U); } return ESP_OK; diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp new file mode 100644 index 00000000..430ce6b2 --- /dev/null +++ b/src/http/HTTPClientState.cpp @@ -0,0 +1,154 @@ +#include "http/HTTPClientState.h" + +const char* const TAG = "HTTPClientState"; + +#include "Common.h" +#include "Logging.h" + +#include + +static const std::size_t HTTP_BUFFER_SIZE = 4096LLU; +static const std::size_t HTTP_DOWNLOAD_SIZE_LIMIT = 200 * 1024 * 1024; // 200 MB + +using namespace OpenShock; + +HTTP::HTTPClientState::HTTPClientState(uint32_t timeoutMs) + : m_handle(nullptr) + , m_reading(false) +{ + esp_http_client_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + + cfg.user_agent = OpenShock::Constants::FW_USERAGENT; + cfg.timeout_ms = static_cast(std::min(timeoutMs, INT32_MAX)); + cfg.disable_auto_redirect = true; + cfg.event_handler = HTTPClientState::EventHandler; + cfg.transport_type = HTTP_TRANSPORT_OVER_SSL; + cfg.user_data = reinterpret_cast(this); + cfg.is_async = false; + cfg.use_global_ca_store = true; + #warning This still uses SSL, upgrade to TLS! (latest ESP-IDF) + + m_handle = esp_http_client_init(&cfg); +} + +HTTP::HTTPClientState::~HTTPClientState() +{ + if (m_handle != nullptr) { + esp_http_client_cleanup(m_handle); + m_handle = nullptr; + } +} + +HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClientState::StartRequest(esp_http_client_method_t method, const char* url, int writeLen) +{ + esp_err_t err; + + err = esp_http_client_set_url(m_handle, url); + if (err != ESP_OK) return {err, false, 0}; + + err = esp_http_client_set_method(m_handle, method); + if (err != ESP_OK) return {err, false, 0}; + + err = esp_http_client_open(m_handle, writeLen); + if (err != ESP_OK) return {err, false, 0}; + + int responseLength = esp_http_client_fetch_headers(m_handle); + if (responseLength == ESP_FAIL) return {err, false, 0}; + + bool isChunked = false; + if (responseLength == 0) { + isChunked = esp_http_client_is_chunked_response(m_handle); + } + + return {ESP_OK, isChunked, static_cast(responseLength)}; +} + +HTTP::HTTPClientState::ReadResult HTTP::HTTPClientState::ReadStreamImpl(DownloadCallback cb) +{ + if (m_handle == nullptr || !m_reading) { + m_reading = false; + return {ReadResultCode::ConnectionClosed, 0}; + } + + std::size_t totalWritten = 0; + uint8_t buffer[HTTP_BUFFER_SIZE]; + + while (true) { + if (totalWritten >= HTTP_DOWNLOAD_SIZE_LIMIT) { + m_reading = false; + return {ReadResultCode::SizeLimitExceeded, totalWritten}; + } + + std::size_t remaining = HTTP_DOWNLOAD_SIZE_LIMIT - totalWritten; + int toRead = static_cast(std::min(HTTP_BUFFER_SIZE, remaining)); + + int n = esp_http_client_read( + m_handle, + reinterpret_cast(buffer), + toRead + ); + + if (n < 0) { + m_reading = false; + return {ReadResultCode::NetworkError, totalWritten}; + } + + if (n == 0) { + // EOF + break; + } + + std::size_t chunkLen = static_cast(n); + if (!cb(totalWritten, buffer, chunkLen)) { + m_reading = false; + return {ReadResultCode::Aborted, totalWritten}; + } + + totalWritten += chunkLen; + } + + m_reading = false; + return {ReadResultCode::Success, totalWritten}; +} + +esp_err_t HTTP::HTTPClientState::EventHandler(esp_http_client_event_t* evt) +{ + HTTPClientState* client = reinterpret_cast(evt->user_data); + + switch (evt->event_id) + { + case HTTP_EVENT_ERROR: + OS_LOGE(TAG, "Got error event"); + break; + case HTTP_EVENT_ON_CONNECTED: + client->m_connected = true; + OS_LOGI(TAG, "Got connected event"); + break; + case HTTP_EVENT_HEADERS_SENT: + OS_LOGI(TAG, "Got headers_sent event"); + break; + case HTTP_EVENT_ON_HEADER: + return client->EventHeaderHandler(evt->header_key, evt->header_value); + case HTTP_EVENT_ON_DATA: + OS_LOGI(TAG, "Got on_data event"); + break; + case HTTP_EVENT_ON_FINISH: + OS_LOGI(TAG, "Got on_finish event"); + break; + case HTTP_EVENT_DISCONNECTED: + client->m_connected = false; + OS_LOGI(TAG, "Got disconnected event"); + break; + default: + OS_LOGE(TAG, "Got unknown event"); + break; + } + + return ESP_OK; +} + +esp_err_t HTTP::HTTPClientState::EventHeaderHandler(std::string_view key, std::string_view value) +{ + OS_LOGI(TAG, "Got header_received event: %s - %s", evt->header_key, evt->header_value); +} diff --git a/src/util/IPAddressUtils.cpp b/src/util/IPAddressUtils.cpp index 15661641..b12c38cc 100644 --- a/src/util/IPAddressUtils.cpp +++ b/src/util/IPAddressUtils.cpp @@ -15,7 +15,7 @@ bool OpenShock::IPV4AddressFromStringView(IPAddress& ip, std::string_view sv) { return false; // Must have 4 octets } - std::uint8_t octets[4]; + uint8_t octets[4]; if (!Convert::ToUint8(parts[0], octets[0]) || !Convert::ToUint8(parts[1], octets[1]) || !Convert::ToUint8(parts[2], octets[2]) || !Convert::ToUint8(parts[3], octets[3])) { return false; } From 0d966e3a533c9aaa3cc439b63a665d46a7fdd0c7 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 03:44:32 +0100 Subject: [PATCH 08/28] More wooooorkkkkk AAAAAAAAAAAAAAAAAAA --- include/OtaUpdateManager.h | 6 +- include/http/HTTPClient.h | 29 +- include/http/HTTPClientState.h | 56 +++- include/http/HTTPError.h | 44 +++ include/http/HTTPResponse.h | 57 +++- include/http/JsonAPI.h | 6 +- include/http/JsonResponse.h | 60 ++++ include/http/ReadResult.h | 17 + src/GatewayConnectionManager.cpp | 83 +++-- src/OtaUpdateManager.cpp | 64 ++-- src/http/HTTPClient.cpp | 56 +--- src/http/HTTPClientState.cpp | 84 +++-- src/http/HTTPRequestManager.cpp | 363 ---------------------- src/http/JsonAPI.cpp | 46 +-- src/serial/SerialInputHandler.cpp | 1 - src/serial/command_handlers/authtoken.cpp | 19 +- src/serial/command_handlers/domain.cpp | 1 - src/util/ParitionUtils.cpp | 1 - 18 files changed, 418 insertions(+), 575 deletions(-) create mode 100644 include/http/HTTPError.h create mode 100644 include/http/JsonResponse.h create mode 100644 include/http/ReadResult.h delete mode 100644 src/http/HTTPRequestManager.cpp diff --git a/include/OtaUpdateManager.h b/include/OtaUpdateManager.h index 606ce120..c833c8e9 100644 --- a/include/OtaUpdateManager.h +++ b/include/OtaUpdateManager.h @@ -20,9 +20,9 @@ namespace OpenShock::OtaUpdateManager { uint8_t filesystemBinaryHash[32]; }; - bool TryGetFirmwareVersion(HTTP::HTTPClient& client, OtaUpdateChannel channel, OpenShock::SemVer& version); - bool TryGetFirmwareBoards(HTTP::HTTPClient& client, const OpenShock::SemVer& version, std::vector& boards); - bool TryGetFirmwareRelease(HTTP::HTTPClient& client, const OpenShock::SemVer& version, FirmwareRelease& release); + bool TryGetFirmwareVersion(OtaUpdateChannel channel, OpenShock::SemVer& version); + bool TryGetFirmwareBoards(const OpenShock::SemVer& version, std::vector& boards); + bool TryGetFirmwareRelease(const OpenShock::SemVer& version, FirmwareRelease& release); bool TryStartFirmwareUpdate(const OpenShock::SemVer& version); diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h index 91c23974..952cfec2 100644 --- a/include/http/HTTPClient.h +++ b/include/http/HTTPClient.h @@ -3,12 +3,11 @@ #include "Common.h" #include "http/HTTPClientState.h" #include "http/HTTPResponse.h" +#include "http/JsonResponse.h" #include #include -#include -#include namespace OpenShock::HTTP { class HTTPClient { @@ -17,7 +16,7 @@ namespace OpenShock::HTTP { public: HTTPClient(uint32_t timeoutMs = 10'000) - : m_state(std::make_shared()) + : m_state(std::make_shared(timeoutMs)) { } @@ -25,11 +24,29 @@ namespace OpenShock::HTTP { return m_state->SetHeader(key, value); } - HTTPResponse Get(const char* url); + inline HTTPResponse Get(const char* url) { + auto response = GetInternal(url); + if (response.error != HTTPError::None) return response.error; - esp_err_t Close(); + return HTTP::HTTPResponse(m_state, response.data.statusCode, response.data.contentLength); + } + template + inline JsonResponse GetJson(const char* url, JsonParserFn jsonParser) { + auto response = GetInternal(url); + if (response.error != HTTPError::None) return response.error; + + return HTTP::JsonResponse(m_state, jsonParser, response.data.statusCode, response.data.contentLength); + } + + inline esp_err_t Close() { + return m_state->Close(); + } private: - esp_err_t Start(esp_http_client_method_t method, const char* url, int writeLen); + struct InternalResult { + HTTPError error; + HTTPClientState::StartRequestResult data; + }; + InternalResult GetInternal(const char* url); std::shared_ptr m_state; }; diff --git a/include/http/HTTPClientState.h b/include/http/HTTPClientState.h index 86600506..9aca55e3 100644 --- a/include/http/HTTPClientState.h +++ b/include/http/HTTPClientState.h @@ -2,11 +2,17 @@ #include "Common.h" #include "http/DownloadCallback.h" +#include "http/HTTPError.h" +#include "http/JsonParserFn.h" +#include "http/ReadResult.h" + +#include #include #include #include +#include #include namespace OpenShock::HTTP { @@ -22,31 +28,47 @@ namespace OpenShock::HTTP { } struct [[nodiscard]] StartRequestResult { - esp_err_t err; + int statusCode; bool isChunked; - uint32_t nAvailable; + uint32_t contentLength; }; - StartRequestResult StartRequest(esp_http_client_method_t method, const char* url, int writeLen); + std::variant StartRequest(esp_http_client_method_t method, const char* url, int writeLen); - enum class ReadResultCode { - Success = 0, - ConnectionClosed = 1, - NetworkError = 2, - SizeLimitExceeded = 3, - Aborted = 4, - }; + // High-throughput streaming logic + ReadResult ReadStreamImpl(DownloadCallback cb); - struct [[nodiscard]] ReadResult { - ReadResultCode result; - std::size_t nRead; - }; + ReadResult ReadStringImpl(uint32_t reserve); - // High-throughput streaming logic - ReadResult ReadStreamImpl(DownloadCallback cb); + template + inline ReadResult ReadJsonImpl(uint32_t reserve, JsonParserFn jsonParser) + { + auto response = ReadStringImpl(reserve); + if (response.error != HTTPError::None) { + return response.error; + } + + cJSON* json = cJSON_ParseWithLength(response.data.c_str(), response.data.length()); + if (json == nullptr) { + return HTTPError::ParseFailed; + } + + T data; + if (!jsonParser(json, data)) { + return HTTPError::ParseFailed; + } + + cJSON_Delete(json); + + return data; + } + + inline esp_err_t Close() { + return esp_http_client_close(m_handle); + } private: static esp_err_t EventHandler(esp_http_client_event_t* evt); - esp_err_t EventHeaderHandler(std::string_view key, std::string_view value); + esp_err_t EventHeaderHandler(std::string key, std::string value); esp_http_client_handle_t m_handle; bool m_reading; diff --git a/include/http/HTTPError.h b/include/http/HTTPError.h new file mode 100644 index 00000000..808557bb --- /dev/null +++ b/include/http/HTTPError.h @@ -0,0 +1,44 @@ +#pragma once + +namespace OpenShock::HTTP { + enum class HTTPError { + None, + InternalError, + RateLimited, + InvalidUrl, + InvalidHttpMethod, + NetworkError, + ConnectionClosed, + SizeLimitExceeded, + Aborted, + ParseFailed + }; + + inline const char* HTTPErrorToString(HTTPError error) { + switch (error) + { + case HTTPError::None: + return ""; + case HTTPError::InternalError: + return ""; + case HTTPError::RateLimited: + return ""; + case HTTPError::InvalidUrl: + return ""; + case HTTPError::InvalidHttpMethod: + return ""; + case HTTPError::NetworkError: + return ""; + case HTTPError::ConnectionClosed: + return ""; + case HTTPError::SizeLimitExceeded: + return ""; + case HTTPError::Aborted: + return ""; + case HTTPError::ParseFailed: + return ""; + default: + return ""; + } + } +} // namespace OpenShock::HTTP diff --git a/include/http/HTTPResponse.h b/include/http/HTTPResponse.h index 10bcbac1..60843f5f 100644 --- a/include/http/HTTPResponse.h +++ b/include/http/HTTPResponse.h @@ -1,22 +1,73 @@ #pragma once #include "Common.h" +#include "http/DownloadCallback.h" #include "http/HTTPClientState.h" +#include "http/JsonParserFn.h" +#include "http/ReadResult.h" +#include + +#include #include +#include namespace OpenShock::HTTP { class HTTPClient; - class HTTPResponse { + class [[nodiscard]] HTTPResponse { + DISABLE_DEFAULT(HTTPResponse); DISABLE_COPY(HTTPResponse); DISABLE_MOVE(HTTPResponse); - friend HTTPClient; + friend class HTTPClient; - HTTPResponse(std::shared_ptr state); + HTTPResponse(std::shared_ptr state, int statusCode, uint32_t contentLength) + : m_state(state) + , m_error(HTTPError::None) + , m_statusCode(statusCode) + , m_contentLength(contentLength) + { + } public: + HTTPResponse(HTTPError error) + : m_state() + , m_error(error) + , m_statusCode(0) + , m_contentLength(0) + { + } + + inline bool Ok() const { return m_error == HTTPError::None && !m_state.expired(); } + inline HTTPError Error() const { return m_error; } + inline int StatusCode() const { return m_statusCode; } + inline uint32_t ContentLength() const { return m_contentLength; } + + inline ReadResult ReadStream(DownloadCallback downloadCallback) { + auto locked = m_state.lock(); + if (locked == nullptr) return HTTPError::ConnectionClosed; + + return locked->ReadStreamImpl(downloadCallback); + } + + inline ReadResult ReadString() { + auto locked = m_state.lock(); + if (locked == nullptr) return HTTPError::ConnectionClosed; + + return locked->ReadStringImpl(m_contentLength); + } + + template + inline ReadResult ReadJson(JsonParserFn jsonParser) + { + auto locked = m_state.lock(); + if (locked == nullptr) return HTTPError::ConnectionClosed; + return locked->ReadJsonImpl(m_contentLength, jsonParser); + } private: std::weak_ptr m_state; + HTTPError m_error; + int m_statusCode; + uint32_t m_contentLength; }; } // namespace OpenShock::HTTP diff --git a/include/http/JsonAPI.h b/include/http/JsonAPI.h index 64887f83..67b9e188 100644 --- a/include/http/JsonAPI.h +++ b/include/http/JsonAPI.h @@ -9,15 +9,15 @@ namespace OpenShock::HTTP::JsonAPI { /// @brief Links the hub to the account with the given account link code, returns the hub token. Valid response codes: 200, 404 /// @param hubToken /// @return - HTTP::Response LinkAccount(HTTP::HTTPClient& client, std::string_view accountLinkCode); + JsonResponse LinkAccount(HTTP::HTTPClient& client, std::string_view accountLinkCode); /// @brief Gets the hub info for the given hub token. Valid response codes: 200, 401 /// @param hubToken /// @return - HTTP::Response GetHubInfo(HTTP::HTTPClient& client, const char* hubToken); + JsonResponse GetHubInfo(HTTP::HTTPClient& client, const char* hubToken); /// @brief Requests a Live Control Gateway to connect to. Valid response codes: 200, 401 /// @param hubToken /// @return - HTTP::Response AssignLcg(HTTP::HTTPClient& client, const char* hubToken); + JsonResponse AssignLcg(HTTP::HTTPClient& client, const char* hubToken); } // namespace OpenShock::HTTP::JsonAPI diff --git a/include/http/JsonResponse.h b/include/http/JsonResponse.h new file mode 100644 index 00000000..49848e22 --- /dev/null +++ b/include/http/JsonResponse.h @@ -0,0 +1,60 @@ +#pragma once + +#include "Common.h" +#include "http/DownloadCallback.h" +#include "http/HTTPClientState.h" +#include "http/JsonParserFn.h" +#include "http/ReadResult.h" + +#include +#include +#include + +namespace OpenShock::HTTP { + class HTTPClient; + template + class [[nodiscard]] JsonResponse { + DISABLE_DEFAULT(JsonResponse); + DISABLE_COPY(JsonResponse); + DISABLE_MOVE(JsonResponse); + + friend class HTTPClient; + + JsonResponse(std::shared_ptr state, JsonParserFn jsonParser, int statusCode, uint32_t contentLength) + : m_state(state) + , m_jsonParser(jsonParser) + , m_error(HTTPError::None) + , m_statusCode(statusCode) + , m_contentLength(contentLength) + { + } + public: + JsonResponse(HTTPError error) + : m_state() + , m_jsonParser() + , m_error(error) + , m_statusCode(0) + , m_contentLength(0) + { + } + + inline bool Ok() const { return m_error == HTTPError::None && !m_state.expired(); } + inline HTTPError Error() const { return m_error; } + inline int StatusCode() const { return m_statusCode; } + inline uint32_t ContentLength() const { return m_contentLength; } + + inline ReadResult ReadJson() + { + auto locked = m_state.lock(); + if (locked == nullptr) return HTTPError::ConnectionClosed; + + return locked->ReadJsonImpl(m_contentLength, m_jsonParser); + } + private: + std::weak_ptr m_state; + JsonParserFn m_jsonParser; + HTTPError m_error; + int m_statusCode; + uint32_t m_contentLength; + }; +} // namespace OpenShock::HTTP diff --git a/include/http/ReadResult.h b/include/http/ReadResult.h new file mode 100644 index 00000000..e4577e63 --- /dev/null +++ b/include/http/ReadResult.h @@ -0,0 +1,17 @@ +#pragma once + +#include "http/HTTPError.h" + +namespace OpenShock::HTTP { + template + struct [[nodiscard]] ReadResult { + HTTPError error{}; + T data{}; + + ReadResult(const T& d) + : error(HTTPError::None), data(d) {} + + ReadResult(const HTTPError& e) + : error(e), data{} {} + }; +} // namespace OpenShock::HTTP diff --git a/src/GatewayConnectionManager.cpp b/src/GatewayConnectionManager.cpp index 5289372d..040a5ad6 100644 --- a/src/GatewayConnectionManager.cpp +++ b/src/GatewayConnectionManager.cpp @@ -107,32 +107,37 @@ AccountLinkResultCode GatewayConnectionManager::Link(std::string_view linkCode) HTTP::HTTPClient client; auto response = HTTP::JsonAPI::LinkAccount(client, linkCode); + if (!response.Ok()) { - if (response.code == 404) { - return AccountLinkResultCode::InvalidCode; + if (response.Error() == HTTP::HTTPError::RateLimited) { + return AccountLinkResultCode::InternalError; // Just return false, don't spam the console with errors + } + + OS_LOGE(TAG, "Error while fetching auth token: %s %d", HTTP::HTTPErrorToString(response.Error()), response.StatusCode()); + return AccountLinkResultCode::InternalError; } - if (response.result == HTTP::ResponseResult::RateLimited) { - OS_LOGW(TAG, "Account Link request got ratelimited"); - return AccountLinkResultCode::RateLimited; + if (response.StatusCode() == 404) { + return AccountLinkResultCode::InvalidCode; } - if (response.result != HTTP::ResponseResult::Success) { - OS_LOGE(TAG, "Error while getting auth token: %s %d", response.ResultToString(), response.code); + if (response.StatusCode() != 200) { + OS_LOGE(TAG, "Unexpected response code: %d", response.StatusCode()); return AccountLinkResultCode::InternalError; } - if (response.code != 200) { - OS_LOGE(TAG, "Unexpected response code: %d", response.code); + auto content = response.ReadJson(); + if (content.error != HTTP::HTTPError::None) { + OS_LOGE(TAG, "Error while reading response: %s %d", HTTP::HTTPErrorToString(response.Error()), response.StatusCode()); return AccountLinkResultCode::InternalError; } - if (response.data.authToken.empty()) { + if (content.data.authToken.empty()) { OS_LOGE(TAG, "Received empty auth token"); return AccountLinkResultCode::InternalError; } - if (!Config::SetBackendAuthToken(std::move(response.data.authToken))) { + if (!Config::SetBackendAuthToken(std::move(content.data.authToken))) { OS_LOGE(TAG, "Failed to save auth token"); return AccountLinkResultCode::InternalError; } @@ -180,30 +185,36 @@ bool FetchHubInfo(const char* authToken) HTTP::HTTPClient client; auto response = HTTP::JsonAPI::GetHubInfo(client, authToken); + if (!response.Ok()) { + if (response.Error() == HTTP::HTTPError::RateLimited) { + return false; // Just return false, don't spam the console with errors + } - if (response.code == 401) { - OS_LOGD(TAG, "Auth token is invalid, waiting 5 minutes before checking again"); - s_lastAuthFailure = OpenShock::micros(); + OS_LOGE(TAG, "Error while fetching hub info: %s %d", HTTP::HTTPErrorToString(response.Error()), response.StatusCode()); return false; } - if (response.result == HTTP::ResponseResult::RateLimited) { - return false; // Just return false, don't spam the console with errors + if (response.StatusCode() == 401) { + OS_LOGD(TAG, "Auth token is invalid, waiting 5 minutes before retrying"); + s_lastAuthFailure = OpenShock::micros(); + return false; } - if (response.result != HTTP::ResponseResult::Success) { - OS_LOGE(TAG, "Error while fetching hub info: %s %d", response.ResultToString(), response.code); + + if (response.StatusCode() != 200) { + OS_LOGE(TAG, "Unexpected response code: %d", response.StatusCode()); return false; } - if (response.code != 200) { - OS_LOGE(TAG, "Unexpected response code: %d", response.code); + auto content = response.ReadJson(); + if (content.error != HTTP::HTTPError::None) { + OS_LOGE(TAG, "Error while reading response: %s %d", HTTP::HTTPErrorToString(response.Error()), response.StatusCode()); return false; } - OS_LOGI(TAG, "Hub ID: %s", response.data.hubId.c_str()); - OS_LOGI(TAG, "Hub Name: %s", response.data.hubName.c_str()); + OS_LOGI(TAG, "Hub ID: %s", content.data.hubId.c_str()); + OS_LOGI(TAG, "Hub Name: %s", content.data.hubName.c_str()); OS_LOGI(TAG, "Shockers:"); - for (auto& shocker : response.data.shockers) { + for (auto& shocker : content.data.shockers) { OS_LOGI(TAG, " [%s] rf=%u model=%u", shocker.id.c_str(), shocker.rfId, shocker.model); } @@ -245,28 +256,34 @@ bool StartConnectingToLCG() HTTP::HTTPClient client; auto response = HTTP::JsonAPI::AssignLcg(client, authToken.c_str()); + if (!response.Ok()) { + if (response.Error() == HTTP::HTTPError::RateLimited) { + return false; // Just return false, don't spam the console with errors + } - if (response.code == 401) { + OS_LOGE(TAG, "Error while fetching LCG endpoint: %s %d", HTTP::HTTPErrorToString(response.Error()), response.StatusCode()); + return false; + } + + if (response.StatusCode() == 401) { OS_LOGD(TAG, "Auth token is invalid, waiting 5 minutes before retrying"); s_lastAuthFailure = OpenShock::micros(); return false; } - if (response.result == HTTP::ResponseResult::RateLimited) { - return false; // Just return false, don't spam the console with errors - } - if (response.result != HTTP::ResponseResult::Success) { - OS_LOGE(TAG, "Error while fetching LCG endpoint: %s %d", response.ResultToString(), response.code); + if (response.StatusCode() != 200) { + OS_LOGE(TAG, "Unexpected response code: %d", response.StatusCode()); return false; } - if (response.code != 200) { - OS_LOGE(TAG, "Unexpected response code: %d", response.code); + auto content = response.ReadJson(); + if (content.error != HTTP::HTTPError::None) { + OS_LOGE(TAG, "Error while reading response: %s %d", HTTP::HTTPErrorToString(response.Error()), response.StatusCode()); return false; } - OS_LOGD(TAG, "Connecting to LCG endpoint { host: '%s', port: %hu, path: '%s' } in country %s", response.data.host.c_str(), response.data.port, response.data.path.c_str(), response.data.country.c_str()); - s_wsClient->connect(response.data.host, response.data.port, response.data.path); + OS_LOGD(TAG, "Connecting to LCG endpoint { host: '%s', port: %hu, path: '%s' } in country %s", content.data.host.c_str(), content.data.port, content.data.path.c_str(), content.data.country.c_str()); + s_wsClient->connect(content.data.host, content.data.port, content.data.path); return true; } diff --git a/src/OtaUpdateManager.cpp b/src/OtaUpdateManager.cpp index 36da3558..950e2d81 100644 --- a/src/OtaUpdateManager.cpp +++ b/src/OtaUpdateManager.cpp @@ -324,7 +324,7 @@ static void otaum_updatetask(void* arg) OS_LOGD(TAG, "Checking for updates"); // Fetch current version. - if (!OtaUpdateManager::TryGetFirmwareVersion(client, config.updateChannel, version)) { + if (!OtaUpdateManager::TryGetFirmwareVersion(config.updateChannel, version)) { OS_LOGE(TAG, "Failed to fetch firmware version"); continue; } @@ -361,7 +361,7 @@ static void otaum_updatetask(void* arg) // Fetch current release. OtaUpdateManager::FirmwareRelease release; - if (!OtaUpdateManager::TryGetFirmwareRelease(client, version, release)) { + if (!OtaUpdateManager::TryGetFirmwareRelease(version, release)) { OS_LOGE(TAG, "Failed to fetch firmware release"); // TODO: Send error message to server _sendFailureMessage("Failed to fetch firmware release"sv); continue; @@ -422,32 +422,30 @@ static void otaum_updatetask(void* arg) esp_restart(); } -static bool _tryGetStringList(HTTP::HTTPClient& client, const char* url, std::vector& list) +static bool _tryGetStringList(const char* url, std::vector& list) { - ; - esp_err_t err = client.Get(url); - if (err != ESP_OK) { + HTTP::HTTPClient client; + auto response = client.Get(url); + if (!response.Ok()) { OS_LOGE(TAG, "Failed to fetch list"); return false; } - int statusCode = client.StatusCode(); + int statusCode = response.StatusCode(); if (statusCode != 200 && statusCode != 304) { OS_LOGE(TAG, "Failed to fetch list"); return false; } - auto response = client.ReadResponseString(); - if (response.result != HTTP::ResponseResult::Success) { - OS_LOGE(TAG, "Failed to fetch list: %s [%u] %s", response.ResultToString(), response.code, response.data.c_str()); + auto content = response.ReadString(); + if (content.error != HTTP::HTTPError::None) { + OS_LOGE(TAG, "Failed to fetch list: %s [%u] %s", HTTP::HTTPErrorToString(response.Error()), response.StatusCode(), content.data.c_str()); return false; } list.clear(); - std::string_view data = response.data; - - auto lines = OpenShock::StringSplitNewLines(data); + auto lines = OpenShock::StringSplitNewLines(content.data); list.reserve(lines.size()); for (auto line : lines) { @@ -530,7 +528,7 @@ bool OtaUpdateManager::Init() return true; } -bool OtaUpdateManager::TryGetFirmwareVersion(HTTP::HTTPClient& client, OtaUpdateChannel channel, OpenShock::SemVer& version) +bool OtaUpdateManager::TryGetFirmwareVersion(OtaUpdateChannel channel, OpenShock::SemVer& version) { const char* channelIndexUrl; switch (channel) { @@ -550,33 +548,34 @@ bool OtaUpdateManager::TryGetFirmwareVersion(HTTP::HTTPClient& client, OtaUpdate OS_LOGD(TAG, "Fetching firmware version from %s", channelIndexUrl); - esp_err_t err = client.Get(channelIndexUrl); - if (err != ESP_OK) { + HTTP::HTTPClient client; + auto response = client.Get(channelIndexUrl); + if (!response.Ok()) { OS_LOGE(TAG, "Failed to fetch firmware version"); return false; } - int statusCode = client.StatusCode(); + int statusCode = response.StatusCode(); if (statusCode != 200 && statusCode != 304) { OS_LOGE(TAG, "Failed to fetch firmware version"); return false; } - auto response = client.ReadResponseString(); - if (response.result != HTTP::ResponseResult::Success) { - OS_LOGE(TAG, "Failed to fetch firmware version: %s [%u] %s", response.ResultToString(), response.code, response.data.c_str()); + auto content = response.ReadString(); + if (content.error != HTTP::HTTPError::None) { + OS_LOGE(TAG, "Failed to fetch firmware version: %s [%u] %s", HTTP::HTTPErrorToString(response.Error()), response.StatusCode(), content.data.c_str()); return false; } - if (!OpenShock::TryParseSemVer(response.data, version)) { - OS_LOGE(TAG, "Failed to parse firmware version: %.*s", response.data.size(), response.data.data()); + if (!OpenShock::TryParseSemVer(content.data, version)) { + OS_LOGE(TAG, "Failed to parse firmware version: %.*s", content.data.size(), content.data.data()); return false; } return true; } -bool OtaUpdateManager::TryGetFirmwareBoards(HTTP::HTTPClient& client, const OpenShock::SemVer& version, std::vector& boards) +bool OtaUpdateManager::TryGetFirmwareBoards(const OpenShock::SemVer& version, std::vector& boards) { std::string channelIndexUrl; if (!FormatToString(channelIndexUrl, OPENSHOCK_FW_CDN_BOARDS_INDEX_URL_FORMAT, version.toString().c_str())) { // TODO: This is abusing the SemVer::toString() method causing alot of string copies, fix this @@ -586,7 +585,7 @@ bool OtaUpdateManager::TryGetFirmwareBoards(HTTP::HTTPClient& client, const Open OS_LOGD(TAG, "Fetching firmware boards from %s", channelIndexUrl.c_str()); - if (!_tryGetStringList(client, channelIndexUrl.c_str(), boards)) { + if (!_tryGetStringList(channelIndexUrl.c_str(), boards)) { OS_LOGE(TAG, "Failed to fetch firmware boards"); return false; } @@ -604,7 +603,7 @@ static bool _tryParseIntoHash(std::string_view hash, uint8_t (&hashBytes)[32]) return true; } -bool OtaUpdateManager::TryGetFirmwareRelease(HTTP::HTTPClient& client, const OpenShock::SemVer& version, FirmwareRelease& release) +bool OtaUpdateManager::TryGetFirmwareRelease(const OpenShock::SemVer& version, FirmwareRelease& release) { auto versionStr = version.toString(); // TODO: This is abusing the SemVer::toString() method causing alot of string copies, fix this @@ -626,25 +625,26 @@ bool OtaUpdateManager::TryGetFirmwareRelease(HTTP::HTTPClient& client, const Ope } // Fetch hashes. - esp_err_t err = client.Get(sha256HashesUrl.c_str()); - if (err != ESP_OK) { + HTTP::HTTPClient client; + auto response = client.Get(sha256HashesUrl.c_str()); + if (!response.Ok()) { OS_LOGE(TAG, "Failed to fetch hashes"); return false; } - int statusCode = client.StatusCode(); + int statusCode = response.StatusCode(); if (statusCode != 200 && statusCode != 304) { OS_LOGE(TAG, "Failed to fetch hashes"); return false; } - auto response = client.ReadResponseString(); - if (response.result != HTTP::ResponseResult::Success) { - OS_LOGE(TAG, "Failed to fetch hashes: %s [%u] %s", response.ResultToString(), response.code, response.data.c_str()); + auto content = response.ReadString(); + if (content.error != HTTP::HTTPError::None) { + OS_LOGE(TAG, "Failed to fetch hashes: %s [%u] %s", HTTP::HTTPErrorToString(response.Error()), response.StatusCode(), content.data.c_str()); return false; } - auto hashesLines = OpenShock::StringSplitNewLines(response.data); + auto hashesLines = OpenShock::StringSplitNewLines(content.data); // Parse hashes. bool foundAppHash = false, foundFilesystemHash = false; diff --git a/src/http/HTTPClient.cpp b/src/http/HTTPClient.cpp index 11c243db..fb3c64a6 100644 --- a/src/http/HTTPClient.cpp +++ b/src/http/HTTPClient.cpp @@ -9,57 +9,23 @@ const char* const TAG = "HTTPClient"; using namespace OpenShock; -HTTP::HTTPResponse HTTP::HTTPClient::Get(const char* url) { - esp_err_t err = m_state->StartRequest(HTTP_METHOD_GET, url, 0); - if (err != ESP_OK) { - // TODO: Do something - } - - m_responseLength = esp_http_client_fetch_headers(m_handle); - if (m_responseLength == ESP_FAIL) { - return ESP_FAIL; - } - - m_statusCode = esp_http_client_get_status_code(m_handle); - - return ESP_OK; -} - -esp_err_t HTTP::HTTPClient::Close() { - return esp_http_client_close(m_handle); -} - -esp_err_t HTTP::HTTPClient::Start(esp_http_client_method_t method, const char* url, int writeLen) { - esp_err_t err; - - m_ratelimiter = HTTP::RateLimiters::GetRateLimiter(url); - if (m_ratelimiter == nullptr) { +HTTP::HTTPClient::InternalResult HTTP::HTTPClient::GetInternal(const char* url) { + auto ratelimiter = HTTP::RateLimiters::GetRateLimiter(url); + if (ratelimiter == nullptr) { OS_LOGW(TAG, "Invalid URL!"); - return ESP_FAIL; + return {HTTPError::InvalidUrl, {}}; } - if (!m_ratelimiter->tryRequest()) { + + if (!ratelimiter->tryRequest()) { OS_LOGW(TAG, "Hit ratelimit, refusing to send request!"); - return ESP_FAIL; + return {HTTPError::RateLimited, {}}; } - return m_state - - return esp_http_client_open(m_handle, writeLen); -} - -esp_err_t HTTP::HTTPClient::EventHandler(esp_http_client_event_t* evt) { -} - -esp_err_t HTTP::HTTPClient::HandleHeader(std::string_view key, std::string_view value) { - if (key == "Retry-After" && m_ratelimiter != nullptr) { - uint32_t seconds; - if (!Convert::ToUint32(value, seconds) || seconds <= 0) { - seconds = 15; - } - OS_LOGI(TAG, "Retry-After: %d seconds, applying delay to rate limiter", seconds); - m_ratelimiter->blockFor(seconds * 1000U); + auto result = m_state->StartRequest(HTTP_METHOD_GET, url, 0); + if (auto error = std::get_if(&result)) { + return {*error, {}}; } - return ESP_OK; + return {HTTPError::None, std::get(result)}; } diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp index 430ce6b2..116dd79f 100644 --- a/src/http/HTTPClientState.cpp +++ b/src/http/HTTPClientState.cpp @@ -3,18 +3,20 @@ const char* const TAG = "HTTPClientState"; #include "Common.h" +#include "Convert.h" #include "Logging.h" #include -static const std::size_t HTTP_BUFFER_SIZE = 4096LLU; -static const std::size_t HTTP_DOWNLOAD_SIZE_LIMIT = 200 * 1024 * 1024; // 200 MB +static const uint32_t HTTP_BUFFER_SIZE = 4096LLU; +static const uint32_t HTTP_DOWNLOAD_SIZE_LIMIT = 200 * 1024 * 1024; // 200 MB using namespace OpenShock; HTTP::HTTPClientState::HTTPClientState(uint32_t timeoutMs) : m_handle(nullptr) , m_reading(false) + , m_headers(false) { esp_http_client_config_t cfg; memset(&cfg, 0, sizeof(cfg)); @@ -40,48 +42,50 @@ HTTP::HTTPClientState::~HTTPClientState() } } -HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClientState::StartRequest(esp_http_client_method_t method, const char* url, int writeLen) +std::variant HTTP::HTTPClientState::StartRequest(esp_http_client_method_t method, const char* url, int writeLen) { esp_err_t err; err = esp_http_client_set_url(m_handle, url); - if (err != ESP_OK) return {err, false, 0}; + if (err != ESP_OK) return HTTPError::InvalidUrl; err = esp_http_client_set_method(m_handle, method); - if (err != ESP_OK) return {err, false, 0}; + if (err != ESP_OK) return HTTPError::InvalidHttpMethod; err = esp_http_client_open(m_handle, writeLen); - if (err != ESP_OK) return {err, false, 0}; + if (err != ESP_OK) return HTTPError::NetworkError; - int responseLength = esp_http_client_fetch_headers(m_handle); - if (responseLength == ESP_FAIL) return {err, false, 0}; + int contentLength = esp_http_client_fetch_headers(m_handle); + if (contentLength == ESP_FAIL) return HTTPError::NetworkError; bool isChunked = false; - if (responseLength == 0) { + if (contentLength == 0) { isChunked = esp_http_client_is_chunked_response(m_handle); } - return {ESP_OK, isChunked, static_cast(responseLength)}; + int code = esp_http_client_get_status_code(m_handle); + + return StartRequestResult {code, isChunked, static_cast(contentLength)}; } -HTTP::HTTPClientState::ReadResult HTTP::HTTPClientState::ReadStreamImpl(DownloadCallback cb) +HTTP::ReadResult HTTP::HTTPClientState::ReadStreamImpl(DownloadCallback cb) { if (m_handle == nullptr || !m_reading) { m_reading = false; - return {ReadResultCode::ConnectionClosed, 0}; + return HTTPError::ConnectionClosed; } - std::size_t totalWritten = 0; - uint8_t buffer[HTTP_BUFFER_SIZE]; + uint32_t totalWritten = 0; + uint8_t buffer[HTTP_BUFFER_SIZE]; while (true) { if (totalWritten >= HTTP_DOWNLOAD_SIZE_LIMIT) { m_reading = false; - return {ReadResultCode::SizeLimitExceeded, totalWritten}; + return HTTPError::SizeLimitExceeded; } - std::size_t remaining = HTTP_DOWNLOAD_SIZE_LIMIT - totalWritten; - int toRead = static_cast(std::min(HTTP_BUFFER_SIZE, remaining)); + uint32_t remaining = HTTP_DOWNLOAD_SIZE_LIMIT - totalWritten; + int toRead = static_cast(std::min(HTTP_BUFFER_SIZE, remaining)); int n = esp_http_client_read( m_handle, @@ -91,7 +95,7 @@ HTTP::HTTPClientState::ReadResult HTTP::HTTPClientState::ReadStreamImpl(Download if (n < 0) { m_reading = false; - return {ReadResultCode::NetworkError, totalWritten}; + return HTTPError::NetworkError; } if (n == 0) { @@ -99,17 +103,37 @@ HTTP::HTTPClientState::ReadResult HTTP::HTTPClientState::ReadStreamImpl(Download break; } - std::size_t chunkLen = static_cast(n); + uint32_t chunkLen = static_cast(n); if (!cb(totalWritten, buffer, chunkLen)) { m_reading = false; - return {ReadResultCode::Aborted, totalWritten}; + return HTTPError::Aborted; } totalWritten += chunkLen; } m_reading = false; - return {ReadResultCode::Success, totalWritten}; + return totalWritten; +} + +HTTP::ReadResult HTTP::HTTPClientState::ReadStringImpl(uint32_t reserve) +{ + std::string result; + if (reserve > 0) { + result.reserve(reserve); + } + + auto writer = [&result](std::size_t offset, const uint8_t* data, std::size_t len) { + result.append(reinterpret_cast(data), len); + return true; + }; + + auto response = ReadStreamImpl(writer); + if (response.error != HTTPError::None) { + return response.error; + } + + return result; } esp_err_t HTTP::HTTPClientState::EventHandler(esp_http_client_event_t* evt) @@ -122,7 +146,6 @@ esp_err_t HTTP::HTTPClientState::EventHandler(esp_http_client_event_t* evt) OS_LOGE(TAG, "Got error event"); break; case HTTP_EVENT_ON_CONNECTED: - client->m_connected = true; OS_LOGI(TAG, "Got connected event"); break; case HTTP_EVENT_HEADERS_SENT: @@ -137,7 +160,6 @@ esp_err_t HTTP::HTTPClientState::EventHandler(esp_http_client_event_t* evt) OS_LOGI(TAG, "Got on_finish event"); break; case HTTP_EVENT_DISCONNECTED: - client->m_connected = false; OS_LOGI(TAG, "Got disconnected event"); break; default: @@ -148,7 +170,19 @@ esp_err_t HTTP::HTTPClientState::EventHandler(esp_http_client_event_t* evt) return ESP_OK; } -esp_err_t HTTP::HTTPClientState::EventHeaderHandler(std::string_view key, std::string_view value) +esp_err_t HTTP::HTTPClientState::EventHeaderHandler(std::string key, std::string value) { - OS_LOGI(TAG, "Got header_received event: %s - %s", evt->header_key, evt->header_value); + OS_LOGI(TAG, "Got header_received event: %.*s - %.*s", key.length(), key.c_str(), key.length(), key.c_str()); + + if (key == "Retry-After") { + uint32_t seconds; + if (!Convert::ToUint32(value, seconds) || seconds <= 0) { + seconds = 15; + } + + OS_LOGI(TAG, "Retry-After: %d seconds, applying delay to rate limiter", seconds); + // TODO: Inform caller + } + + m_headers.emplace_back(std::move(key), std::move(value)); } diff --git a/src/http/HTTPRequestManager.cpp b/src/http/HTTPRequestManager.cpp deleted file mode 100644 index 1394823f..00000000 --- a/src/http/HTTPRequestManager.cpp +++ /dev/null @@ -1,363 +0,0 @@ - -/* -const char* const TAG = "HTTPRequestManager"; - -#include "Common.h" -#include "Core.h" -#include "Logging.h" -#include "http/HTTPClient.h" -#include "RateLimiter.h" -#include "SimpleMutex.h" -#include "util/HexUtils.h" -#include "util/StringUtils.h" - -#include -#include -#include -#include -#include - -using namespace std::string_view_literals; - -const std::size_t HTTP_BUFFER_SIZE = 4096LLU; -const int HTTP_DOWNLOAD_SIZE_LIMIT = 200 * 1024 * 1024; // 200 MB - -using namespace OpenShock; - -struct StreamReaderResult { - HTTP::DownloadResult result; - std::size_t nWritten; -}; - -constexpr bool _isCRLF(const uint8_t* buffer) -{ - return buffer[0] == '\r' && buffer[1] == '\n'; -} -constexpr bool _tryFindCRLF(std::size_t& pos, const uint8_t* buffer, std::size_t len) -{ - const uint8_t* cur = buffer; - const uint8_t* end = buffer + len - 1; - - while (cur < end) { - if (_isCRLF(cur)) { - pos = static_cast(cur - buffer); - return true; - } - - ++cur; - } - - return false; -} - -enum ParserState : uint8_t { - Ok, - NeedMoreData, - Invalid, -}; - -ParserState _parseChunkHeader(const uint8_t* buffer, std::size_t bufferLen, std::size_t& headerLen, std::size_t& payloadLen) -{ - if (bufferLen < 5) { // Bare minimum: "0\r\n\r\n" - return ParserState::NeedMoreData; - } - - // Find the first CRLF - if (!_tryFindCRLF(headerLen, buffer, bufferLen)) { - return ParserState::NeedMoreData; - } - - // Header must have at least one character - if (headerLen == 0) { - OS_LOGW(TAG, "Invalid chunk header length"); - return ParserState::Invalid; - } - - // Check for end of size field (possibly followed by extensions which is separated by a semicolon) - std::size_t sizeFieldEnd = headerLen; - for (std::size_t i = 0; i < headerLen; ++i) { - if (buffer[i] == ';') { - sizeFieldEnd = i; - break; - } - } - - // Bounds check - if (sizeFieldEnd == 0 || sizeFieldEnd > 16) { - OS_LOGW(TAG, "Invalid chunk size field length"); - return ParserState::Invalid; - } - - std::string_view sizeField(reinterpret_cast(buffer), sizeFieldEnd); - - // Parse the chunk size - if (!HexUtils::TryParseHexToInt(sizeField.data(), sizeField.length(), payloadLen)) { - OS_LOGW(TAG, "Failed to parse chunk size"); - return ParserState::Invalid; - } - - if (payloadLen > HTTP_DOWNLOAD_SIZE_LIMIT) { - OS_LOGW(TAG, "Chunk size too large"); - return ParserState::Invalid; - } - - // Set the header length to the end of the CRLF - headerLen += 2; - - return ParserState::Ok; -} - -ParserState _parseChunk(const uint8_t* buffer, std::size_t bufferLen, std::size_t& payloadPos, std::size_t& payloadLen) -{ - if (payloadPos == 0) { - ParserState state = _parseChunkHeader(buffer, bufferLen, payloadPos, payloadLen); - if (state != ParserState::Ok) { - return state; - } - } - - std::size_t totalLen = payloadPos + payloadLen + 2; // +2 for CRLF - if (bufferLen < totalLen) { - return ParserState::NeedMoreData; - } - - // Check for CRLF - if (!_isCRLF(buffer + totalLen - 2)) { - OS_LOGW(TAG, "Invalid chunk payload CRLF"); - return ParserState::Invalid; - } - - return ParserState::Ok; -} - -void _alignChunk(uint8_t* buffer, std::size_t& bufferCursor, std::size_t payloadPos, std::size_t payloadLen) -{ - std::size_t totalLen = payloadPos + payloadLen + 2; // +2 for CRLF - std::size_t remaining = bufferCursor - totalLen; - if (remaining > 0) { - memmove(buffer, buffer + totalLen, remaining); - bufferCursor = remaining; - } else { - bufferCursor = 0; - } -} - -StreamReaderResult _readStreamDataChunked(HTTP::HTTPClient client, WiFiClient* stream, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) -{ - std::size_t totalWritten = 0; - HTTP::DownloadResult result = HTTP::DownloadResult::Success; - - uint8_t* buffer = static_cast(malloc(HTTP_BUFFER_SIZE)); - if (buffer == nullptr) { - OS_LOGE(TAG, "Out of memory"); - return {HTTP::DownloadResult::RequestFailed, 0}; - } - - ParserState state = ParserState::NeedMoreData; - std::size_t bufferCursor = 0, payloadPos = 0, payloadSize = 0; - - while (client.connected() && state != ParserState::Invalid) { - if (begin + timeoutMs < OpenShock::millis()) { - OS_LOGW(TAG, "Request timed out"); - result = HTTP::DownloadResult::TimedOut; - break; - } - - std::size_t bytesAvailable = stream->available(); - if (bytesAvailable == 0) { - vTaskDelay(pdMS_TO_TICKS(5)); - continue; - } - - std::size_t bytesRead = stream->readBytes(buffer + bufferCursor, HTTP_BUFFER_SIZE - bufferCursor); - if (bytesRead == 0) { - OS_LOGW(TAG, "No bytes read"); - result = HTTP::DownloadResult::RequestFailed; - break; - } - - bufferCursor += bytesRead; - - while (bufferCursor > 0) { - state = _parseChunk(buffer, bufferCursor, payloadPos, payloadSize); - if (state == ParserState::Invalid) { - OS_LOGE(TAG, "Failed to parse chunk"); - result = HTTP::DownloadResult::RequestFailed; - state = ParserState::Invalid; // Mark to exit both loops - break; - } - OS_LOGD(TAG, "Chunk parsed: %zu %zu", payloadPos, payloadSize); - - if (state == ParserState::NeedMoreData) { - if (bufferCursor == HTTP_BUFFER_SIZE) { - OS_LOGE(TAG, "Chunk too large"); - result = HTTP::DownloadResult::RequestFailed; - state = ParserState::Invalid; // Mark to exit both loops - } - break; // If chunk size good, this only exits one loop - } - - // Check for zero chunk size (end of transfer) - if (payloadSize == 0) { - state = ParserState::Invalid; // Mark to exit both loops - break; - } - - if (!downloadCallback(totalWritten, buffer + payloadPos, payloadSize)) { - result = HTTP::DownloadResult::Cancelled; - state = ParserState::Invalid; // Mark to exit both loops - break; - } - - totalWritten += payloadSize; - _alignChunk(buffer, bufferCursor, payloadPos, payloadSize); - payloadSize = 0; - payloadPos = 0; - } - - if (state == ParserState::NeedMoreData) { - vTaskDelay(pdMS_TO_TICKS(5)); - } - } - - free(buffer); - - return {result, totalWritten}; -} - -StreamReaderResult _readStreamData(HTTP::HTTPClient client, WiFiClient* stream, std::size_t contentLength, HTTP::DownloadCallback downloadCallback, int64_t begin, int timeoutMs) -{ - std::size_t nWritten = 0; - HTTP::DownloadResult result = HTTP::DownloadResult::Success; - - uint8_t* buffer = static_cast(malloc(HTTP_BUFFER_SIZE)); - - while (client.connected() && nWritten < contentLength) { - if (begin + timeoutMs < OpenShock::millis()) { - OS_LOGW(TAG, "Request timed out"); - result = HTTP::DownloadResult::TimedOut; - break; - } - - std::size_t bytesAvailable = stream->available(); - if (bytesAvailable == 0) { - vTaskDelay(pdMS_TO_TICKS(5)); - continue; - } - - std::size_t bytesToRead = std::min(bytesAvailable, HTTP_BUFFER_SIZE); - - std::size_t bytesRead = stream->readBytes(buffer, bytesToRead); - if (bytesRead == 0) { - OS_LOGW(TAG, "No bytes read"); - result = HTTP::DownloadResult::RequestFailed; - break; - } - - if (!downloadCallback(nWritten, buffer, bytesRead)) { - OS_LOGW(TAG, "Request cancelled by callback"); - result = HTTP::DownloadResult::Cancelled; - break; - } - - nWritten += bytesRead; - - vTaskDelay(pdMS_TO_TICKS(10)); - } - - free(buffer); - - return {result, nWritten}; -} - -HTTP::Response _doGetStream( - HTTP::HTTPClient& client, - const char* url, - tcb::span acceptedCodes, - HTTP::GotContentLengthCallback contentLengthCallback, - HTTP::DownloadCallback downloadCallback, - int timeoutMs -) -{ - esp_err_t err; - - int64_t begin = OpenShock::millis(); - - err = client.Get(url); - if (err != ESP_OK) { - OS_LOGE(TAG, "Failed to begin HTTP request"); - return {HTTP::DownloadResult::RequestFailed, 0, 0}; - } - - err = esp_http_res - auto responseCode = response.ResponseCode(); - if (responseCode == HTTP_CODE_REQUEST_TIMEOUT || begin + timeoutMs < OpenShock::millis()) { - OS_LOGW(TAG, "Request timed out"); - return {HTTP::DownloadResult::TimedOut, responseCode, 0}; - } - - if (responseCode == HTTP_CODE_TOO_MANY_REQUESTS) { - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After - - // Get "Retry-After" header - std::string retryAfterStr = client.header("Retry-After"); - - // Try to parse it as an integer (delay-seconds) - long retryAfter = 0; - if (retryAfterStr.length() > 0 && std::all_of(retryAfterStr.begin(), retryAfterStr.end(), isdigit)) { - retryAfter = retryAfterStr.toInt(); - } - - // If header missing/unparseable, default to 15 seconds - if (retryAfter <= 0) { - retryAfter = 15; - } - - // Apply the block-for time - rateLimiter->blockFor(retryAfter * 1000); - - return {HTTP::DownloadResult::RateLimited, responseCode, 0}; - } - - if (responseCode == 418) { - OS_LOGW(TAG, "The server refused to brew coffee because it is, permanently, a teapot."); - } - - if (std::find(acceptedCodes.begin(), acceptedCodes.end(), responseCode) == acceptedCodes.end()) { - OS_LOGD(TAG, "Received unexpected response code %d", responseCode); - return {HTTP::DownloadResult::CodeRejected, responseCode, 0}; - } - - int contentLength = client.getSize(); - if (contentLength == 0) { - return {HTTP::DownloadResult::Success, responseCode, 0}; - } - - if (contentLength > 0) { - if (contentLength > HTTP_DOWNLOAD_SIZE_LIMIT) { - OS_LOGE(TAG, "Content-Length too large"); - return {HTTP::DownloadResult::RequestFailed, responseCode, 0}; - } - - if (!contentLengthCallback(contentLength)) { - OS_LOGW(TAG, "Request cancelled by callback"); - return {HTTP::DownloadResult::Cancelled, responseCode, 0}; - } - } - - WiFiClient* stream = client.getStreamPtr(); - if (stream == nullptr) { - OS_LOGE(TAG, "Failed to get stream"); - return {HTTP::DownloadResult::RequestFailed, 0, 0}; - } - - StreamReaderResult result; - if (contentLength > 0) { - result = _readStreamData(client, stream, contentLength, downloadCallback, begin, timeoutMs); - } else { - result = _readStreamDataChunked(client, stream, downloadCallback, begin, timeoutMs); - } - - return {result.result, responseCode, result.nWritten}; -} -*/ diff --git a/src/http/JsonAPI.cpp b/src/http/JsonAPI.cpp index d0270a08..dd332e7d 100644 --- a/src/http/JsonAPI.cpp +++ b/src/http/JsonAPI.cpp @@ -2,15 +2,16 @@ #include "Common.h" #include "config/Config.h" +#include "http/HTTPClient.h" #include "util/StringUtils.h" using namespace OpenShock; -HTTP::Response HTTP::JsonAPI::LinkAccount(HTTP::HTTPClient& client, std::string_view accountLinkCode) +HTTP::JsonResponse HTTP::JsonAPI::LinkAccount(HTTP::HTTPClient& client, std::string_view accountLinkCode) { std::string domain; if (!Config::GetBackendDomain(domain)) { - return {HTTP::RequestResult::InternalError, 0, {}}; + return HTTPError::InternalError; } char uri[OPENSHOCK_URI_BUFFER_SIZE]; @@ -18,23 +19,14 @@ HTTP::Response HTTP::JsonAPI::LinkA client.SetHeader("Accept", "application/json"); - esp_err_t err = client.Get(uri); - if (err != ESP_OK) { - return {HTTP::RequestResult::InternalError, 0, {}}; - } - - if (client.StatusCode() != 200) { - return {HTTP::RequestResult::InternalError, 0, {}}; - } - - return client.ReadResponseJSON(Serialization::JsonAPI::ParseAccountLinkJsonResponse); + return client.GetJson(uri, Serialization::JsonAPI::ParseAccountLinkJsonResponse); } -HTTP::Response HTTP::JsonAPI::GetHubInfo(HTTP::HTTPClient& client, const char* hubToken) +HTTP::JsonResponse HTTP::JsonAPI::GetHubInfo(HTTP::HTTPClient& client, const char* hubToken) { std::string domain; if (!Config::GetBackendDomain(domain)) { - return {HTTP::RequestResult::InternalError, 0, {}}; + return HTTPError::InternalError; } char uri[OPENSHOCK_URI_BUFFER_SIZE]; @@ -43,23 +35,14 @@ HTTP::Response HTTP::JsonAPI::GetHubInf client.SetHeader("Accept", "application/json"); client.SetHeader("DeviceToken", hubToken); - esp_err_t err = client.Get(uri); - if (err != ESP_OK) { - return {HTTP::RequestResult::InternalError, 0, {}}; - } - - if (client.StatusCode() != 200) { - return {HTTP::RequestResult::InternalError, 0, {}}; - } - - return client.ReadResponseJSON(Serialization::JsonAPI::ParseHubInfoJsonResponse); + return client.GetJson(uri, Serialization::JsonAPI::ParseHubInfoJsonResponse); } -HTTP::Response HTTP::JsonAPI::AssignLcg(HTTP::HTTPClient& client, const char* hubToken) +HTTP::JsonResponse HTTP::JsonAPI::AssignLcg(HTTP::HTTPClient& client, const char* hubToken) { std::string domain; if (!Config::GetBackendDomain(domain)) { - return {HTTP::RequestResult::InternalError, 0, {}}; + return HTTPError::InternalError; } char uri[OPENSHOCK_URI_BUFFER_SIZE]; @@ -68,14 +51,5 @@ HTTP::Response HTTP::JsonAPI::AssignL client.SetHeader("Accept", "application/json"); client.SetHeader("DeviceToken", hubToken); - esp_err_t err = client.Get(uri); - if (err != ESP_OK) { - return {HTTP::RequestResult::InternalError, 0, {}}; - } - - if (client.StatusCode() != 200) { - return {HTTP::RequestResult::InternalError, 0, {}}; - } - - return client.ReadResponseJSON(Serialization::JsonAPI::ParseAssignLcgJsonResponse); + return client.GetJson(uri, Serialization::JsonAPI::ParseAssignLcgJsonResponse); } diff --git a/src/serial/SerialInputHandler.cpp b/src/serial/SerialInputHandler.cpp index b6c757c2..7d7d81de 100644 --- a/src/serial/SerialInputHandler.cpp +++ b/src/serial/SerialInputHandler.cpp @@ -10,7 +10,6 @@ const char* const TAG = "SerialInputHandler"; #include "Core.h" #include "estop/EStopManager.h" #include "FormatHelpers.h" -#include "http/HTTPRequestManager.h" #include "Logging.h" #include "serial/command_handlers/CommandEntry.h" #include "serial/command_handlers/common.h" diff --git a/src/serial/command_handlers/authtoken.cpp b/src/serial/command_handlers/authtoken.cpp index 18c9ade6..f962bb9b 100644 --- a/src/serial/command_handlers/authtoken.cpp +++ b/src/serial/command_handlers/authtoken.cpp @@ -1,6 +1,7 @@ #include "serial/command_handlers/common.h" #include "config/Config.h" +#include "http/HTTPClient.h" #include "http/JsonAPI.h" #include @@ -18,15 +19,21 @@ void _handleAuthtokenCommand(std::string_view arg, bool isAutomated) { return; } - auto apiResponse = OpenShock::HTTP::JsonAPI::GetHubInfo(arg); - if (apiResponse.code == 401) { - SERPR_ERROR("Invalid auth token, refusing to save it!"); - return; + std::string token = std::string(arg); + + // Scope to immidiatley destroy client after use + { + OpenShock::HTTP::HTTPClient client; + auto apiResponse = OpenShock::HTTP::JsonAPI::GetHubInfo(client, token.c_str()); + + if (apiResponse.StatusCode() == 401) { + SERPR_ERROR("Invalid auth token, refusing to save it!"); + return; + } } // If we have some other kind of request fault just set it anyway, we probably arent connected to a network - - bool result = OpenShock::Config::SetBackendAuthToken(std::string(arg)); + bool result = OpenShock::Config::SetBackendAuthToken(std::move(token)); if (result) { SERPR_SUCCESS("Saved config"); diff --git a/src/serial/command_handlers/domain.cpp b/src/serial/command_handlers/domain.cpp index 91185593..5f1663c9 100644 --- a/src/serial/command_handlers/domain.cpp +++ b/src/serial/command_handlers/domain.cpp @@ -1,7 +1,6 @@ #include "serial/command_handlers/common.h" #include "config/Config.h" -#include "http/HTTPRequestManager.h" #include "serialization/JsonAPI.h" #include diff --git a/src/util/ParitionUtils.cpp b/src/util/ParitionUtils.cpp index 7b9c9569..6ccbd86c 100644 --- a/src/util/ParitionUtils.cpp +++ b/src/util/ParitionUtils.cpp @@ -4,7 +4,6 @@ const char* const TAG = "PartitionUtils"; #include "Core.h" #include "Hashing.h" -#include "http/HTTPRequestManager.h" #include "Logging.h" #include "util/HexUtils.h" From 979a17d182acc1808ef454e6659ee88f1c965c84 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 03:50:21 +0100 Subject: [PATCH 09/28] These are not needed lmao --- src/http/JsonAPI.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http/JsonAPI.cpp b/src/http/JsonAPI.cpp index dd332e7d..03631861 100644 --- a/src/http/JsonAPI.cpp +++ b/src/http/JsonAPI.cpp @@ -19,7 +19,7 @@ HTTP::JsonResponse HTTP::JsonAPI::L client.SetHeader("Accept", "application/json"); - return client.GetJson(uri, Serialization::JsonAPI::ParseAccountLinkJsonResponse); + return client.GetJson(uri, Serialization::JsonAPI::ParseAccountLinkJsonResponse); } HTTP::JsonResponse HTTP::JsonAPI::GetHubInfo(HTTP::HTTPClient& client, const char* hubToken) @@ -35,7 +35,7 @@ HTTP::JsonResponse HTTP::JsonAPI::GetHu client.SetHeader("Accept", "application/json"); client.SetHeader("DeviceToken", hubToken); - return client.GetJson(uri, Serialization::JsonAPI::ParseHubInfoJsonResponse); + return client.GetJson(uri, Serialization::JsonAPI::ParseHubInfoJsonResponse); } HTTP::JsonResponse HTTP::JsonAPI::AssignLcg(HTTP::HTTPClient& client, const char* hubToken) @@ -51,5 +51,5 @@ HTTP::JsonResponse HTTP::JsonAPI::Ass client.SetHeader("Accept", "application/json"); client.SetHeader("DeviceToken", hubToken); - return client.GetJson(uri, Serialization::JsonAPI::ParseAssignLcgJsonResponse); + return client.GetJson(uri, Serialization::JsonAPI::ParseAssignLcgJsonResponse); } From 8cc417b41555860df305be07cd56eb1cb3712b58 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 03:54:38 +0100 Subject: [PATCH 10/28] nvm... --- src/http/JsonAPI.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http/JsonAPI.cpp b/src/http/JsonAPI.cpp index 03631861..dd332e7d 100644 --- a/src/http/JsonAPI.cpp +++ b/src/http/JsonAPI.cpp @@ -19,7 +19,7 @@ HTTP::JsonResponse HTTP::JsonAPI::L client.SetHeader("Accept", "application/json"); - return client.GetJson(uri, Serialization::JsonAPI::ParseAccountLinkJsonResponse); + return client.GetJson(uri, Serialization::JsonAPI::ParseAccountLinkJsonResponse); } HTTP::JsonResponse HTTP::JsonAPI::GetHubInfo(HTTP::HTTPClient& client, const char* hubToken) @@ -35,7 +35,7 @@ HTTP::JsonResponse HTTP::JsonAPI::GetHu client.SetHeader("Accept", "application/json"); client.SetHeader("DeviceToken", hubToken); - return client.GetJson(uri, Serialization::JsonAPI::ParseHubInfoJsonResponse); + return client.GetJson(uri, Serialization::JsonAPI::ParseHubInfoJsonResponse); } HTTP::JsonResponse HTTP::JsonAPI::AssignLcg(HTTP::HTTPClient& client, const char* hubToken) @@ -51,5 +51,5 @@ HTTP::JsonResponse HTTP::JsonAPI::Ass client.SetHeader("Accept", "application/json"); client.SetHeader("DeviceToken", hubToken); - return client.GetJson(uri, Serialization::JsonAPI::ParseAssignLcgJsonResponse); + return client.GetJson(uri, Serialization::JsonAPI::ParseAssignLcgJsonResponse); } From c0ee503ca0c100ab5878d136d5b0e5f07f4c611f Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 04:23:12 +0100 Subject: [PATCH 11/28] more work --- include/http/HTTPClientState.h | 2 +- include/http/HTTPResponse.h | 6 +-- src/http/HTTPClientState.cpp | 3 +- src/serial/command_handlers/domain.cpp | 25 ++++++------ src/util/ParitionUtils.cpp | 56 +++++++++++--------------- 5 files changed, 42 insertions(+), 50 deletions(-) diff --git a/include/http/HTTPClientState.h b/include/http/HTTPClientState.h index 9aca55e3..2c45563d 100644 --- a/include/http/HTTPClientState.h +++ b/include/http/HTTPClientState.h @@ -28,7 +28,7 @@ namespace OpenShock::HTTP { } struct [[nodiscard]] StartRequestResult { - int statusCode; + uint32_t statusCode; bool isChunked; uint32_t contentLength; }; diff --git a/include/http/HTTPResponse.h b/include/http/HTTPResponse.h index 60843f5f..3ab1cf4c 100644 --- a/include/http/HTTPResponse.h +++ b/include/http/HTTPResponse.h @@ -36,10 +36,10 @@ namespace OpenShock::HTTP { , m_contentLength(0) { } - + inline bool Ok() const { return m_error == HTTPError::None && !m_state.expired(); } inline HTTPError Error() const { return m_error; } - inline int StatusCode() const { return m_statusCode; } + inline uint32_t StatusCode() const { return m_statusCode; } inline uint32_t ContentLength() const { return m_contentLength; } inline ReadResult ReadStream(DownloadCallback downloadCallback) { @@ -67,7 +67,7 @@ namespace OpenShock::HTTP { private: std::weak_ptr m_state; HTTPError m_error; - int m_statusCode; + uint32_t m_statusCode; uint32_t m_contentLength; }; } // namespace OpenShock::HTTP diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp index 116dd79f..ded29662 100644 --- a/src/http/HTTPClientState.cpp +++ b/src/http/HTTPClientState.cpp @@ -64,8 +64,9 @@ std::variant HTTP::H } int code = esp_http_client_get_status_code(m_handle); + if (code < 0) code = 0; - return StartRequestResult {code, isChunked, static_cast(contentLength)}; + return StartRequestResult {static_cast(code), isChunked, static_cast(contentLength)}; } HTTP::ReadResult HTTP::HTTPClientState::ReadStreamImpl(DownloadCallback cb) diff --git a/src/serial/command_handlers/domain.cpp b/src/serial/command_handlers/domain.cpp index 5f1663c9..d03104bf 100644 --- a/src/serial/command_handlers/domain.cpp +++ b/src/serial/command_handlers/domain.cpp @@ -1,7 +1,8 @@ #include "serial/command_handlers/common.h" #include "config/Config.h" -#include "serialization/JsonAPI.h" +#include "http/HTTPClient.h" +#include "http/JsonAPI.h" #include @@ -32,21 +33,19 @@ void _handleDomainCommand(std::string_view arg, bool isAutomated) { char uri[OPENSHOCK_URI_BUFFER_SIZE]; sprintf(uri, "https://%.*s/1", arg.length(), arg.data()); - auto resp = OpenShock::HTTP::GetJSON( - uri, - { - {"Accept", "application/json"} - }, - OpenShock::Serialization::JsonAPI::ParseBackendVersionJsonResponse, - std::array {200} - ); - - if (resp.result != OpenShock::HTTP::RequestResult::Success) { - SERPR_ERROR("Tried to connect to \"%.*s\", but failed with status [%d] (%s), refusing to save domain to config", arg.length(), arg.data(), resp.code, resp.ResultToString()); + OpenShock::HTTP::HTTPClient client; + auto response = client.GetJson(uri, OpenShock::Serialization::JsonAPI::ParseBackendVersionJsonResponse); + if (!response.Ok() || response.StatusCode() != 200) { + SERPR_ERROR("Tried to connect to \"%.*s\", but failed with status [%d] (%s), refusing to save domain to config", arg.length(), arg.data(), response.StatusCode(), OpenShock::HTTP::HTTPErrorToString(response.Error())); return; } - OS_LOGI(TAG, "Successfully connected to \"%.*s\", version: %s, commit: %s, current time: %s", arg.length(), arg.data(), resp.data.version.c_str(), resp.data.commit.c_str(), resp.data.currentTime.c_str()); + auto content = response.ReadJson(); + if (content.error != OpenShock::HTTP::HTTPError::None) { + #error TODO: Handle this + } + + OS_LOGI(TAG, "Successfully connected to \"%.*s\", version: %s, commit: %s, current time: %s", arg.length(), arg.data(), content.data.version.c_str(), content.data.commit.c_str(), content.data.currentTime.c_str()); bool result = OpenShock::Config::SetBackendDomain(std::string(arg)); diff --git a/src/util/ParitionUtils.cpp b/src/util/ParitionUtils.cpp index 6ccbd86c..e75a1080 100644 --- a/src/util/ParitionUtils.cpp +++ b/src/util/ParitionUtils.cpp @@ -4,6 +4,7 @@ const char* const TAG = "PartitionUtils"; #include "Core.h" #include "Hashing.h" +#include "http/HTTPClient.h" #include "Logging.h" #include "util/HexUtils.h" @@ -34,25 +35,6 @@ bool OpenShock::FlashPartitionFromUrl(const esp_partition_t* partition, const ch std::size_t contentWritten = 0; int64_t lastProgress = 0; - auto sizeValidator = [partition, &contentLength, progressCallback, &lastProgress](std::size_t size) -> bool { - if (size > partition->size) { - OS_LOGE(TAG, "Remote partition binary is too large"); - return false; - } - - // Erase app partition. - if (esp_partition_erase_range(partition, 0, partition->size) != ESP_OK) { - OS_LOGE(TAG, "Failed to erase partition in preparation for update"); - return false; - } - - contentLength = size; - - lastProgress = OpenShock::millis(); - progressCallback(0, contentLength, 0.0f); - - return true; - }; auto dataWriter = [partition, &sha256, &contentLength, &contentWritten, progressCallback, &lastProgress](std::size_t offset, const uint8_t* data, std::size_t length) -> bool { if (esp_partition_write(partition, offset, data, length) != ESP_OK) { OS_LOGE(TAG, "Failed to write to partition"); @@ -76,23 +58,33 @@ bool OpenShock::FlashPartitionFromUrl(const esp_partition_t* partition, const ch }; // Start streaming binary to app partition. - auto appBinaryResponse = OpenShock::HTTP::Download( - remoteUrl, - { - {"Accept", "application/octet-stream"} - }, - sizeValidator, - dataWriter, - std::array {200, 304}, - 180'000 - ); // 3 minutes - if (appBinaryResponse.result != OpenShock::HTTP::RequestResult::Success) { - OS_LOGE(TAG, "Failed to download remote partition binary: [%u]", appBinaryResponse.code); + OpenShock::HTTP::HTTPClient client(180'000); // 3 minutes timeout + auto response = client.Get(remoteUrl); + if (!response.Ok() || response.StatusCode() != 200 || response.StatusCode() != 304) { + OS_LOGE(TAG, "Failed to download remote partition binary: [%u]", response.StatusCode()); return false; } + if (response.ContentLength() > partition->size) { + OS_LOGE(TAG, "Remote partition binary is too large"); + return false; + } + + // Erase app partition. + if (esp_partition_erase_range(partition, 0, partition->size) != ESP_OK) { + OS_LOGE(TAG, "Failed to erase partition in preparation for update"); + return false; + } + + contentLength = response.ContentLength(); + + lastProgress = OpenShock::millis(); + progressCallback(0, contentLength, 0.0f); + + contentLength = response.ReadStream(dataWriter); + progressCallback(contentLength, contentLength, 1.0f); - OS_LOGD(TAG, "Wrote %u bytes to partition", appBinaryResponse.data); + OS_LOGD(TAG, "Wrote %u bytes to partition", contentLength); std::array localHash; if (!sha256.finish(localHash)) { From e8d279006591a1aaecead46465611cd5ded5029b Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 10:35:34 +0100 Subject: [PATCH 12/28] More fixes and touchup --- include/OtaUpdateManager.h | 1 - include/http/HTTPClient.h | 3 ++- include/http/HTTPClientState.h | 2 +- include/http/HTTPResponse.h | 8 +++----- include/http/JsonParserFn.h | 3 +-- include/http/JsonResponse.h | 7 +++---- src/OtaUpdateManager.cpp | 8 +++++--- src/http/HTTPClient.cpp | 2 ++ src/http/HTTPClientState.cpp | 11 +++++++++-- src/http/RateLimiters.cpp | 2 ++ src/serial/command_handlers/domain.cpp | 3 ++- src/serialization/JsonSerial.cpp | 2 +- src/util/ParitionUtils.cpp | 8 ++++++-- 13 files changed, 37 insertions(+), 23 deletions(-) diff --git a/include/OtaUpdateManager.h b/include/OtaUpdateManager.h index c833c8e9..2d319488 100644 --- a/include/OtaUpdateManager.h +++ b/include/OtaUpdateManager.h @@ -1,7 +1,6 @@ #pragma once #include "FirmwareBootType.h" -#include "http/HTTPClient.h" #include "OtaUpdateChannel.h" #include "SemVer.h" diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h index 952cfec2..f24ad1b6 100644 --- a/include/http/HTTPClient.h +++ b/include/http/HTTPClient.h @@ -5,8 +5,9 @@ #include "http/HTTPResponse.h" #include "http/JsonResponse.h" -#include +#include +#include #include namespace OpenShock::HTTP { diff --git a/include/http/HTTPClientState.h b/include/http/HTTPClientState.h index 2c45563d..966bf7b6 100644 --- a/include/http/HTTPClientState.h +++ b/include/http/HTTPClientState.h @@ -28,7 +28,7 @@ namespace OpenShock::HTTP { } struct [[nodiscard]] StartRequestResult { - uint32_t statusCode; + uint16_t statusCode; bool isChunked; uint32_t contentLength; }; diff --git a/include/http/HTTPResponse.h b/include/http/HTTPResponse.h index 3ab1cf4c..240570d9 100644 --- a/include/http/HTTPResponse.h +++ b/include/http/HTTPResponse.h @@ -6,8 +6,6 @@ #include "http/JsonParserFn.h" #include "http/ReadResult.h" -#include - #include #include #include @@ -21,7 +19,7 @@ namespace OpenShock::HTTP { friend class HTTPClient; - HTTPResponse(std::shared_ptr state, int statusCode, uint32_t contentLength) + HTTPResponse(std::shared_ptr state, uint16_t statusCode, uint32_t contentLength) : m_state(state) , m_error(HTTPError::None) , m_statusCode(statusCode) @@ -39,7 +37,7 @@ namespace OpenShock::HTTP { inline bool Ok() const { return m_error == HTTPError::None && !m_state.expired(); } inline HTTPError Error() const { return m_error; } - inline uint32_t StatusCode() const { return m_statusCode; } + inline uint16_t StatusCode() const { return m_statusCode; } inline uint32_t ContentLength() const { return m_contentLength; } inline ReadResult ReadStream(DownloadCallback downloadCallback) { @@ -67,7 +65,7 @@ namespace OpenShock::HTTP { private: std::weak_ptr m_state; HTTPError m_error; - uint32_t m_statusCode; + uint16_t m_statusCode; uint32_t m_contentLength; }; } // namespace OpenShock::HTTP diff --git a/include/http/JsonParserFn.h b/include/http/JsonParserFn.h index ba9a10f1..3a334701 100644 --- a/include/http/JsonParserFn.h +++ b/include/http/JsonParserFn.h @@ -1,10 +1,9 @@ #pragma once -#include - #include #include +class cJSON; namespace OpenShock::HTTP { template using JsonParserFn = std::function; diff --git a/include/http/JsonResponse.h b/include/http/JsonResponse.h index 49848e22..5af1529f 100644 --- a/include/http/JsonResponse.h +++ b/include/http/JsonResponse.h @@ -8,7 +8,6 @@ #include #include -#include namespace OpenShock::HTTP { class HTTPClient; @@ -20,7 +19,7 @@ namespace OpenShock::HTTP { friend class HTTPClient; - JsonResponse(std::shared_ptr state, JsonParserFn jsonParser, int statusCode, uint32_t contentLength) + JsonResponse(std::shared_ptr state, JsonParserFn jsonParser, uint16_t statusCode, uint32_t contentLength) : m_state(state) , m_jsonParser(jsonParser) , m_error(HTTPError::None) @@ -40,7 +39,7 @@ namespace OpenShock::HTTP { inline bool Ok() const { return m_error == HTTPError::None && !m_state.expired(); } inline HTTPError Error() const { return m_error; } - inline int StatusCode() const { return m_statusCode; } + inline uint16_t StatusCode() const { return m_statusCode; } inline uint32_t ContentLength() const { return m_contentLength; } inline ReadResult ReadJson() @@ -54,7 +53,7 @@ namespace OpenShock::HTTP { std::weak_ptr m_state; JsonParserFn m_jsonParser; HTTPError m_error; - int m_statusCode; + uint16_t m_statusCode; uint32_t m_contentLength; }; } // namespace OpenShock::HTTP diff --git a/src/OtaUpdateManager.cpp b/src/OtaUpdateManager.cpp index 950e2d81..b0807a1b 100644 --- a/src/OtaUpdateManager.cpp +++ b/src/OtaUpdateManager.cpp @@ -1,3 +1,5 @@ +#include + #include "OtaUpdateManager.h" const char* const TAG = "OtaUpdateManager"; @@ -431,7 +433,7 @@ static bool _tryGetStringList(const char* url, std::vector& list) return false; } - int statusCode = response.StatusCode(); + uint16_t statusCode = response.StatusCode(); if (statusCode != 200 && statusCode != 304) { OS_LOGE(TAG, "Failed to fetch list"); return false; @@ -555,7 +557,7 @@ bool OtaUpdateManager::TryGetFirmwareVersion(OtaUpdateChannel channel, OpenShock return false; } - int statusCode = response.StatusCode(); + uint16_t statusCode = response.StatusCode(); if (statusCode != 200 && statusCode != 304) { OS_LOGE(TAG, "Failed to fetch firmware version"); return false; @@ -632,7 +634,7 @@ bool OtaUpdateManager::TryGetFirmwareRelease(const OpenShock::SemVer& version, F return false; } - int statusCode = response.StatusCode(); + uint16_t statusCode = response.StatusCode(); if (statusCode != 200 && statusCode != 304) { OS_LOGE(TAG, "Failed to fetch hashes"); return false; diff --git a/src/http/HTTPClient.cpp b/src/http/HTTPClient.cpp index fb3c64a6..af5982cc 100644 --- a/src/http/HTTPClient.cpp +++ b/src/http/HTTPClient.cpp @@ -1,3 +1,5 @@ +#include + #include "http/HTTPClient.h" const char* const TAG = "HTTPClient"; diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp index ded29662..9edc006c 100644 --- a/src/http/HTTPClientState.cpp +++ b/src/http/HTTPClientState.cpp @@ -1,3 +1,5 @@ +#include + #include "http/HTTPClientState.h" const char* const TAG = "HTTPClientState"; @@ -64,9 +66,12 @@ std::variant HTTP::H } int code = esp_http_client_get_status_code(m_handle); - if (code < 0) code = 0; + if (code < 0 || code > 599) { + OS_LOGE(TAG, "Returned statusCode is invalid (%i)", code); + return HTTPError::InternalError; + } - return StartRequestResult {static_cast(code), isChunked, static_cast(contentLength)}; + return StartRequestResult {static_cast(code), isChunked, static_cast(contentLength)}; } HTTP::ReadResult HTTP::HTTPClientState::ReadStreamImpl(DownloadCallback cb) @@ -186,4 +191,6 @@ esp_err_t HTTP::HTTPClientState::EventHeaderHandler(std::string key, std::string } m_headers.emplace_back(std::move(key), std::move(value)); + + return ESP_OK; } diff --git a/src/http/RateLimiters.cpp b/src/http/RateLimiters.cpp index c19647bc..5a13b4f5 100644 --- a/src/http/RateLimiters.cpp +++ b/src/http/RateLimiters.cpp @@ -1,3 +1,5 @@ +#include + #include "http/RateLimiters.h" #include "SimpleMutex.h" diff --git a/src/serial/command_handlers/domain.cpp b/src/serial/command_handlers/domain.cpp index d03104bf..ad9df01e 100644 --- a/src/serial/command_handlers/domain.cpp +++ b/src/serial/command_handlers/domain.cpp @@ -42,7 +42,8 @@ void _handleDomainCommand(std::string_view arg, bool isAutomated) { auto content = response.ReadJson(); if (content.error != OpenShock::HTTP::HTTPError::None) { - #error TODO: Handle this + SERPR_ERROR("Tried to read response from backend, but failed (%s), refusing to save domain to config", OpenShock::HTTP::HTTPErrorToString(response.Error())); + return; } OS_LOGI(TAG, "Successfully connected to \"%.*s\", version: %s, commit: %s, current time: %s", arg.length(), arg.data(), content.data.version.c_str(), content.data.commit.c_str(), content.data.currentTime.c_str()); diff --git a/src/serialization/JsonSerial.cpp b/src/serialization/JsonSerial.cpp index e2481d40..a089b07d 100644 --- a/src/serialization/JsonSerial.cpp +++ b/src/serialization/JsonSerial.cpp @@ -22,7 +22,7 @@ bool JsonSerial::ParseShockerCommand(const cJSON* root, JsonSerial::ShockerComma OS_LOGE(TAG, "value at 'model' is not a string"); return false; } - ShockerModelType modelType; + ShockerModelType modelType = ShockerModelType::MIN; if (!ShockerModelTypeFromString(model->valuestring, modelType)) { OS_LOGE(TAG, "value at 'model' is not a valid shocker model (caixianlin, petrainer, petrainer998dr)"); return false; diff --git a/src/util/ParitionUtils.cpp b/src/util/ParitionUtils.cpp index e75a1080..ab84f4e4 100644 --- a/src/util/ParitionUtils.cpp +++ b/src/util/ParitionUtils.cpp @@ -58,7 +58,7 @@ bool OpenShock::FlashPartitionFromUrl(const esp_partition_t* partition, const ch }; // Start streaming binary to app partition. - OpenShock::HTTP::HTTPClient client(180'000); // 3 minutes timeout + HTTP::HTTPClient client(180'000); // 3 minutes timeout auto response = client.Get(remoteUrl); if (!response.Ok() || response.StatusCode() != 200 || response.StatusCode() != 304) { OS_LOGE(TAG, "Failed to download remote partition binary: [%u]", response.StatusCode()); @@ -81,7 +81,11 @@ bool OpenShock::FlashPartitionFromUrl(const esp_partition_t* partition, const ch lastProgress = OpenShock::millis(); progressCallback(0, contentLength, 0.0f); - contentLength = response.ReadStream(dataWriter); + auto streamResult = response.ReadStream(dataWriter); + if (streamResult.error != HTTP::HTTPError::None) { + OS_LOGE(TAG, "Failed to download partition: %s", HTTP::HTTPErrorToString(streamResult.error)); + return false; + } progressCallback(contentLength, contentLength, 1.0f); OS_LOGD(TAG, "Wrote %u bytes to partition", contentLength); From d83d888eb33e18388da88e0a4e973f2d80f79cc0 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 10:36:08 +0100 Subject: [PATCH 13/28] Update HTTPClientState.cpp --- src/http/HTTPClientState.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp index 9edc006c..cae696e7 100644 --- a/src/http/HTTPClientState.cpp +++ b/src/http/HTTPClientState.cpp @@ -68,7 +68,7 @@ std::variant HTTP::H int code = esp_http_client_get_status_code(m_handle); if (code < 0 || code > 599) { OS_LOGE(TAG, "Returned statusCode is invalid (%i)", code); - return HTTPError::InternalError; + return HTTPError::NetworkError; } return StartRequestResult {static_cast(code), isChunked, static_cast(contentLength)}; From 3a274f5d44ef4b9ed54346ddc9ef3c1bc2bb0e16 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 10:51:50 +0100 Subject: [PATCH 14/28] More work --- include/http/JsonAPI.h | 1 + include/http/JsonParserFn.h | 1 + include/http/RateLimiters.h | 1 + include/serialization/JsonAPI.h | 4 ++-- src/GatewayConnectionManager.cpp | 2 +- src/serialization/JsonAPI.cpp | 2 ++ 6 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/http/JsonAPI.h b/include/http/JsonAPI.h index 67b9e188..82af28f7 100644 --- a/include/http/JsonAPI.h +++ b/include/http/JsonAPI.h @@ -1,6 +1,7 @@ #pragma once #include "http/HTTPClient.h" +#include "http/JsonResponse.h" #include "serialization/JsonAPI.h" #include diff --git a/include/http/JsonParserFn.h b/include/http/JsonParserFn.h index 3a334701..40e8b6c9 100644 --- a/include/http/JsonParserFn.h +++ b/include/http/JsonParserFn.h @@ -4,6 +4,7 @@ #include class cJSON; + namespace OpenShock::HTTP { template using JsonParserFn = std::function; diff --git a/include/http/RateLimiters.h b/include/http/RateLimiters.h index 0e1991d7..cd37c726 100644 --- a/include/http/RateLimiters.h +++ b/include/http/RateLimiters.h @@ -3,6 +3,7 @@ #include "RateLimiter.h" #include +#include namespace OpenShock::HTTP::RateLimiters { std::shared_ptr GetRateLimiter(std::string_view url); diff --git a/include/serialization/JsonAPI.h b/include/serialization/JsonAPI.h index a4a17a6c..ddf3a568 100644 --- a/include/serialization/JsonAPI.h +++ b/include/serialization/JsonAPI.h @@ -2,12 +2,12 @@ #include "ShockerModelType.h" -#include - #include #include #include +class cJSON; + namespace OpenShock::Serialization::JsonAPI { struct LcgInstanceDetailsResponse { std::string name; diff --git a/src/GatewayConnectionManager.cpp b/src/GatewayConnectionManager.cpp index 040a5ad6..d299ca3c 100644 --- a/src/GatewayConnectionManager.cpp +++ b/src/GatewayConnectionManager.cpp @@ -113,7 +113,7 @@ AccountLinkResultCode GatewayConnectionManager::Link(std::string_view linkCode) return AccountLinkResultCode::InternalError; // Just return false, don't spam the console with errors } - OS_LOGE(TAG, "Error while fetching auth token: %s %d", HTTP::HTTPErrorToString(response.Error()), response.StatusCode()); + OS_LOGE(TAG, "Error while linking account: %s %d", HTTP::HTTPErrorToString(response.Error()), response.StatusCode()); return AccountLinkResultCode::InternalError; } diff --git a/src/serialization/JsonAPI.cpp b/src/serialization/JsonAPI.cpp index 215a4cac..c62c0d36 100644 --- a/src/serialization/JsonAPI.cpp +++ b/src/serialization/JsonAPI.cpp @@ -2,6 +2,8 @@ const char* const TAG = "JsonAPI"; +#include + #include "Logging.h" #define ESP_LOGJSONE(err, root) OS_LOGE(TAG, "Invalid JSON response (" err "): %s", cJSON_PrintUnformatted(root)) From bb3631605adfba69a4bb630a7120578b64a429a5 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 12:21:29 +0100 Subject: [PATCH 15/28] Arduino is and always has been a shitty ecosystem --- src/OtaUpdateManager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/OtaUpdateManager.cpp b/src/OtaUpdateManager.cpp index b0807a1b..0027ce3b 100644 --- a/src/OtaUpdateManager.cpp +++ b/src/OtaUpdateManager.cpp @@ -1,5 +1,8 @@ #include +#include +#include + #include "OtaUpdateManager.h" const char* const TAG = "OtaUpdateManager"; @@ -24,9 +27,6 @@ const char* const TAG = "OtaUpdateManager"; #include #include -#include -#include - #include #include From 0bfc3771a7f578fe58691ff01a2d8fd3d00e92cb Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 12:28:06 +0100 Subject: [PATCH 16/28] Revert nitpick --- src/util/IPAddressUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/IPAddressUtils.cpp b/src/util/IPAddressUtils.cpp index b12c38cc..15661641 100644 --- a/src/util/IPAddressUtils.cpp +++ b/src/util/IPAddressUtils.cpp @@ -15,7 +15,7 @@ bool OpenShock::IPV4AddressFromStringView(IPAddress& ip, std::string_view sv) { return false; // Must have 4 octets } - uint8_t octets[4]; + std::uint8_t octets[4]; if (!Convert::ToUint8(parts[0], octets[0]) || !Convert::ToUint8(parts[1], octets[1]) || !Convert::ToUint8(parts[2], octets[2]) || !Convert::ToUint8(parts[3], octets[3])) { return false; } From 454a508d06d74e0866c6c651a709b78fd441727f Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 12:29:47 +0100 Subject: [PATCH 17/28] Revert more nitpicking --- src/serialization/JsonSerial.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serialization/JsonSerial.cpp b/src/serialization/JsonSerial.cpp index a089b07d..e2481d40 100644 --- a/src/serialization/JsonSerial.cpp +++ b/src/serialization/JsonSerial.cpp @@ -22,7 +22,7 @@ bool JsonSerial::ParseShockerCommand(const cJSON* root, JsonSerial::ShockerComma OS_LOGE(TAG, "value at 'model' is not a string"); return false; } - ShockerModelType modelType = ShockerModelType::MIN; + ShockerModelType modelType; if (!ShockerModelTypeFromString(model->valuestring, modelType)) { OS_LOGE(TAG, "value at 'model' is not a valid shocker model (caixianlin, petrainer, petrainer998dr)"); return false; From a6bc157225d894919aaa6a9dd588f3cb75230f3d Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 14:07:54 +0100 Subject: [PATCH 18/28] Pass along headers and Retry-After --- include/http/HTTPClient.h | 14 +++++------- include/http/HTTPClientState.h | 22 +++++++++++++------ include/http/HTTPResponse.h | 19 +++++++++++++++- include/http/JsonResponse.h | 21 +++++++++++++++++- src/http/HTTPClient.cpp | 13 ++++------- src/http/HTTPClientState.cpp | 40 ++++++++++++++++++++++------------ 6 files changed, 88 insertions(+), 41 deletions(-) diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h index f24ad1b6..4ebe6119 100644 --- a/include/http/HTTPClient.h +++ b/include/http/HTTPClient.h @@ -27,27 +27,23 @@ namespace OpenShock::HTTP { inline HTTPResponse Get(const char* url) { auto response = GetInternal(url); - if (response.error != HTTPError::None) return response.error; + if (response.error != HTTPError::None) return HTTP::HTTPResponse(response.error, response.retryAfterSeconds); - return HTTP::HTTPResponse(m_state, response.data.statusCode, response.data.contentLength); + return HTTP::HTTPResponse(m_state, response.statusCode, response.contentLength, std::move(response.headers)); } template inline JsonResponse GetJson(const char* url, JsonParserFn jsonParser) { auto response = GetInternal(url); - if (response.error != HTTPError::None) return response.error; + if (response.error != HTTPError::None) return HTTP::JsonResponse(response.error, response.retryAfterSeconds); - return HTTP::JsonResponse(m_state, jsonParser, response.data.statusCode, response.data.contentLength); + return HTTP::JsonResponse(m_state, jsonParser, response.statusCode, response.contentLength, std::move(response.headers)); } inline esp_err_t Close() { return m_state->Close(); } private: - struct InternalResult { - HTTPError error; - HTTPClientState::StartRequestResult data; - }; - InternalResult GetInternal(const char* url); + HTTPClientState::StartRequestResult GetInternal(const char* url); std::shared_ptr m_state; }; diff --git a/include/http/HTTPClientState.h b/include/http/HTTPClientState.h index 966bf7b6..ba08e25b 100644 --- a/include/http/HTTPClientState.h +++ b/include/http/HTTPClientState.h @@ -10,10 +10,9 @@ #include +#include #include #include -#include -#include namespace OpenShock::HTTP { class HTTPClientState { @@ -27,13 +26,21 @@ namespace OpenShock::HTTP { return esp_http_client_set_header(m_handle, key, value); } + struct HeaderEntry { + std::string key; + std::string value; + }; + struct [[nodiscard]] StartRequestResult { - uint16_t statusCode; - bool isChunked; - uint32_t contentLength; + HTTPError error{}; + uint32_t retryAfterSeconds{}; + uint16_t statusCode{}; + bool isChunked{}; + uint32_t contentLength{}; + std::map headers{}; }; - std::variant StartRequest(esp_http_client_method_t method, const char* url, int writeLen); + StartRequestResult StartRequest(esp_http_client_method_t method, const char* url, int writeLen); // High-throughput streaming logic ReadResult ReadStreamImpl(DownloadCallback cb); @@ -72,6 +79,7 @@ namespace OpenShock::HTTP { esp_http_client_handle_t m_handle; bool m_reading; - std::vector> m_headers; + uint32_t m_retryAfterSeconds; + std::map m_headers; }; } // namespace OpenShock::HTTP diff --git a/include/http/HTTPResponse.h b/include/http/HTTPResponse.h index 240570d9..e5cdaf26 100644 --- a/include/http/HTTPResponse.h +++ b/include/http/HTTPResponse.h @@ -7,6 +7,7 @@ #include "http/ReadResult.h" #include +#include #include #include @@ -19,24 +20,38 @@ namespace OpenShock::HTTP { friend class HTTPClient; - HTTPResponse(std::shared_ptr state, uint16_t statusCode, uint32_t contentLength) + HTTPResponse(std::shared_ptr state, uint16_t statusCode, uint32_t contentLength, std::map headers) : m_state(state) , m_error(HTTPError::None) + , m_retryAfterSeconds(0) , m_statusCode(statusCode) , m_contentLength(contentLength) + , m_headers(std::move(headers)) { } public: HTTPResponse(HTTPError error) : m_state() , m_error(error) + , m_retryAfterSeconds() , m_statusCode(0) , m_contentLength(0) + , m_headers() + { + } + HTTPResponse(HTTPError error, uint32_t retryAfterSeconds) + : m_state() + , m_error(error) + , m_retryAfterSeconds(retryAfterSeconds) + , m_statusCode(0) + , m_contentLength(0) + , m_headers() { } inline bool Ok() const { return m_error == HTTPError::None && !m_state.expired(); } inline HTTPError Error() const { return m_error; } + inline uint32_t RetryAfterSeconds() const { return m_retryAfterSeconds; } inline uint16_t StatusCode() const { return m_statusCode; } inline uint32_t ContentLength() const { return m_contentLength; } @@ -65,7 +80,9 @@ namespace OpenShock::HTTP { private: std::weak_ptr m_state; HTTPError m_error; + uint32_t m_retryAfterSeconds; uint16_t m_statusCode; uint32_t m_contentLength; + std::map m_headers; }; } // namespace OpenShock::HTTP diff --git a/include/http/JsonResponse.h b/include/http/JsonResponse.h index 5af1529f..a25c7181 100644 --- a/include/http/JsonResponse.h +++ b/include/http/JsonResponse.h @@ -7,7 +7,9 @@ #include "http/ReadResult.h" #include +#include #include +#include namespace OpenShock::HTTP { class HTTPClient; @@ -19,12 +21,14 @@ namespace OpenShock::HTTP { friend class HTTPClient; - JsonResponse(std::shared_ptr state, JsonParserFn jsonParser, uint16_t statusCode, uint32_t contentLength) + JsonResponse(std::shared_ptr state, JsonParserFn jsonParser, uint16_t statusCode, uint32_t contentLength, std::map headers) : m_state(state) , m_jsonParser(jsonParser) , m_error(HTTPError::None) + , m_retryAfterSeconds(0) , m_statusCode(statusCode) , m_contentLength(contentLength) + , m_headers(std::move(headers)) { } public: @@ -32,13 +36,26 @@ namespace OpenShock::HTTP { : m_state() , m_jsonParser() , m_error(error) + , m_retryAfterSeconds(0) , m_statusCode(0) , m_contentLength(0) + , m_headers() + { + } + JsonResponse(HTTPError error, uint32_t retryAfterSeconds) + : m_state() + , m_jsonParser() + , m_error(error) + , m_retryAfterSeconds(retryAfterSeconds) + , m_statusCode(0) + , m_contentLength(0) + , m_headers() { } inline bool Ok() const { return m_error == HTTPError::None && !m_state.expired(); } inline HTTPError Error() const { return m_error; } + inline uint32_t RetryAfterSeconds() const { return m_retryAfterSeconds; } inline uint16_t StatusCode() const { return m_statusCode; } inline uint32_t ContentLength() const { return m_contentLength; } @@ -53,7 +70,9 @@ namespace OpenShock::HTTP { std::weak_ptr m_state; JsonParserFn m_jsonParser; HTTPError m_error; + uint32_t m_retryAfterSeconds; uint16_t m_statusCode; uint32_t m_contentLength; + std::map m_headers; }; } // namespace OpenShock::HTTP diff --git a/src/http/HTTPClient.cpp b/src/http/HTTPClient.cpp index af5982cc..bd6807bd 100644 --- a/src/http/HTTPClient.cpp +++ b/src/http/HTTPClient.cpp @@ -11,23 +11,18 @@ const char* const TAG = "HTTPClient"; using namespace OpenShock; -HTTP::HTTPClient::InternalResult HTTP::HTTPClient::GetInternal(const char* url) { +HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClient::GetInternal(const char* url) { auto ratelimiter = HTTP::RateLimiters::GetRateLimiter(url); if (ratelimiter == nullptr) { OS_LOGW(TAG, "Invalid URL!"); - return {HTTPError::InvalidUrl, {}}; + return { .error = HTTPError::InvalidUrl }; } if (!ratelimiter->tryRequest()) { OS_LOGW(TAG, "Hit ratelimit, refusing to send request!"); - return {HTTPError::RateLimited, {}}; + return { .error = HTTPError::RateLimited }; } - auto result = m_state->StartRequest(HTTP_METHOD_GET, url, 0); - if (auto error = std::get_if(&result)) { - return {*error, {}}; - } - - return {HTTPError::None, std::get(result)}; + return m_state->StartRequest(HTTP_METHOD_GET, url, 0); } diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp index cae696e7..2550a7cc 100644 --- a/src/http/HTTPClientState.cpp +++ b/src/http/HTTPClientState.cpp @@ -18,7 +18,8 @@ using namespace OpenShock; HTTP::HTTPClientState::HTTPClientState(uint32_t timeoutMs) : m_handle(nullptr) , m_reading(false) - , m_headers(false) + , m_retryAfterSeconds(0) + , m_headers() { esp_http_client_config_t cfg; memset(&cfg, 0, sizeof(cfg)); @@ -44,34 +45,45 @@ HTTP::HTTPClientState::~HTTPClientState() } } -std::variant HTTP::HTTPClientState::StartRequest(esp_http_client_method_t method, const char* url, int writeLen) +HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClientState::StartRequest(esp_http_client_method_t method, const char* url, int writeLen) { esp_err_t err; + m_headers.clear(); + err = esp_http_client_set_url(m_handle, url); - if (err != ESP_OK) return HTTPError::InvalidUrl; + if (err != ESP_OK) return { .error = HTTPError::InvalidUrl }; err = esp_http_client_set_method(m_handle, method); - if (err != ESP_OK) return HTTPError::InvalidHttpMethod; + if (err != ESP_OK) return { .error = HTTPError::InvalidHttpMethod }; err = esp_http_client_open(m_handle, writeLen); - if (err != ESP_OK) return HTTPError::NetworkError; + if (err != ESP_OK) return { .error = HTTPError::NetworkError }; int contentLength = esp_http_client_fetch_headers(m_handle); - if (contentLength == ESP_FAIL) return HTTPError::NetworkError; + if (contentLength == ESP_FAIL) return { .error = HTTPError::NetworkError }; + + if (m_retryAfterSeconds > 0) { + return { .error = HTTPError::RateLimited, .retryAfterSeconds = m_retryAfterSeconds }; + } bool isChunked = false; if (contentLength == 0) { isChunked = esp_http_client_is_chunked_response(m_handle); } - int code = esp_http_client_get_status_code(m_handle); - if (code < 0 || code > 599) { - OS_LOGE(TAG, "Returned statusCode is invalid (%i)", code); - return HTTPError::NetworkError; + int statusCode = esp_http_client_get_status_code(m_handle); + if (statusCode < 0 || statusCode > 599) { + OS_LOGE(TAG, "Returned statusCode is invalid (%i)", statusCode); + return { .error = HTTPError::NetworkError }; } - return StartRequestResult {static_cast(code), isChunked, static_cast(contentLength)}; + return StartRequestResult { + .statusCode = static_cast(statusCode), + .isChunked = isChunked, + .contentLength = static_cast(contentLength), + .headers = std::move(m_headers) + }; } HTTP::ReadResult HTTP::HTTPClientState::ReadStreamImpl(DownloadCallback cb) @@ -181,16 +193,16 @@ esp_err_t HTTP::HTTPClientState::EventHeaderHandler(std::string key, std::string OS_LOGI(TAG, "Got header_received event: %.*s - %.*s", key.length(), key.c_str(), key.length(), key.c_str()); if (key == "Retry-After") { - uint32_t seconds; + uint32_t seconds = 0; if (!Convert::ToUint32(value, seconds) || seconds <= 0) { seconds = 15; } OS_LOGI(TAG, "Retry-After: %d seconds, applying delay to rate limiter", seconds); - // TODO: Inform caller + m_retryAfterSeconds = seconds; } - m_headers.emplace_back(std::move(key), std::move(value)); + m_headers[key] = std::move(value); return ESP_OK; } From 7e79d41727057d07e9c856c5c15821f2d91c3a39 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 14:34:48 +0100 Subject: [PATCH 19/28] Fix more issues --- include/http/HTTPError.h | 23 ++++++++++++----------- src/http/HTTPClientState.cpp | 17 ++++++++++++++--- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/include/http/HTTPError.h b/include/http/HTTPError.h index 808557bb..93e0bb3e 100644 --- a/include/http/HTTPError.h +++ b/include/http/HTTPError.h @@ -3,6 +3,7 @@ namespace OpenShock::HTTP { enum class HTTPError { None, + ClientBusy, InternalError, RateLimited, InvalidUrl, @@ -18,27 +19,27 @@ namespace OpenShock::HTTP { switch (error) { case HTTPError::None: - return ""; + return "None"; case HTTPError::InternalError: - return ""; + return "InternalError"; case HTTPError::RateLimited: - return ""; + return "RateLimited"; case HTTPError::InvalidUrl: - return ""; + return "InvalidUrl"; case HTTPError::InvalidHttpMethod: - return ""; + return "InvalidHttpMethod"; case HTTPError::NetworkError: - return ""; + return "NetworkError"; case HTTPError::ConnectionClosed: - return ""; + return "ConnectionClosed"; case HTTPError::SizeLimitExceeded: - return ""; + return "SizeLimitExceeded"; case HTTPError::Aborted: - return ""; + return "Aborted"; case HTTPError::ParseFailed: - return ""; + return "ParseFailed"; default: - return ""; + return "Unknown"; } } } // namespace OpenShock::HTTP diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp index 2550a7cc..038477bf 100644 --- a/src/http/HTTPClientState.cpp +++ b/src/http/HTTPClientState.cpp @@ -49,6 +49,11 @@ HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClientState::StartRequest(es { esp_err_t err; + if (m_reading) { + return { .error = HTTPError::ClientBusy }; + } + + m_retryAfterSeconds = 0; m_headers.clear(); err = esp_http_client_set_url(m_handle, url); @@ -61,10 +66,12 @@ HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClientState::StartRequest(es if (err != ESP_OK) return { .error = HTTPError::NetworkError }; int contentLength = esp_http_client_fetch_headers(m_handle); - if (contentLength == ESP_FAIL) return { .error = HTTPError::NetworkError }; + if (contentLength < 0) return { .error = HTTPError::NetworkError }; if (m_retryAfterSeconds > 0) { - return { .error = HTTPError::RateLimited, .retryAfterSeconds = m_retryAfterSeconds }; + uint32_t retryAfterSeconds = m_retryAfterSeconds; + m_retryAfterSeconds = 0; + return { .error = HTTPError::RateLimited, .retryAfterSeconds = retryAfterSeconds }; } bool isChunked = false; @@ -78,6 +85,8 @@ HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClientState::StartRequest(es return { .error = HTTPError::NetworkError }; } + m_reading = true; + return StartRequestResult { .statusCode = static_cast(statusCode), .isChunked = isChunked, @@ -192,7 +201,9 @@ esp_err_t HTTP::HTTPClientState::EventHeaderHandler(std::string key, std::string { OS_LOGI(TAG, "Got header_received event: %.*s - %.*s", key.length(), key.c_str(), key.length(), key.c_str()); - if (key == "Retry-After") { + std::transform(key.begin(), key.end(), key.begin(), [](unsigned char c) { return std::tolower(c); }); + + if (key == "retry-after") { uint32_t seconds = 0; if (!Convert::ToUint32(value, seconds) || seconds <= 0) { seconds = 15; From 614e4c859b97b17e75a9cbd3e44fb33f62db8ece Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 14:45:56 +0100 Subject: [PATCH 20/28] Last touchup --- include/http/GotContentLengthCallback.h | 8 -------- include/http/HTTPError.h | 2 ++ src/http/HTTPClientState.cpp | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) delete mode 100644 include/http/GotContentLengthCallback.h diff --git a/include/http/GotContentLengthCallback.h b/include/http/GotContentLengthCallback.h deleted file mode 100644 index ab4ac6d5..00000000 --- a/include/http/GotContentLengthCallback.h +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once - -#include -#include - -namespace OpenShock::HTTP { - using GotContentLengthCallback = std::function; -} diff --git a/include/http/HTTPError.h b/include/http/HTTPError.h index 93e0bb3e..37d3ef37 100644 --- a/include/http/HTTPError.h +++ b/include/http/HTTPError.h @@ -20,6 +20,8 @@ namespace OpenShock::HTTP { { case HTTPError::None: return "None"; + case HTTPError::ClientBusy: + return "ClientBusy"; case HTTPError::InternalError: return "InternalError"; case HTTPError::RateLimited: diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp index 038477bf..dbcbaafa 100644 --- a/src/http/HTTPClientState.cpp +++ b/src/http/HTTPClientState.cpp @@ -199,7 +199,7 @@ esp_err_t HTTP::HTTPClientState::EventHandler(esp_http_client_event_t* evt) esp_err_t HTTP::HTTPClientState::EventHeaderHandler(std::string key, std::string value) { - OS_LOGI(TAG, "Got header_received event: %.*s - %.*s", key.length(), key.c_str(), key.length(), key.c_str()); + OS_LOGI(TAG, "Got header_received event: %.*s - %.*s", key.length(), key.c_str(), value.length(), value.c_str()); std::transform(key.begin(), key.end(), key.begin(), [](unsigned char c) { return std::tolower(c); }); From 68cf5301f1489298347c87e30df9306a603ae76e Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 14:46:18 +0100 Subject: [PATCH 21/28] Update include/http/HTTPClient.h Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- include/http/HTTPClient.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h index 4ebe6119..84a726c5 100644 --- a/include/http/HTTPClient.h +++ b/include/http/HTTPClient.h @@ -5,7 +5,7 @@ #include "http/HTTPResponse.h" #include "http/JsonResponse.h" -#include +#include #include #include From b59e4b9d307d172da44262207f256d4b3a1a868f Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 14:49:50 +0100 Subject: [PATCH 22/28] Update src/util/DomainUtils.cpp Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/util/DomainUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/DomainUtils.cpp b/src/util/DomainUtils.cpp index cc57e720..7bc4b657 100644 --- a/src/util/DomainUtils.cpp +++ b/src/util/DomainUtils.cpp @@ -8,7 +8,7 @@ std::string_view OpenShock::DomainUtils::GetDomainFromUrl(std::string_view url) // Remove the protocol eg. "https://api.example.com:443/path" -> "api.example.com:443/path" auto seperator = url.find("://"); if (seperator != std::string_view::npos) { - url.substr(seperator + 3); + url = url.substr(seperator + 3); } // Remove the path eg. "api.example.com:443/path" -> "api.example.com:443" From 435e6ef6061c80fc00e62a3b24d82b2c1e260571 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 14:50:51 +0100 Subject: [PATCH 23/28] Update src/util/DomainUtils.cpp Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/util/DomainUtils.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/util/DomainUtils.cpp b/src/util/DomainUtils.cpp index 7bc4b657..f79cf1e1 100644 --- a/src/util/DomainUtils.cpp +++ b/src/util/DomainUtils.cpp @@ -6,31 +6,31 @@ std::string_view OpenShock::DomainUtils::GetDomainFromUrl(std::string_view url) } // Remove the protocol eg. "https://api.example.com:443/path" -> "api.example.com:443/path" - auto seperator = url.find("://"); - if (seperator != std::string_view::npos) { - url = url.substr(seperator + 3); + auto separator = url.find("://"); + if (separator != std::string_view::npos) { + url.substr(separator + 3); } // Remove the path eg. "api.example.com:443/path" -> "api.example.com:443" - seperator = url.find('/'); - if (seperator != std::string_view::npos) { - url = url.substr(0, seperator); + separator = url.find('/'); + if (separator != std::string_view::npos) { + url = url.substr(0, separator); } // Remove the port eg. "api.example.com:443" -> "api.example.com" - seperator = url.rfind(':'); - if (seperator != std::string_view::npos) { - url = url.substr(0, seperator); + separator = url.rfind(':'); + if (separator != std::string_view::npos) { + url = url.substr(0, separator); } // Remove all subdomains eg. "api.example.com" -> "example.com" - seperator = url.rfind('.'); - if (seperator == std::string_view::npos) { + separator = url.rfind('.'); + if (separator == std::string_view::npos) { return url; // E.g. "localhost" } - seperator = url.rfind('.', seperator - 1); - if (seperator != std::string_view::npos) { - url = url.substr(seperator + 1); + separator = url.rfind('.', separator - 1); + if (separator != std::string_view::npos) { + url = url.substr(separator + 1); } return url; From 3579ceeb69404aadb380918c9d73920f8a61bdd9 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 14:51:57 +0100 Subject: [PATCH 24/28] Update src/serial/command_handlers/authtoken.cpp Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/serial/command_handlers/authtoken.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serial/command_handlers/authtoken.cpp b/src/serial/command_handlers/authtoken.cpp index f962bb9b..9524a890 100644 --- a/src/serial/command_handlers/authtoken.cpp +++ b/src/serial/command_handlers/authtoken.cpp @@ -21,7 +21,7 @@ void _handleAuthtokenCommand(std::string_view arg, bool isAutomated) { std::string token = std::string(arg); - // Scope to immidiatley destroy client after use + // Scope to immediately destroy client after use { OpenShock::HTTP::HTTPClient client; auto apiResponse = OpenShock::HTTP::JsonAPI::GetHubInfo(client, token.c_str()); From c9f2094b28f04f071605f94aa8db3078f73c0ce5 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 5 Dec 2025 14:56:13 +0100 Subject: [PATCH 25/28] Update src/util/ParitionUtils.cpp Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/util/ParitionUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/ParitionUtils.cpp b/src/util/ParitionUtils.cpp index ab84f4e4..66c0d241 100644 --- a/src/util/ParitionUtils.cpp +++ b/src/util/ParitionUtils.cpp @@ -60,7 +60,7 @@ bool OpenShock::FlashPartitionFromUrl(const esp_partition_t* partition, const ch // Start streaming binary to app partition. HTTP::HTTPClient client(180'000); // 3 minutes timeout auto response = client.Get(remoteUrl); - if (!response.Ok() || response.StatusCode() != 200 || response.StatusCode() != 304) { + if (!response.Ok() || (response.StatusCode() != 200 && response.StatusCode() != 304)) { OS_LOGE(TAG, "Failed to download remote partition binary: [%u]", response.StatusCode()); return false; } From da86842d539222659895c7343602ad97609ce77e Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Mon, 8 Dec 2025 03:35:47 +0100 Subject: [PATCH 26/28] Fixed url assignment issues --- include/http/HTTPClient.h | 19 +++++++------ include/http/HTTPClientState.h | 11 ++++---- include/http/JsonAPI.h | 6 ++--- platformio.ini | 2 ++ src/GatewayConnectionManager.cpp | 9 +++---- src/OtaUpdateManager.cpp | 12 ++++----- src/http/HTTPClient.cpp | 28 ------------------- src/http/HTTPClientState.cpp | 33 ++++++++++++++++++----- src/http/JsonAPI.cpp | 18 ++++++++----- src/serial/command_handlers/authtoken.cpp | 12 +++------ src/serial/command_handlers/domain.cpp | 4 +-- src/util/ParitionUtils.cpp | 4 +-- 12 files changed, 77 insertions(+), 81 deletions(-) delete mode 100644 src/http/HTTPClient.cpp diff --git a/include/http/HTTPClient.h b/include/http/HTTPClient.h index 84a726c5..77388283 100644 --- a/include/http/HTTPClient.h +++ b/include/http/HTTPClient.h @@ -4,6 +4,7 @@ #include "http/HTTPClientState.h" #include "http/HTTPResponse.h" #include "http/JsonResponse.h" +#include "RateLimiter.h" #include @@ -16,24 +17,28 @@ namespace OpenShock::HTTP { DISABLE_MOVE(HTTPClient); public: - HTTPClient(uint32_t timeoutMs = 10'000) - : m_state(std::make_shared(timeoutMs)) + HTTPClient(const char* url, uint32_t timeoutMs = 10'000) + : m_state(std::make_shared(url, timeoutMs)) { } + inline esp_err_t SetUrl(const char* url) { + return m_state->SetUrl(url); + } + inline esp_err_t SetHeader(const char* key, const char* value) { return m_state->SetHeader(key, value); } - inline HTTPResponse Get(const char* url) { - auto response = GetInternal(url); + inline HTTPResponse Get() { + auto response = m_state->StartRequest(HTTP_METHOD_GET, 0); if (response.error != HTTPError::None) return HTTP::HTTPResponse(response.error, response.retryAfterSeconds); return HTTP::HTTPResponse(m_state, response.statusCode, response.contentLength, std::move(response.headers)); } template - inline JsonResponse GetJson(const char* url, JsonParserFn jsonParser) { - auto response = GetInternal(url); + inline JsonResponse GetJson(JsonParserFn jsonParser) { + auto response = m_state->StartRequest(HTTP_METHOD_GET, 0); if (response.error != HTTPError::None) return HTTP::JsonResponse(response.error, response.retryAfterSeconds); return HTTP::JsonResponse(m_state, jsonParser, response.statusCode, response.contentLength, std::move(response.headers)); @@ -43,8 +48,6 @@ namespace OpenShock::HTTP { return m_state->Close(); } private: - HTTPClientState::StartRequestResult GetInternal(const char* url); - std::shared_ptr m_state; }; } // namespace OpenShock::HTTP diff --git a/include/http/HTTPClientState.h b/include/http/HTTPClientState.h index ba08e25b..f1f79c5d 100644 --- a/include/http/HTTPClientState.h +++ b/include/http/HTTPClientState.h @@ -19,12 +19,12 @@ namespace OpenShock::HTTP { DISABLE_COPY(HTTPClientState); DISABLE_MOVE(HTTPClientState); public: - HTTPClientState(uint32_t timeoutMs); + HTTPClientState(const char* url, uint32_t timeoutMs); ~HTTPClientState(); - inline esp_err_t SetHeader(const char* key, const char* value) { - return esp_http_client_set_header(m_handle, key, value); - } + esp_err_t SetUrl(const char* url); + + esp_err_t SetHeader(const char* key, const char* value); struct HeaderEntry { std::string key; @@ -40,7 +40,7 @@ namespace OpenShock::HTTP { std::map headers{}; }; - StartRequestResult StartRequest(esp_http_client_method_t method, const char* url, int writeLen); + StartRequestResult StartRequest(esp_http_client_method_t method, int writeLen); // High-throughput streaming logic ReadResult ReadStreamImpl(DownloadCallback cb); @@ -71,6 +71,7 @@ namespace OpenShock::HTTP { } inline esp_err_t Close() { + if (m_handle == nullptr) return ESP_FAIL; return esp_http_client_close(m_handle); } private: diff --git a/include/http/JsonAPI.h b/include/http/JsonAPI.h index 82af28f7..258b83ce 100644 --- a/include/http/JsonAPI.h +++ b/include/http/JsonAPI.h @@ -10,15 +10,15 @@ namespace OpenShock::HTTP::JsonAPI { /// @brief Links the hub to the account with the given account link code, returns the hub token. Valid response codes: 200, 404 /// @param hubToken /// @return - JsonResponse LinkAccount(HTTP::HTTPClient& client, std::string_view accountLinkCode); + JsonResponse LinkAccount(std::string_view accountLinkCode); /// @brief Gets the hub info for the given hub token. Valid response codes: 200, 401 /// @param hubToken /// @return - JsonResponse GetHubInfo(HTTP::HTTPClient& client, const char* hubToken); + JsonResponse GetHubInfo(const char* hubToken); /// @brief Requests a Live Control Gateway to connect to. Valid response codes: 200, 401 /// @param hubToken /// @return - JsonResponse AssignLcg(HTTP::HTTPClient& client, const char* hubToken); + JsonResponse AssignLcg(const char* hubToken); } // namespace OpenShock::HTTP::JsonAPI diff --git a/platformio.ini b/platformio.ini index 2d686c1f..026460df 100644 --- a/platformio.ini +++ b/platformio.ini @@ -44,6 +44,8 @@ build_flags = -Wno-unused -Wno-unknown-pragmas -DCONFIG_ASYNC_TCP_QUEUE_SIZE=256 + -DCONFIG_ESP_TLS_INSECURE=y + -DCONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y build_unflags = -std=gnu++11 lib_deps = diff --git a/src/GatewayConnectionManager.cpp b/src/GatewayConnectionManager.cpp index d299ca3c..d3da4636 100644 --- a/src/GatewayConnectionManager.cpp +++ b/src/GatewayConnectionManager.cpp @@ -105,8 +105,7 @@ AccountLinkResultCode GatewayConnectionManager::Link(std::string_view linkCode) return AccountLinkResultCode::InvalidCode; } - HTTP::HTTPClient client; - auto response = HTTP::JsonAPI::LinkAccount(client, linkCode); + auto response = HTTP::JsonAPI::LinkAccount(linkCode); if (!response.Ok()) { if (response.Error() == HTTP::HTTPError::RateLimited) { @@ -183,8 +182,7 @@ bool FetchHubInfo(const char* authToken) return false; } - HTTP::HTTPClient client; - auto response = HTTP::JsonAPI::GetHubInfo(client, authToken); + auto response = HTTP::JsonAPI::GetHubInfo(authToken); if (!response.Ok()) { if (response.Error() == HTTP::HTTPError::RateLimited) { return false; // Just return false, don't spam the console with errors @@ -254,8 +252,7 @@ bool StartConnectingToLCG() return false; } - HTTP::HTTPClient client; - auto response = HTTP::JsonAPI::AssignLcg(client, authToken.c_str()); + auto response = HTTP::JsonAPI::AssignLcg(authToken.c_str()); if (!response.Ok()) { if (response.Error() == HTTP::HTTPError::RateLimited) { return false; // Just return false, don't spam the console with errors diff --git a/src/OtaUpdateManager.cpp b/src/OtaUpdateManager.cpp index 0027ce3b..b366996b 100644 --- a/src/OtaUpdateManager.cpp +++ b/src/OtaUpdateManager.cpp @@ -426,8 +426,8 @@ static void otaum_updatetask(void* arg) static bool _tryGetStringList(const char* url, std::vector& list) { - HTTP::HTTPClient client; - auto response = client.Get(url); + HTTP::HTTPClient client(url); + auto response = client.Get(); if (!response.Ok()) { OS_LOGE(TAG, "Failed to fetch list"); return false; @@ -550,8 +550,8 @@ bool OtaUpdateManager::TryGetFirmwareVersion(OtaUpdateChannel channel, OpenShock OS_LOGD(TAG, "Fetching firmware version from %s", channelIndexUrl); - HTTP::HTTPClient client; - auto response = client.Get(channelIndexUrl); + HTTP::HTTPClient client(channelIndexUrl); + auto response = client.Get(); if (!response.Ok()) { OS_LOGE(TAG, "Failed to fetch firmware version"); return false; @@ -627,8 +627,8 @@ bool OtaUpdateManager::TryGetFirmwareRelease(const OpenShock::SemVer& version, F } // Fetch hashes. - HTTP::HTTPClient client; - auto response = client.Get(sha256HashesUrl.c_str()); + HTTP::HTTPClient client(sha256HashesUrl.c_str()); + auto response = client.Get(); if (!response.Ok()) { OS_LOGE(TAG, "Failed to fetch hashes"); return false; diff --git a/src/http/HTTPClient.cpp b/src/http/HTTPClient.cpp deleted file mode 100644 index bd6807bd..00000000 --- a/src/http/HTTPClient.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include - -#include "http/HTTPClient.h" - -const char* const TAG = "HTTPClient"; - -#include "Convert.h" -#include "http/RateLimiters.h" -#include "Logging.h" -#include "util/DomainUtils.h" - -using namespace OpenShock; - -HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClient::GetInternal(const char* url) { - auto ratelimiter = HTTP::RateLimiters::GetRateLimiter(url); - if (ratelimiter == nullptr) { - OS_LOGW(TAG, "Invalid URL!"); - return { .error = HTTPError::InvalidUrl }; - } - - if (!ratelimiter->tryRequest()) { - OS_LOGW(TAG, "Hit ratelimit, refusing to send request!"); - return { .error = HTTPError::RateLimited }; - } - - - return m_state->StartRequest(HTTP_METHOD_GET, url, 0); -} diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp index dbcbaafa..5165fc4a 100644 --- a/src/http/HTTPClientState.cpp +++ b/src/http/HTTPClientState.cpp @@ -15,7 +15,7 @@ static const uint32_t HTTP_DOWNLOAD_SIZE_LIMIT = 200 * 1024 * 1024; // 200 MB using namespace OpenShock; -HTTP::HTTPClientState::HTTPClientState(uint32_t timeoutMs) +HTTP::HTTPClientState::HTTPClientState(const char* url, uint32_t timeoutMs) : m_handle(nullptr) , m_reading(false) , m_retryAfterSeconds(0) @@ -24,6 +24,7 @@ HTTP::HTTPClientState::HTTPClientState(uint32_t timeoutMs) esp_http_client_config_t cfg; memset(&cfg, 0, sizeof(cfg)); + cfg.url = url; cfg.user_agent = OpenShock::Constants::FW_USERAGENT; cfg.timeout_ms = static_cast(std::min(timeoutMs, INT32_MAX)); cfg.disable_auto_redirect = true; @@ -31,8 +32,7 @@ HTTP::HTTPClientState::HTTPClientState(uint32_t timeoutMs) cfg.transport_type = HTTP_TRANSPORT_OVER_SSL; cfg.user_data = reinterpret_cast(this); cfg.is_async = false; - cfg.use_global_ca_store = true; - #warning This still uses SSL, upgrade to TLS! (latest ESP-IDF) + cfg.use_global_ca_store = false; m_handle = esp_http_client_init(&cfg); } @@ -45,8 +45,30 @@ HTTP::HTTPClientState::~HTTPClientState() } } -HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClientState::StartRequest(esp_http_client_method_t method, const char* url, int writeLen) +esp_err_t HTTP::HTTPClientState::SetUrl(const char* url) { + if (m_handle == nullptr) { + return ESP_FAIL; + } + + return esp_http_client_set_url(m_handle, url); +} + +esp_err_t HTTP::HTTPClientState::SetHeader(const char* key, const char* value) +{ + if (m_handle == nullptr) { + return ESP_FAIL; + } + + return esp_http_client_set_header(m_handle, key, value); +} + +HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClientState::StartRequest(esp_http_client_method_t method, int writeLen) +{ + if (m_handle == nullptr) { + return { .error = HTTPError::ConnectionClosed }; + } + esp_err_t err; if (m_reading) { @@ -56,9 +78,6 @@ HTTP::HTTPClientState::StartRequestResult HTTP::HTTPClientState::StartRequest(es m_retryAfterSeconds = 0; m_headers.clear(); - err = esp_http_client_set_url(m_handle, url); - if (err != ESP_OK) return { .error = HTTPError::InvalidUrl }; - err = esp_http_client_set_method(m_handle, method); if (err != ESP_OK) return { .error = HTTPError::InvalidHttpMethod }; diff --git a/src/http/JsonAPI.cpp b/src/http/JsonAPI.cpp index dd332e7d..0b563c78 100644 --- a/src/http/JsonAPI.cpp +++ b/src/http/JsonAPI.cpp @@ -7,7 +7,7 @@ using namespace OpenShock; -HTTP::JsonResponse HTTP::JsonAPI::LinkAccount(HTTP::HTTPClient& client, std::string_view accountLinkCode) +HTTP::JsonResponse HTTP::JsonAPI::LinkAccount(std::string_view accountLinkCode) { std::string domain; if (!Config::GetBackendDomain(domain)) { @@ -17,12 +17,14 @@ HTTP::JsonResponse HTTP::JsonAPI::L char uri[OPENSHOCK_URI_BUFFER_SIZE]; sprintf(uri, "https://%s/1/device/pair/%.*s", domain.c_str(), accountLinkCode.length(), accountLinkCode.data()); + HTTP::HTTPClient client(uri); + client.SetHeader("Accept", "application/json"); - return client.GetJson(uri, Serialization::JsonAPI::ParseAccountLinkJsonResponse); + return client.GetJson(Serialization::JsonAPI::ParseAccountLinkJsonResponse); } -HTTP::JsonResponse HTTP::JsonAPI::GetHubInfo(HTTP::HTTPClient& client, const char* hubToken) +HTTP::JsonResponse HTTP::JsonAPI::GetHubInfo(const char* hubToken) { std::string domain; if (!Config::GetBackendDomain(domain)) { @@ -32,13 +34,15 @@ HTTP::JsonResponse HTTP::JsonAPI::GetHu char uri[OPENSHOCK_URI_BUFFER_SIZE]; sprintf(uri, "https://%s/1/device/self", domain.c_str()); + HTTP::HTTPClient client(uri); + client.SetHeader("Accept", "application/json"); client.SetHeader("DeviceToken", hubToken); - return client.GetJson(uri, Serialization::JsonAPI::ParseHubInfoJsonResponse); + return client.GetJson(Serialization::JsonAPI::ParseHubInfoJsonResponse); } -HTTP::JsonResponse HTTP::JsonAPI::AssignLcg(HTTP::HTTPClient& client, const char* hubToken) +HTTP::JsonResponse HTTP::JsonAPI::AssignLcg(const char* hubToken) { std::string domain; if (!Config::GetBackendDomain(domain)) { @@ -48,8 +52,10 @@ HTTP::JsonResponse HTTP::JsonAPI::Ass char uri[OPENSHOCK_URI_BUFFER_SIZE]; sprintf(uri, "https://%s/2/device/assignLCG?version=2", domain.c_str()); + HTTP::HTTPClient client(uri); + client.SetHeader("Accept", "application/json"); client.SetHeader("DeviceToken", hubToken); - return client.GetJson(uri, Serialization::JsonAPI::ParseAssignLcgJsonResponse); + return client.GetJson(Serialization::JsonAPI::ParseAssignLcgJsonResponse); } diff --git a/src/serial/command_handlers/authtoken.cpp b/src/serial/command_handlers/authtoken.cpp index 9524a890..ff8b39a8 100644 --- a/src/serial/command_handlers/authtoken.cpp +++ b/src/serial/command_handlers/authtoken.cpp @@ -21,15 +21,11 @@ void _handleAuthtokenCommand(std::string_view arg, bool isAutomated) { std::string token = std::string(arg); - // Scope to immediately destroy client after use - { - OpenShock::HTTP::HTTPClient client; - auto apiResponse = OpenShock::HTTP::JsonAPI::GetHubInfo(client, token.c_str()); + auto apiResponse = OpenShock::HTTP::JsonAPI::GetHubInfo(token.c_str()); - if (apiResponse.StatusCode() == 401) { - SERPR_ERROR("Invalid auth token, refusing to save it!"); - return; - } + if (apiResponse.StatusCode() == 401) { + SERPR_ERROR("Invalid auth token, refusing to save it!"); + return; } // If we have some other kind of request fault just set it anyway, we probably arent connected to a network diff --git a/src/serial/command_handlers/domain.cpp b/src/serial/command_handlers/domain.cpp index ad9df01e..7671c9ef 100644 --- a/src/serial/command_handlers/domain.cpp +++ b/src/serial/command_handlers/domain.cpp @@ -33,8 +33,8 @@ void _handleDomainCommand(std::string_view arg, bool isAutomated) { char uri[OPENSHOCK_URI_BUFFER_SIZE]; sprintf(uri, "https://%.*s/1", arg.length(), arg.data()); - OpenShock::HTTP::HTTPClient client; - auto response = client.GetJson(uri, OpenShock::Serialization::JsonAPI::ParseBackendVersionJsonResponse); + OpenShock::HTTP::HTTPClient client(uri); + auto response = client.GetJson(OpenShock::Serialization::JsonAPI::ParseBackendVersionJsonResponse); if (!response.Ok() || response.StatusCode() != 200) { SERPR_ERROR("Tried to connect to \"%.*s\", but failed with status [%d] (%s), refusing to save domain to config", arg.length(), arg.data(), response.StatusCode(), OpenShock::HTTP::HTTPErrorToString(response.Error())); return; diff --git a/src/util/ParitionUtils.cpp b/src/util/ParitionUtils.cpp index 66c0d241..b6d43f41 100644 --- a/src/util/ParitionUtils.cpp +++ b/src/util/ParitionUtils.cpp @@ -58,8 +58,8 @@ bool OpenShock::FlashPartitionFromUrl(const esp_partition_t* partition, const ch }; // Start streaming binary to app partition. - HTTP::HTTPClient client(180'000); // 3 minutes timeout - auto response = client.Get(remoteUrl); + HTTP::HTTPClient client(remoteUrl, 180'000); // 3 minutes timeout + auto response = client.Get(); if (!response.Ok() || (response.StatusCode() != 200 && response.StatusCode() != 304)) { OS_LOGE(TAG, "Failed to download remote partition binary: [%u]", response.StatusCode()); return false; From 3de2c163998c6422bbf20cc676ffb43bfcda07c8 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Mon, 15 Dec 2025 18:13:50 +0100 Subject: [PATCH 27/28] Update platformio.ini Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- platformio.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/platformio.ini b/platformio.ini index 026460df..2d686c1f 100644 --- a/platformio.ini +++ b/platformio.ini @@ -44,8 +44,6 @@ build_flags = -Wno-unused -Wno-unknown-pragmas -DCONFIG_ASYNC_TCP_QUEUE_SIZE=256 - -DCONFIG_ESP_TLS_INSECURE=y - -DCONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y build_unflags = -std=gnu++11 lib_deps = From 6b85a36902c26069c911d9ce31a8ebe25fee8fae Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Wed, 17 Dec 2025 14:02:56 +0100 Subject: [PATCH 28/28] Experiement with global CA store --- src/http/HTTPClientState.cpp | 2 +- src/main.cpp | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/http/HTTPClientState.cpp b/src/http/HTTPClientState.cpp index 5165fc4a..7e178684 100644 --- a/src/http/HTTPClientState.cpp +++ b/src/http/HTTPClientState.cpp @@ -32,7 +32,7 @@ HTTP::HTTPClientState::HTTPClientState(const char* url, uint32_t timeoutMs) cfg.transport_type = HTTP_TRANSPORT_OVER_SSL; cfg.user_data = reinterpret_cast(this); cfg.is_async = false; - cfg.use_global_ca_store = false; + cfg.use_global_ca_store = true; m_handle = esp_http_client_init(&cfg); } diff --git a/src/main.cpp b/src/main.cpp index 153f4988..49c56aa7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -19,6 +19,8 @@ const char* const TAG = "main"; #include +#include + #include // Internal setup function, returns true if setup succeeded, false otherwise. @@ -89,11 +91,17 @@ void appSetup() } } +extern const uint8_t* global_ca_crt_bundle_start asm("_binary_certificates_x509_crt_bundle_start"); +extern const uint8_t* global_ca_crt_bundle_end asm("_binary_certificates_x509_crt_bundle_end"); + // Arduino setup function void setup() { ::Serial.begin(115'200); + esp_tls_init_global_ca_store(); + esp_tls_set_global_ca_store(global_ca_crt_bundle_start, static_cast(global_ca_crt_bundle_end - global_ca_crt_bundle_start)); + OpenShock::Config::Init(); if (!OpenShock::Events::Init()) {