From 02987770d2aa50795710481f2f7816c9929c5700 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 31 Oct 2025 14:49:51 -0400 Subject: [PATCH 1/5] impr: catch `error` returns from HTTP client --- src/hb_http.erl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/hb_http.erl b/src/hb_http.erl index 5b1f5e2b5..273c3b54d 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -104,7 +104,15 @@ request(Method, Peer, Path, RawMessage, Opts) -> ), StartTime = os:system_time(millisecond), % Perform the HTTP request. - {_ErlStatus, Status, Headers, Body} = hb_http_client:request(Req, Opts), + Res = hb_http_client:request(Req, Opts), + process_response(Method, Peer, Path, Req, StartTime, Res, Opts). + +%% @doc Process a raw response from the HTTP client. +process_response( + Method, Peer, Path, Req, StartTime, + {_ErlStatus, Status, Headers, Body}, + Opts + ) -> % Process the response. EndTime = os:system_time(millisecond), ?event(http_outbound, @@ -213,7 +221,10 @@ request(Method, Peer, Path, RawMessage, Opts) -> Body, Opts ) - end. + end; +process_response(_, _, _, _, _, {error, Reason}, _Opts) -> + ?event(http, {http_request_failed, {reason, Reason}}), + {error, {http_request_failed, Reason}}. %% @doc Convert a HTTP status code to a status atom. response_status_to_atom(Status) -> From c7cf1d67c63464fe9859b183c24db47624f71214 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 31 Oct 2025 14:50:13 -0400 Subject: [PATCH 2/5] chore: add HTTP reouting failover test --- src/dev_relay.erl | 61 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/src/dev_relay.erl b/src/dev_relay.erl index 17343a77c..dadb3004e 100644 --- a/src/dev_relay.erl +++ b/src/dev_relay.erl @@ -133,7 +133,6 @@ call(M1, RawM2, Opts) -> ?event(debug_relay, {relay_call, {without_http_params, TargetMod4}}), ?event(debug_relay, {relay_call, {with_http_params, TargetMod5}}), true = hb_message:verify(TargetMod5), - ?event(debug_relay, {relay_call, {verified, true}}), Client = case hb_maps:get(<<"http-client">>, BaseTarget, not_found, Opts) of @@ -304,4 +303,62 @@ commit_request_test() -> #{} ), ?event({res, Res}), - ?assertEqual(<<"value">>, Res). \ No newline at end of file + ?assertEqual(<<"value">>, Res). + +relay_failover_test() -> + application:ensure_all_started([hb]), + PeerWallet = ar_wallet:new(), + RelayWallet = ar_wallet:new(), + Peer = hb_http_server:start_node(#{ priv_wallet => PeerWallet }), + Node = + hb_http_server:start_node(NodeOpts = #{ + relay_allow_commit_request => true, + priv_wallet => RelayWallet, + routes => + [ + #{ + <<"template">> => <<"/~meta@1.0/info.*">>, + <<"nodes">> => [ + #{ + % Note: Will need update when Google runs + % HyperBEAM. + <<"prefix">> => <<"http://google.com/">> + }, + #{ + <<"prefix">> => <<"http://doesnotroute.invalid/">> + }, + #{ + <<"prefix">> => Peer + } + ] + } + ], + on => #{ + <<"request">> => + #{ + <<"device">> => <<"router@1.0">>, + <<"path">> => <<"preprocess">>, + <<"commit-request">> => true + } + } + }), + % Validate that the server can forward requests through the `hb_http:get` API. + {ok, DirectRecvdAddr} = + hb_http:request( + #{ <<"path">> => <<"~meta@1.0/info/address">> }, + NodeOpts + ), + ?assertEqual(hb_util:human_id(PeerWallet), DirectRecvdAddr), + % Validate that the relay device is able to forward requests to the peer. + {ok, RelayRecvdAddr} = + hb_http:get( + Node, + <<"~relay@1.0/call?relay-path=~meta@1.0/info/address">>, + #{} + ), + ?assertEqual(hb_util:human_id(PeerWallet), RelayRecvdAddr), + ?hr(), + timer:sleep(100), + % Validate that the server forwards requests from clients to the peer. + {ok, ClientRecvdAddr} = hb_http:get(Node, <<"~meta@1.0/info/address">>, #{}), + ?assertEqual(hb_util:human_id(PeerWallet), ClientRecvdAddr). \ No newline at end of file From 19da3617cd9e0fb9d68197a2e97e194865420ed7 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 31 Oct 2025 14:50:44 -0400 Subject: [PATCH 3/5] wip: router processor support for multi-node peer results --- src/dev_router.erl | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/dev_router.erl b/src/dev_router.erl index 775c63c3c..8ea2ef9a8 100644 --- a/src/dev_router.erl +++ b/src/dev_router.erl @@ -545,7 +545,48 @@ preprocess(Base, RawReq, Opts) -> }] }} end; - {ok, _Method, Node, _Path, _MsgWithoutMeta, _ReqOpts} -> + {ok, _Method, RawPeers, _Path, _MsgWithoutMeta, _ReqOpts} -> + ?event(debug_preprocess, {raw_peers, RawPeers}), + Peer = + if is_map(RawPeers) -> + Nodes = + hb_maps:get( + <<"nodes">>, + RawPeers, + [hb_maps:get(<<"node">>, RawPeers, <<>>, Opts)], + Opts + ), + NewNodes = + lists:map( + fun(P) -> + URI = + uri_string:parse( + hb_maps:get( + <<"uri">>, + P, + <<>>, + Opts + ) + ), + P#{ + <<"uri">> => + hb_util:bin( + uri_string:recompose( + URI#{ + path => <<"user-path">> + } + ) + ) + } + + end, + Nodes + ), + RawPeers#{ + <<"nodes">> => NewNodes + }; + true -> RawPeers + end, ?event(debug_preprocess, {matched_route, {explicit, Res}}), CommitRequest = hb_util:atom( @@ -603,7 +644,7 @@ preprocess(Base, RawReq, Opts) -> <<"device">> => <<"relay@1.0">>, <<"relay-device">> => <<"apply@1.0">>, <<"method">> => <<"POST">>, - <<"peer">> => Node + <<"peer">> => Peer }, #{ <<"path">> => <<"call">>, From ac7418986a031333e470ab0e8ac0d73d7a569b65 Mon Sep 17 00:00:00 2001 From: Peter Farber Date: Mon, 17 Nov 2025 10:32:31 -0500 Subject: [PATCH 4/5] fix: relay multi-node failover and add per-node HTTP timeouts --- src/dev_relay.erl | 166 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 153 insertions(+), 13 deletions(-) diff --git a/src/dev_relay.erl b/src/dev_relay.erl index dadb3004e..488a1068b 100644 --- a/src/dev_relay.erl +++ b/src/dev_relay.erl @@ -134,6 +134,8 @@ call(M1, RawM2, Opts) -> ?event(debug_relay, {relay_call, {with_http_params, TargetMod5}}), true = hb_message:verify(TargetMod5), ?event(debug_relay, {relay_call, {verified, true}}), + RequestMethod = + hb_maps:get(<<"method">>, TargetMod5, RelayMethod, Opts), Client = case hb_maps:get(<<"http-client">>, BaseTarget, not_found, Opts) of not_found -> hb_opts:get(relay_http_client, Opts); @@ -146,14 +148,35 @@ call(M1, RawM2, Opts) -> not_found -> hb_http:request(TargetMod5, HTTPOpts); _ -> - ?event(debug_relay, {relaying_to_peer, RelayPeer}), - hb_http:request( - RelayMethod, - RelayPeer, - RelayPath, - TargetMod5, - HTTPOpts - ) + case hb_ao:get(<<"nodes">>, RelayPeer, not_found, Opts) of + not_found -> + ?event(debug_relay, {relaying_to_peer, RelayPeer}), + hb_http:request( + RequestMethod, + RelayPeer, + RelayPath, + TargetMod5, + HTTPOpts + ); + Nodes when is_list(Nodes) -> + relay_nodes_in_order( + hb_util:message_to_ordered_list(Nodes, Opts), + RequestMethod, + RelayPath, + TargetMod5, + HTTPOpts, + Opts + ); + _ -> + ?event(debug_relay, {relaying_to_peer, RelayPeer}), + hb_http:request( + RequestMethod, + RelayPeer, + RelayPath, + TargetMod5, + HTTPOpts + ) + end end, case Res of {ok, R} -> @@ -185,6 +208,118 @@ request(_Base, Req, Opts) -> } }. +%% @doc Try each node in order, respecting per-node HTTP timeouts. Stops at the +%% first admissible response or when all nodes fail/time out. +relay_nodes_in_order([], _Method, _Path, _Message, _HTTPOpts, _Opts) -> + {error, no_viable_responses}; +relay_nodes_in_order( + [Node|Rest], + Method, + Path, + Message, + HTTPOpts, + Opts + ) -> + case hb_ao:get(<<"prefix">>, Node, not_found, Opts) of + not_found -> + relay_nodes_in_order( + Rest, + Method, + Path, + Message, + HTTPOpts, + Opts + ); + Peer -> + {PeerTimeout, HTTPOpts1} = peer_http_opts(Node, HTTPOpts, Opts), + ?event(debug_relay, {relaying_to_peer, Peer}), + RequestFun = + fun() -> + hb_http:request(Method, Peer, Path, Message, HTTPOpts1) + end, + case relay_request_with_timeout(RequestFun, PeerTimeout) of + {ok, Res} -> + case relay_response_ok(Res, Opts) of + true -> {ok, Res}; + false -> + relay_nodes_in_order( + Rest, + Method, + Path, + Message, + HTTPOpts, + Opts + ) + end; + {error, _Reason} -> + relay_nodes_in_order( + Rest, + Method, + Path, + Message, + HTTPOpts, + Opts + ) + end + end. + +relay_response_ok(Res, Opts) -> + Status = hb_util:int(hb_ao:get(<<"status">>, Res, 500, Opts)), + Status < 400. + +%% @doc Run a request with an optional hard timeout. When no timeout is provided +%% the request executes in the caller; otherwise we spawn and kill the worker if +%% it exceeds the limit. +relay_request_with_timeout( + RequestFun, + Timeout + ) when Timeout == not_found; Timeout == undefined -> + RequestFun(); +relay_request_with_timeout(RequestFun, Timeout) -> + Parent = self(), + Ref = make_ref(), + Worker = + spawn(fun() -> + Parent ! {Ref, RequestFun()} + end), + receive + {Ref, Res} -> Res + after Timeout -> + exit(Worker, kill), + {error, relay_peer_timeout} + end. + +peer_http_opts(Node, HTTPOpts, Opts) -> + NodeOpts = + case hb_maps:get(<<"opts">>, Node, #{}, Opts) of + Map when is_map(Map) -> Map; + _ -> #{} + end, + Normalized = hb_opts:mimic_default_types(NodeOpts, new_atoms, Opts), + case peer_timeout(Node, NodeOpts, Opts) of + not_found -> + {not_found, maps:merge(HTTPOpts, Normalized)}; + Timeout -> + TimeoutMs = hb_util:int(Timeout), + { + TimeoutMs, + maps:merge( + HTTPOpts, + Normalized#{ + http_request_send_timeout => TimeoutMs, + http_connect_timeout => TimeoutMs + } + ) + } + end. + +peer_timeout(Node, NodeOpts, Opts) -> + case hb_ao:get(<<"http-timeout">>, Node, not_found, Opts) of + not_found -> + hb_maps:get(<<"http-timeout">>, NodeOpts, not_found, Opts); + Timeout -> Timeout + end. + %%% Tests @@ -320,15 +455,20 @@ relay_failover_test() -> <<"template">> => <<"/~meta@1.0/info.*">>, <<"nodes">> => [ #{ - % Note: Will need update when Google runs - % HyperBEAM. - <<"prefix">> => <<"http://google.com/">> + % Remote peer used to exercise timeout-driven + % failover. When Google one day runs HB, we can + % lower this again. + <<"prefix">> => <<"http://google.com/">>, + <<"http-timeout">> => 10000 }, #{ - <<"prefix">> => <<"http://doesnotroute.invalid/">> + <<"prefix">> => <<"http://doesnotroute.invalid/">>, + <<"http-timeout">> => 2000 }, #{ - <<"prefix">> => Peer + % Local peer that should eventually succeed. + <<"prefix">> => Peer, + <<"http-timeout">> => 5000 } ] } From 44a4d08b36a99a0f4fe2aa9d60fd049209a1a0fe Mon Sep 17 00:00:00 2001 From: Peter Farber Date: Wed, 19 Nov 2025 16:53:49 -0500 Subject: [PATCH 5/5] chore: route dev_relay through hb_http multi_dispatch and preserve explicit methods --- src/dev_relay.erl | 191 ++++++++++++++-------------------------------- src/hb_http.erl | 14 +++- 2 files changed, 69 insertions(+), 136 deletions(-) diff --git a/src/dev_relay.erl b/src/dev_relay.erl index 488a1068b..9400abc85 100644 --- a/src/dev_relay.erl +++ b/src/dev_relay.erl @@ -144,40 +144,29 @@ call(M1, RawM2, Opts) -> % Let `hb_http:request/2' handle finding the peer and dispatching the % request, unless the peer is explicitly given. HTTPOpts = Opts#{ http_client => Client, http_only_result => false }, - Res = case RelayPeer of - not_found -> - hb_http:request(TargetMod5, HTTPOpts); - _ -> - case hb_ao:get(<<"nodes">>, RelayPeer, not_found, Opts) of - not_found -> - ?event(debug_relay, {relaying_to_peer, RelayPeer}), - hb_http:request( - RequestMethod, - RelayPeer, - RelayPath, - TargetMod5, - HTTPOpts - ); - Nodes when is_list(Nodes) -> - relay_nodes_in_order( - hb_util:message_to_ordered_list(Nodes, Opts), - RequestMethod, - RelayPath, - TargetMod5, - HTTPOpts, - Opts - ); - _ -> - ?event(debug_relay, {relaying_to_peer, RelayPeer}), - hb_http:request( - RequestMethod, - RelayPeer, - RelayPath, - TargetMod5, - HTTPOpts - ) - end - end, + Res = + case RelayPeer of + not_found -> + hb_http:request(TargetMod5, HTTPOpts); + Peer when is_map(Peer) -> + Prepared = prepare_relay_peer(Peer, Opts), + hb_http:request( + RequestMethod, + Prepared, + RelayPath, + TargetMod5, + HTTPOpts + ); + Peer -> + ?event(debug_relay, {relaying_to_peer, Peer}), + hb_http:request( + RequestMethod, + Peer, + RelayPath, + TargetMod5, + HTTPOpts + ) + end, case Res of {ok, R} -> {ok, hb_maps:without([<<"set-cookie">>], R)}; @@ -208,118 +197,50 @@ request(_Base, Req, Opts) -> } }. -%% @doc Try each node in order, respecting per-node HTTP timeouts. Stops at the -%% first admissible response or when all nodes fail/time out. -relay_nodes_in_order([], _Method, _Path, _Message, _HTTPOpts, _Opts) -> - {error, no_viable_responses}; -relay_nodes_in_order( - [Node|Rest], - Method, - Path, - Message, - HTTPOpts, - Opts - ) -> - case hb_ao:get(<<"prefix">>, Node, not_found, Opts) of - not_found -> - relay_nodes_in_order( - Rest, - Method, - Path, - Message, - HTTPOpts, - Opts - ); - Peer -> - {PeerTimeout, HTTPOpts1} = peer_http_opts(Node, HTTPOpts, Opts), - ?event(debug_relay, {relaying_to_peer, Peer}), - RequestFun = - fun() -> - hb_http:request(Method, Peer, Path, Message, HTTPOpts1) - end, - case relay_request_with_timeout(RequestFun, PeerTimeout) of - {ok, Res} -> - case relay_response_ok(Res, Opts) of - true -> {ok, Res}; - false -> - relay_nodes_in_order( - Rest, - Method, - Path, - Message, - HTTPOpts, - Opts - ) - end; - {error, _Reason} -> - relay_nodes_in_order( - Rest, - Method, - Path, - Message, - HTTPOpts, - Opts - ) - end +prepare_relay_peer(Peer, Opts) -> + case hb_ao:get(<<"nodes">>, Peer, not_found, Opts) of + Nodes when is_list(Nodes) -> + Peer#{ <<"nodes">> => prepare_relay_nodes(Nodes, Opts) }; + _ -> + Peer end. -relay_response_ok(Res, Opts) -> - Status = hb_util:int(hb_ao:get(<<"status">>, Res, 500, Opts)), - Status < 400. - -%% @doc Run a request with an optional hard timeout. When no timeout is provided -%% the request executes in the caller; otherwise we spawn and kill the worker if -%% it exceeds the limit. -relay_request_with_timeout( - RequestFun, - Timeout - ) when Timeout == not_found; Timeout == undefined -> - RequestFun(); -relay_request_with_timeout(RequestFun, Timeout) -> - Parent = self(), - Ref = make_ref(), - Worker = - spawn(fun() -> - Parent ! {Ref, RequestFun()} - end), - receive - {Ref, Res} -> Res - after Timeout -> - exit(Worker, kill), - {error, relay_peer_timeout} - end. +prepare_relay_nodes(Nodes, Opts) -> + [ + prepare_relay_node(Node, Opts) + || + Node <- hb_util:message_to_ordered_list(Nodes, Opts) + ]. -peer_http_opts(Node, HTTPOpts, Opts) -> - NodeOpts = +prepare_relay_node(Node, Opts) -> + NormalizedOpts = case hb_maps:get(<<"opts">>, Node, #{}, Opts) of - Map when is_map(Map) -> Map; + Map when is_map(Map) -> hb_opts:mimic_default_types(Map, new_atoms, Opts); _ -> #{} end, - Normalized = hb_opts:mimic_default_types(NodeOpts, new_atoms, Opts), - case peer_timeout(Node, NodeOpts, Opts) of + Node#{ + <<"opts">> => apply_node_timeout(Node, NormalizedOpts, Opts) + }. + +apply_node_timeout(Node, NodeOpts, Opts) -> + Timeout = + case hb_ao:get(<<"http-timeout">>, Node, not_found, Opts) of + not_found -> + hb_maps:get(<<"http-timeout">>, NodeOpts, not_found, Opts); + TimeoutValue -> + TimeoutValue + end, + case Timeout of not_found -> - {not_found, maps:merge(HTTPOpts, Normalized)}; - Timeout -> + NodeOpts; + _ -> TimeoutMs = hb_util:int(Timeout), - { - TimeoutMs, - maps:merge( - HTTPOpts, - Normalized#{ - http_request_send_timeout => TimeoutMs, - http_connect_timeout => TimeoutMs - } - ) + NodeOpts#{ + http_request_send_timeout => TimeoutMs, + http_connect_timeout => TimeoutMs } end. -peer_timeout(Node, NodeOpts, Opts) -> - case hb_ao:get(<<"http-timeout">>, Node, not_found, Opts) of - not_found -> - hb_maps:get(<<"http-timeout">>, NodeOpts, not_found, Opts); - Timeout -> Timeout - end. - %%% Tests diff --git a/src/hb_http.erl b/src/hb_http.erl index 273c3b54d..9ccad4165 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -69,6 +69,13 @@ request(Method, Config = #{ <<"nodes">> := Nodes }, Path, Message, Opts) when is % `multirequest' functionality, rather than a single request. hb_http_multi:request(Config, Method, Path, Message, Opts); request(Method, #{ <<"opts">> := ReqOpts, <<"uri">> := URI }, _Path, Message, Opts) -> + ExplicitMethod = + hb_maps:get( + <<"method">>, + Message, + not_found, + Opts + ), % The request has a set of additional options, so we apply them to the % request. MergedOpts = @@ -85,7 +92,12 @@ request(Method, #{ <<"opts">> := ReqOpts, <<"uri">> := URI }, _Path, Message, Op Message#{ <<"path">> => URI, <<"method">> => Method }, MergedOpts ), - request(NewMethod, Node, NewPath, NewMsg, NewOpts); + FinalMethod = + case ExplicitMethod of + not_found -> NewMethod; + _ -> Method + end, + request(FinalMethod, Node, NewPath, NewMsg, NewOpts); request(Method, Peer, Path, RawMessage, Opts) -> ?event({request, {method, Method}, {peer, Peer}, {path, Path}, {message, RawMessage}}), Req =