From 0f8c0dc8329a803e0aeed2b6b854c0514a1230eb Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 20 Nov 2025 14:54:29 -0500 Subject: [PATCH 01/22] wip: normalize unsigned ID gen for ANS104 --- src/dev_codec_ans104.erl | 25 ++++++++-- src/dev_codec_ans104_from.erl | 85 ++++++++++++++++++--------------- src/hb_message_test_vectors.erl | 4 +- 3 files changed, 70 insertions(+), 44 deletions(-) diff --git a/src/dev_codec_ans104.erl b/src/dev_codec_ans104.erl index 5b90ad154..27f158136 100644 --- a/src/dev_codec_ans104.erl +++ b/src/dev_codec_ans104.erl @@ -45,17 +45,34 @@ commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) -> <<"ans104@1.0">>, Opts ), - {ok, SignedStructured}; + ?event({signed_structured, SignedStructured}), + commit(SignedStructured, Req#{ <<"type">> => <<"unsigned">> }, Opts); commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> % Remove the commitments from the message, convert it to ANS-104, then back. % This forces the message to be normalized and the unsigned ID to be % recalculated. + ?event({adding_unsigned_commitment, Msg}), + WithoutExistingUnsigned = + hb_message:without_commitments( + #{ <<"type">> => <<"unsigned-sha256">> }, + Msg, + Opts + ), + ?event({without_existing_unsigned, WithoutExistingUnsigned}), + WithoutExistingUnsignedEncoded = + hb_message:convert( + WithoutExistingUnsigned, + <<"ans104@1.0">>, + <<"structured@1.0">>, + Opts + ), + ?event({without_existing_unsigned_encoded, WithoutExistingUnsignedEncoded}), { ok, hb_message:convert( - hb_maps:without([<<"commitments">>], Msg, Opts), - <<"ans104@1.0">>, + WithoutExistingUnsignedEncoded, <<"structured@1.0">>, + <<"ans104@1.0">>, Opts ) }. @@ -131,7 +148,7 @@ to(RawTABM, Req, Opts) when is_map(RawTABM) -> % Ensure that the TABM is fully loaded if the `bundle` key is set to true. ?event({to, {inbound, RawTABM}, {req, Req}}), MaybeCommitment = hb_message:commitment( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, + #{ <<"commitment-device">> => <<"ans104@1.0">>, <<"type">> => <<"rsa-pss-sha256">> }, RawTABM, Opts ), diff --git a/src/dev_codec_ans104_from.erl b/src/dev_codec_ans104_from.erl index a60e90505..6642e3045 100644 --- a/src/dev_codec_ans104_from.erl +++ b/src/dev_codec_ans104_from.erl @@ -159,50 +159,63 @@ base(CommittedKeys, Fields, Tags, Data, Opts) -> %% @doc Return a message with the appropriate commitments added to it. with_commitments( Item, Device, FieldCommitments, Tags, Base, CommittedKeys, Opts) -> + {UnsignedID, UnsignedCommitment} = + unsigned_commitment( + Item, + Device, + FieldCommitments, + Tags, + CommittedKeys, + Opts + ), case Item#tx.signature of ?DEFAULT_SIG -> - case normal_tags(Item#tx.tags) of - true -> Base; - false -> - with_unsigned_commitment( - Item, Device, FieldCommitments, Tags, Base, - CommittedKeys, Opts) - end; - _ -> with_signed_commitment( - Item, Device, FieldCommitments, Tags, Base, CommittedKeys, Opts) + Base#{ <<"commitments">> => #{ UnsignedID => UnsignedCommitment } }; + _ -> + {SignedID, SignedCommitment} = + signed_commitment( + Item, + Device, + FieldCommitments, + Tags, + CommittedKeys, + Opts + ), + Base#{ + <<"commitments">> => + #{ + SignedID => SignedCommitment, + UnsignedID => UnsignedCommitment + } + } end. %% @doc Returns a commitments message for an item, containing an unsigned %% commitment. -with_unsigned_commitment( - Item, Device, CommittedFields, Tags, - UncommittedMessage, CommittedKeys, Opts) -> +unsigned_commitment(Item, Device, CommittedFields, Tags, CommittedKeys, Opts) -> ID = hb_util:human_id(Item#tx.unsigned_id), - UncommittedMessage#{ - <<"commitments">> => #{ - ID => - filter_unset( - hb_maps:merge( - CommittedFields, - #{ - <<"commitment-device">> => Device, - <<"committed">> => CommittedKeys, - <<"type">> => <<"unsigned-sha256">>, - <<"bundle">> => bundle_commitment_key(Tags, Opts), - <<"original-tags">> => original_tags(Item, Opts) - }, - Opts - ), - Opts - ) - } + { + ID, + filter_unset( + hb_maps:merge( + CommittedFields, + #{ + <<"commitment-device">> => Device, + <<"committed">> => CommittedKeys, + <<"type">> => <<"unsigned-sha256">>, + <<"signature">> => ID, + <<"bundle">> => bundle_commitment_key(Tags, Opts), + <<"original-tags">> => original_tags(Item, Opts) + }, + Opts + ), + Opts + ) }. %% @doc Returns a commitments message for an item, containing a signed %% commitment. -with_signed_commitment( - Item, Device, FieldCommitments, Tags, - UncommittedMessage, CommittedKeys, Opts) -> +signed_commitment(Item, Device, FieldCommitments, Tags, CommittedKeys, Opts) -> Address = hb_util:human_id(ar_wallet:to_address(Item#tx.owner)), ID = hb_util:human_id(Item#tx.id), ExtraCommitments = hb_maps:merge( @@ -229,11 +242,7 @@ with_signed_commitment( ), Opts ), - UncommittedMessage#{ - <<"commitments">> => #{ - ID => Commitment - } - }. + {ID, Commitment}. %% @doc Return the bundle key for an item. bundle_commitment_key(Tags, Opts) -> diff --git a/src/hb_message_test_vectors.erl b/src/hb_message_test_vectors.erl index bd3a3f28d..194963d69 100644 --- a/src/hb_message_test_vectors.erl +++ b/src/hb_message_test_vectors.erl @@ -9,8 +9,8 @@ %% Disable/enable as needed. run_test() -> hb:init(), - nested_empty_map_test( - <<"structured@1.0">>, + signed_message_encode_decode_verify_test( + <<"ans104@1.0">>, test_opts(normal) ). From 9c54682ef102ae9917e5d7c2854624d0940f6897 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Thu, 20 Nov 2025 15:28:47 -0500 Subject: [PATCH 02/22] wip: handle commitments on empty messages --- src/hb_util.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hb_util.erl b/src/hb_util.erl index 5a3502014..998876f2d 100644 --- a/src/hb_util.erl +++ b/src/hb_util.erl @@ -449,7 +449,15 @@ message_to_ordered_list(Message, Opts) -> fun hb_ao:normalize_key/1, lists:sort(lists:map(fun int/1, Keys)) ), - message_to_ordered_list(NormMessage, SortedKeys, erlang:hd(SortedKeys), Opts). + case SortedKeys of + [] -> []; + _ -> + message_to_ordered_list( + NormMessage, + SortedKeys, + erlang:hd(SortedKeys), + Opts) + end. message_to_ordered_list(_Message, [], _Key, _Opts) -> []; message_to_ordered_list(Message, [Key|Keys], Key, Opts) -> From e182ea9def5060bd46d3e6dd7f6c6931933f981b Mon Sep 17 00:00:00 2001 From: James Piechota Date: Thu, 20 Nov 2025 15:38:30 -0500 Subject: [PATCH 03/22] wip: hb_message_test_vectors pass --- src/dev_codec_ans104.erl | 9 +++++++-- src/dev_codec_tx.erl | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/dev_codec_ans104.erl b/src/dev_codec_ans104.erl index 27f158136..769b76d38 100644 --- a/src/dev_codec_ans104.erl +++ b/src/dev_codec_ans104.erl @@ -35,9 +35,11 @@ commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) -> % Convert the given message to an ANS-104 TX record, sign it, and convert % it back to a structured message. + ?event({commit, {input_message, Msg}}), {ok, TX} = to(hb_private:reset(Msg), Req, Opts), Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), Signed = ar_bundles:sign_item(TX, Wallet), + ?event({commit, {signed_item, Signed}}), SignedStructured = hb_message:convert( Signed, @@ -45,7 +47,7 @@ commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) -> <<"ans104@1.0">>, Opts ), - ?event({signed_structured, SignedStructured}), + ?event({commit, {signed_structured, SignedStructured}}), commit(SignedStructured, Req#{ <<"type">> => <<"unsigned">> }, Opts); commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> % Remove the commitments from the message, convert it to ANS-104, then back. @@ -148,7 +150,10 @@ to(RawTABM, Req, Opts) when is_map(RawTABM) -> % Ensure that the TABM is fully loaded if the `bundle` key is set to true. ?event({to, {inbound, RawTABM}, {req, Req}}), MaybeCommitment = hb_message:commitment( - #{ <<"commitment-device">> => <<"ans104@1.0">>, <<"type">> => <<"rsa-pss-sha256">> }, + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, RawTABM, Opts ), diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index 3363969d2..51fdb6ceb 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -118,7 +118,10 @@ to(RawTABM, Req, Opts) when is_map(RawTABM) -> % Ensure that the TABM is fully loaded if the `bundle` key is set to true. ?event({to, {inbound, RawTABM}, {req, Req}}), MaybeCommitment = hb_message:commitment( - #{ <<"commitment-device">> => <<"tx@1.0">> }, + #{ + <<"commitment-device">> => <<"tx@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, RawTABM, Opts ), From 17963c1470a22374c2f4b2fd45e1ded2fc277d86 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Thu, 20 Nov 2025 16:23:45 -0500 Subject: [PATCH 04/22] wip: more unsigned commitment fixes --- src/dev_codec_ans104.erl | 45 ++++++++++++++++++++++++++----------- src/dev_codec_ans104_to.erl | 34 +++++++++++++++++++++++----- src/dev_codec_tx.erl | 10 ++------- src/hb_message.erl | 12 +++++----- 4 files changed, 70 insertions(+), 31 deletions(-) diff --git a/src/dev_codec_ans104.erl b/src/dev_codec_ans104.erl index 769b76d38..a1c6a5105 100644 --- a/src/dev_codec_ans104.erl +++ b/src/dev_codec_ans104.erl @@ -149,14 +149,8 @@ to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX}; to(RawTABM, Req, Opts) when is_map(RawTABM) -> % Ensure that the TABM is fully loaded if the `bundle` key is set to true. ?event({to, {inbound, RawTABM}, {req, Req}}), - MaybeCommitment = hb_message:commitment( - #{ - <<"commitment-device">> => <<"ans104@1.0">>, - <<"type">> => <<"rsa-pss-sha256">> - }, - RawTABM, - Opts - ), + MaybeCommitment = dev_codec_ans104_to:commitment( + <<"ans104@1.0">>, RawTABM, Opts), IsBundle = dev_codec_ans104_to:is_bundle(MaybeCommitment, Req, Opts), MaybeBundle = dev_codec_ans104_to:maybe_load(RawTABM, IsBundle, Opts), ?event({to, {maybe_bundle, MaybeBundle}}), @@ -260,9 +254,9 @@ unsigned_duplicated_tag_name_test() -> ] }), Msg = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}), - ?event({msg, Msg}), + ?event(debug_test, {msg, Msg}), TX2 = hb_message:convert(Msg, <<"ans104@1.0">>, <<"structured@1.0">>, #{}), - ?event({tx2, TX2}), + ?event(debug_test, {tx2, TX2}), ?assertEqual(TX, TX2). signed_duplicated_tag_name_test() -> @@ -491,7 +485,7 @@ unsigned_lowercase_bundle_map_tags_test() -> } }, {ok, UnsignedTX} = dev_codec_ans104:to(UnsignedTABM, #{}, #{}), - ?event({tx, UnsignedTX}), + ?event(debug_test, {tx, UnsignedTX}), ?assertEqual([ {<<"bundle-format">>, <<"binary">>}, {<<"bundle-version">>, <<"2.0.0">>}, @@ -501,8 +495,33 @@ unsigned_lowercase_bundle_map_tags_test() -> ], UnsignedTX#tx.tags), ?assert(UnsignedTX#tx.manifest =/= undefined), {ok, TABM} = dev_codec_ans104:from(UnsignedTX, #{}, #{}), - ?event({tabm, TABM}), - ?assertEqual(UnsignedTABM, TABM). + ?event(debug_test, {tabm, {explicit, TABM}}), + ExpectedTABM = UnsignedTABM#{ + <<"commitments">> => #{ + <<"q6hcdZlyNre_X3L5Z7zeSkjOKUP6L88BcQ_D0JOjrLs">> => #{ + <<"bundle">> => <<"true">>, + <<"comitment-device">> => <<"ans104@1.0">>, + <<"committed">> => [<<"data">>, <<"a1">>, <<"c1">>], + <<"signature">> => <<"q6hcdZlyNre_X3L5Z7zeSkjOKUP6L88BcQ_D0JOjrLs">>, + <<"type">> => <<"unsigned-sha256">> + } + }, + <<"data">> => #{ + <<"commitments">> => #{ + <<"IkDi2KTYxvbyeJ3t0JR1wuN9vJunYI1hj3wQjFloSG8s">> => #{ + <<"bundle">> => <<"false">>, + <<"comitment-device">> => <<"ans104@1.0">>, + <<"committed">> => [<<"data">>, <<"a2">>, <<"c2">>], + <<"signature">> => <<"IkDi2KTYxvbyeJ3t0JR1wuN9vJunYI1hj3wQjFloSG8s">>, + <<"type">> => <<"unsigned-sha256">> + } + }, + <<"data">> => <<"testdata">>, + <<"a2">> => <<"value2">>, + <<"c2">> => <<"value3">> + } + }, + ?assertEqual(ExpectedTABM, TABM). unsigned_mixedcase_bundle_list_tags_1_test() -> UnsignedTX = dev_arweave_common:normalize(#tx{ diff --git a/src/dev_codec_ans104_to.erl b/src/dev_codec_ans104_to.erl index 68a6ba033..2fc361fd1 100644 --- a/src/dev_codec_ans104_to.erl +++ b/src/dev_codec_ans104_to.erl @@ -1,7 +1,7 @@ %%% @doc Library functions for encoding messages to the ANS-104 format. -module(dev_codec_ans104_to). -export([is_bundle/3, maybe_load/3, data/3, tags/5, excluded_tags/3]). --export([siginfo/4, fields_to_tx/4]). +-export([commitment/3, siginfo/4, fields_to_tx/4]). -include("include/hb.hrl"). is_bundle({ok, _, Commitment}, _Req, Opts) -> @@ -37,6 +37,28 @@ maybe_load(RawTABM, true, Opts) -> maybe_load(RawTABM, false, _Opts) -> RawTABM. +commitment(Device, TABM, Opts) -> + SignedCommitment = hb_message:commitment( + #{ + <<"commitment-device">> => Device, + <<"type">> => <<"rsa-pss-sha256">> + }, + TABM, + Opts + ), + case SignedCommitment of + {ok, _, _} -> SignedCommitment; + _ -> + hb_message:commitment( + #{ + <<"commitment-device">> => Device, + <<"type">> => <<"unsigned-sha256">> + }, + TABM, + Opts + ) + end. + %% @doc Calculate the fields for a message, returning an initial TX record. %% One of the nuances here is that the `target' field must be set correctly. %% If the message has a commitment, we extract the `field-target' if found and @@ -55,10 +77,7 @@ siginfo(Message, multiple_matches, _FieldsFun, _Opts) -> %% tags, and last TX from the commitment. If the value is not present, the %% default value is used. commitment_to_tx(Commitment, FieldsFun, Opts) -> - Signature = - hb_util:decode( - maps:get(<<"signature">>, Commitment, hb_util:encode(?DEFAULT_SIG)) - ), + Signature = signature_from_commitment(Commitment), Owner = case hb_maps:find(<<"keyid">>, Commitment, Opts) of {ok, KeyID} -> @@ -82,6 +101,11 @@ commitment_to_tx(Commitment, FieldsFun, Opts) -> }, FieldsFun(TX, ?FIELD_PREFIX, Commitment, Opts). +signature_from_commitment(#{ <<"type">> := <<"unsigned-sha256">> }) -> + ?DEFAULT_SIG; +signature_from_commitment(Commitment) -> + hb_util:decode(maps:get(<<"signature">>, Commitment)). + %% @doc Convert a HyperBEAM-compatible map into an ANS-104 encoded tag list, %% recreating the original order of the tags. diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index 51fdb6ceb..8be4b59cf 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -117,14 +117,8 @@ to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX}; to(RawTABM, Req, Opts) when is_map(RawTABM) -> % Ensure that the TABM is fully loaded if the `bundle` key is set to true. ?event({to, {inbound, RawTABM}, {req, Req}}), - MaybeCommitment = hb_message:commitment( - #{ - <<"commitment-device">> => <<"tx@1.0">>, - <<"type">> => <<"rsa-pss-sha256">> - }, - RawTABM, - Opts - ), + MaybeCommitment = dev_codec_ans104_to:commitment( + <<"tx@1.0">>, RawTABM, Opts), IsBundle = dev_codec_ans104_to:is_bundle(MaybeCommitment, Req, Opts), MaybeBundle = dev_codec_ans104_to:maybe_load(RawTABM, IsBundle, Opts), ?event({to, {raw_tabm, RawTABM}, {is_bundle, IsBundle}, {maybe_bundle, MaybeBundle}, {req, Req}, {opts, Opts}}), diff --git a/src/hb_message.erl b/src/hb_message.erl index 21ed45ddf..2f4e96cb5 100644 --- a/src/hb_message.erl +++ b/src/hb_message.erl @@ -842,11 +842,13 @@ commitments(_Spec, _Msg, _Opts) -> %% @doc Return the devices for which there are commitments on a message. commitment_devices(#{ <<"commitments">> := Commitments }, Opts) -> - lists:map( - fun(CommMsg) -> - hb_ao:get(<<"commitment-device">>, CommMsg, Opts) - end, - maps:values(Commitments) + hb_util:unique( + lists:map( + fun(CommMsg) -> + hb_ao:get(<<"commitment-device">>, CommMsg, Opts) + end, + maps:values(Commitments) + ) ); commitment_devices(_Msg, _Opts) -> []. From a656b983af954a8b663f3486c27466960062259c Mon Sep 17 00:00:00 2001 From: James Piechota Date: Thu, 20 Nov 2025 17:02:41 -0500 Subject: [PATCH 05/22] wip: dev_codec_ans104 passes --- src/dev_codec_ans104.erl | 25 +++++++++++++++---------- src/dev_codec_ans104_to.erl | 30 +++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/dev_codec_ans104.erl b/src/dev_codec_ans104.erl index a1c6a5105..d8569084a 100644 --- a/src/dev_codec_ans104.erl +++ b/src/dev_codec_ans104.erl @@ -495,12 +495,11 @@ unsigned_lowercase_bundle_map_tags_test() -> ], UnsignedTX#tx.tags), ?assert(UnsignedTX#tx.manifest =/= undefined), {ok, TABM} = dev_codec_ans104:from(UnsignedTX, #{}, #{}), - ?event(debug_test, {tabm, {explicit, TABM}}), ExpectedTABM = UnsignedTABM#{ <<"commitments">> => #{ <<"q6hcdZlyNre_X3L5Z7zeSkjOKUP6L88BcQ_D0JOjrLs">> => #{ <<"bundle">> => <<"true">>, - <<"comitment-device">> => <<"ans104@1.0">>, + <<"commitment-device">> => <<"ans104@1.0">>, <<"committed">> => [<<"data">>, <<"a1">>, <<"c1">>], <<"signature">> => <<"q6hcdZlyNre_X3L5Z7zeSkjOKUP6L88BcQ_D0JOjrLs">>, <<"type">> => <<"unsigned-sha256">> @@ -508,11 +507,11 @@ unsigned_lowercase_bundle_map_tags_test() -> }, <<"data">> => #{ <<"commitments">> => #{ - <<"IkDi2KTYxvbyeJ3t0JR1wuN9vJunYI1hj3wQjFloSG8s">> => #{ + <<"IkDi2KTYxvbyeJ3t0JR1wuN9vJunYI1hj3wQjFloSG8">> => #{ <<"bundle">> => <<"false">>, - <<"comitment-device">> => <<"ans104@1.0">>, + <<"commitment-device">> => <<"ans104@1.0">>, <<"committed">> => [<<"data">>, <<"a2">>, <<"c2">>], - <<"signature">> => <<"IkDi2KTYxvbyeJ3t0JR1wuN9vJunYI1hj3wQjFloSG8s">>, + <<"signature">> => <<"IkDi2KTYxvbyeJ3t0JR1wuN9vJunYI1hj3wQjFloSG8">>, <<"type">> => <<"unsigned-sha256">> } }, @@ -521,6 +520,8 @@ unsigned_lowercase_bundle_map_tags_test() -> <<"c2">> => <<"value3">> } }, + ?event(debug_test, {expected_tabm, {explicit, ExpectedTABM}}), + ?event(debug_test, {tabm, {explicit, TABM}}), ?assertEqual(ExpectedTABM, TABM). unsigned_mixedcase_bundle_list_tags_1_test() -> @@ -541,7 +542,6 @@ unsigned_mixedcase_bundle_list_tags_1_test() -> } ] }), - ?event(debug_test, {unsigned_tx, UnsignedTX}), ?assertEqual([ {<<"TagA1">>, <<"value1">>}, {<<"TagA2">>, <<"value2">>}, @@ -566,6 +566,7 @@ unsigned_mixedcase_bundle_list_tags_1_test() -> ExpectedCommitment, hb_maps:with([<<"committed">>, <<"original-tags">>], Commitment, #{})), {ok, TX} = dev_codec_ans104:to(UnsignedTABM, #{}, #{}), + ?event(debug_test, {expected_tx, UnsignedTX}), ?event(debug_test, {tx, TX}), ?assertEqual(UnsignedTX, TX), ok. @@ -691,7 +692,8 @@ signed_lowercase_bundle_map_tags_test() -> ?assert(SignedTX#tx.manifest =/= undefined), {ok, SignedTABM} = dev_codec_ans104:from(SignedTX, #{}, #{}), ?event({signed_tabm, SignedTABM}), - ?assertEqual(UnsignedTABM, hb_maps:without([<<"commitments">>], SignedTABM)), + % Recursively exclude commitments from the SignedTABM for the match test. + ?assert(hb_message:match(UnsignedTABM, SignedTABM, only_present, #{})), Commitment = hb_message:commitment( hb_util:human_id(SignedTX#tx.id), SignedTABM), ?event({commitment, Commitment}), @@ -749,7 +751,8 @@ signed_mixedcase_bundle_map_tags_test() -> ?assert(SignedTX#tx.manifest =/= undefined), {ok, SignedTABM} = dev_codec_ans104:from(SignedTX, #{}, #{}), ?event(debug_test, {signed_tabm, SignedTABM}), - ?assertEqual(UnsignedTABM, hb_maps:without([<<"commitments">>], SignedTABM)), + % Recursively exclude commitments from the SignedTABM for the match test. + ?assert(hb_message:match(UnsignedTABM, SignedTABM, only_present, #{})), Commitment = hb_message:commitment( hb_util:human_id(SignedTX#tx.id), SignedTABM), ?event(debug_test, {commitment, Commitment}), @@ -803,7 +806,8 @@ test_bundle_commitment(Commit, Encode, Decode) -> #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => ToBool(Commit) }), ?event(debug_test, {committed, Label, {explicit, Committed}}), ?assert(hb_message:verify(Committed, all, Opts), Label), - {ok, _, CommittedCommitment} = hb_message:commitment(#{}, Committed, Opts), + {ok, _, CommittedCommitment} = hb_message:commitment( + #{ <<"type">> => <<"rsa-pss-sha256">> }, Committed, Opts), ?assertEqual( [<<"list">>], hb_maps:get(<<"committed">>, CommittedCommitment, Opts), Label), @@ -825,7 +829,8 @@ test_bundle_commitment(Commit, Encode, Decode) -> Opts), ?event(debug_test, {decoded, Label, {explicit, Decoded}}), ?assert(hb_message:verify(Decoded, all, Opts), Label), - {ok, _, DecodedCommitment} = hb_message:commitment(#{}, Decoded, Opts), + {ok, _, DecodedCommitment} = hb_message:commitment( + #{ <<"type">> => <<"rsa-pss-sha256">> }, Decoded, Opts), ?assertEqual( [<<"list">>], hb_maps:get(<<"committed">>, DecodedCommitment, Opts), Label), diff --git a/src/dev_codec_ans104_to.erl b/src/dev_codec_ans104_to.erl index 2fc361fd1..ce4bf1a9e 100644 --- a/src/dev_codec_ans104_to.erl +++ b/src/dev_codec_ans104_to.erl @@ -31,12 +31,36 @@ maybe_load(RawTABM, true, Opts) -> Opts ), % Ensure the commitments from the original message are the only - % ones in the fully loaded message. - LoadedComms = maps:get(<<"commitments">>, RawTABM, #{}), - LoadedTABM#{ <<"commitments">> => LoadedComms }; + % ones in the fully loaded message, recursively for nested maps. + replace_commitments_recursive(LoadedTABM, RawTABM); maybe_load(RawTABM, false, _Opts) -> RawTABM. +%% @doc Recursively replace commitments from RawTABM into LoadedTABM. +%% For each nested map in LoadedTABM, if there's a corresponding map in RawTABM, +%% recursively apply commitment replacement. +replace_commitments_recursive(LoadedTABM, RawTABM) + when is_map(LoadedTABM), is_map(RawTABM) -> + % First, replace commitments at this level + RawCommitments = maps:get(<<"commitments">>, RawTABM, #{}), + LoadedTABM2 = LoadedTABM#{ <<"commitments">> => RawCommitments }, + % Then recursively process nested maps + maps:map( + fun(Key, Value) when is_map(Value) -> + case maps:get(Key, RawTABM, undefined) of + RawValue when is_map(RawValue) -> + replace_commitments_recursive(Value, RawValue); + _ -> + Value + end; + (_Key, Value) -> + Value + end, + LoadedTABM2 + ); +replace_commitments_recursive(LoadedTABM, _RawTABM) -> + LoadedTABM. + commitment(Device, TABM, Opts) -> SignedCommitment = hb_message:commitment( #{ From 028c756845486fc6ec6d78bbb518c00287becafb Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 21 Nov 2025 09:49:58 -0500 Subject: [PATCH 06/22] fix: some TX codec tests --- src/dev_codec_tx.erl | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index 8be4b59cf..902e900ab 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -857,7 +857,7 @@ do_unsigned_tx_roundtrip(UnsignedTX, UnsignedTABM, Req) -> TABM = hb_util:ok(from(DeserializedTX, Req, #{})), ?event(debug_test, {unsigned_tx_roundtrip, {expected_tabm, UnsignedTABM}, {actual_tabm, TABM}}), - ?assertEqual(UnsignedTABM, TABM, unsigned_tx_roundtrip), + ?assert(hb_message:match(UnsignedTABM, TABM), unsigned_tx_roundtrip), % TABM -> TX TX = hb_util:ok(to(TABM, Req, #{})), ExpectedTX = UnsignedTX#tx{ unsigned_id = ar_tx:id(UnsignedTX, unsigned) }, @@ -887,7 +887,7 @@ do_signed_tx_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Req) -> #{ hb_util:human_id(SignedTX#tx.id) => SignedCommitment }}, ?event(debug_test, {signed_tx_roundtrip, {expected_tabm, SignedTABM}, {actual_tabm, TABM}}), - ?assertEqual(SignedTABM, TABM, signed_tx_roundtrip), + ?assert(hb_message:match(SignedTABM, TABM), signed_tx_roundtrip), % TABM -> TX TX = hb_util:ok(to(TABM, Req, #{})), ExpectedTX = SignedTX#tx{ @@ -921,7 +921,7 @@ do_unsigned_tabm_roundtrip(UnsignedTX0, UnsignedTABM, Req) -> TABM = hb_util:ok(from(DeserializedTX, Req, #{})), ?event(debug_test, {unsigned_tabm_roundtrip, {expected_tabm, UnsignedTABM}, {actual_tabm, TABM}}), - ?assertEqual(UnsignedTABM, TABM, unsigned_tabm_roundtrip). + ?assert(hb_message:match(UnsignedTABM, TABM), unsigned_tabm_roundtrip). do_signed_tabm_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Device, Req) -> % Commit TABM @@ -931,7 +931,10 @@ do_signed_tabm_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Device, Req) -> ?event(debug_test, {signed_tabm_roundtrip, {signed_tabm, SignedTABM}}), ?assert(hb_message:verify(SignedTABM), signed_tabm_roundtrip), {ok, _, SignedCommitment} = hb_message:commitment( - #{ <<"commitment-device">> => <<"tx@1.0">> }, + #{ + <<"commitment-device">> => <<"tx@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, SignedTABM, #{} ), @@ -960,7 +963,7 @@ do_signed_tabm_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Device, Req) -> }, SignedTX, signed_tabm_roundtrip), % TX -> TABM FinalTABM = hb_util:ok(from(SignedTX, Req, #{})), - ?assertEqual(SignedTABM, FinalTABM, signed_tabm_roundtrip). + ?assert(hb_message:match(SignedTABM, FinalTABM), signed_tabm_roundtrip). bundle_commitment_test() -> test_bundle_commitment(unbundled, unbundled, unbundled), @@ -986,7 +989,14 @@ test_bundle_commitment(Commit, Encode, Decode) -> #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => ToBool(Commit) }), ?event(debug_test, {committed, Label, {explicit, Committed}}), ?assert(hb_message:verify(Committed, all, Opts), Label), - {ok, _, CommittedCommitment} = hb_message:commitment(#{}, Committed, Opts), + {ok, _, CommittedCommitment} = + hb_message:commitment( + #{ + <<"type">> => <<"rsa-pss-sha256">> + }, + Committed, + Opts + ), ?assertEqual( [<<"list">>], hb_maps:get(<<"committed">>, CommittedCommitment, Opts), Label), @@ -1008,7 +1018,14 @@ test_bundle_commitment(Commit, Encode, Decode) -> Opts), ?event(debug_test, {decoded, Label, {explicit, Decoded}}), ?assert(hb_message:verify(Decoded, all, Opts), Label), - {ok, _, DecodedCommitment} = hb_message:commitment(#{}, Decoded, Opts), + {ok, _, DecodedCommitment} = + hb_message:commitment( + #{ + <<"type">> => <<"rsa-pss-sha256">> + }, + Decoded, + Opts + ), ?assertEqual( [<<"list">>], hb_maps:get(<<"committed">>, DecodedCommitment, Opts), Label), From 79cb95f362a2e1f62ebf8ff1d0080546635412ac Mon Sep 17 00:00:00 2001 From: James Piechota Date: Fri, 21 Nov 2025 11:40:33 -0500 Subject: [PATCH 07/22] fix: hb_message_test_vectors --- src/dev_codec_ans104_to.erl | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/dev_codec_ans104_to.erl b/src/dev_codec_ans104_to.erl index ce4bf1a9e..2be8fc963 100644 --- a/src/dev_codec_ans104_to.erl +++ b/src/dev_codec_ans104_to.erl @@ -44,17 +44,20 @@ replace_commitments_recursive(LoadedTABM, RawTABM) % First, replace commitments at this level RawCommitments = maps:get(<<"commitments">>, RawTABM, #{}), LoadedTABM2 = LoadedTABM#{ <<"commitments">> => RawCommitments }, - % Then recursively process nested maps + % Then recursively process nested maps, but do not recurse into the + % `commitments' map itself (we only replace it at each level). maps:map( - fun(Key, Value) when is_map(Value) -> - case maps:get(Key, RawTABM, undefined) of - RawValue when is_map(RawValue) -> - replace_commitments_recursive(Value, RawValue); - _ -> - Value - end; + fun(<<"commitments">>, Value) -> + Value; + (Key, Value) when is_map(Value) -> + case maps:get(Key, RawTABM, undefined) of + RawValue when is_map(RawValue) -> + replace_commitments_recursive(Value, RawValue); + _ -> + Value + end; (_Key, Value) -> - Value + Value end, LoadedTABM2 ); From 33927ba6b408aed6cd187e9188e5a92f6e678bfb Mon Sep 17 00:00:00 2001 From: James Piechota Date: Fri, 21 Nov 2025 12:42:27 -0500 Subject: [PATCH 08/22] fix: dev_bundler_xxx tests pass --- src/dev_bundler.erl | 29 ++++++++++++---- src/dev_bundler_cache.erl | 67 ++++++++++++++++++++++++++---------- src/dev_bundler_dispatch.erl | 9 ++++- 3 files changed, 78 insertions(+), 27 deletions(-) diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index 7b2bc0d77..3035f11e5 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -346,24 +346,39 @@ dispatch_blocking_test() -> recover_unbundled_items_test() -> Opts = #{store => hb_test_utils:test_store(hb_store_lmdb)}, % Create and cache some items - Item1 = hb_message:convert(new_data_item(1, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - Item2 = hb_message:convert(new_data_item(2, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - Item3 = hb_message:convert(new_data_item(3, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), + Item1 = hb_message:convert( + new_data_item(1, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), + Item2 = hb_message:convert( + new_data_item(2, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), + Item3 = hb_message:convert( + new_data_item(3, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), ok = dev_bundler_cache:write_item(Item1, Opts), ok = dev_bundler_cache:write_item(Item2, Opts), ok = dev_bundler_cache:write_item(Item3, Opts), % Bundle Item2 with a fake TX - FakeTX = ar_tx:sign(#tx{format = 2, tags = [{<<"test">>, <<"tx">>}]}, hb:wallet()), - StructuredTX = hb_message:convert(FakeTX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), + FakeTX = ar_tx:sign( + #tx{format = 2, tags = [{<<"test">>, <<"tx">>}]}, hb:wallet()), + StructuredTX = hb_message:convert( + FakeTX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), ok = dev_bundler_cache:write_tx(StructuredTX, [Item2], Opts), % Now recover unbundled items {RecoveredItems, RecoveredBytes} = recover_unbundled_items(Opts), ?assertEqual(3924, RecoveredBytes), RecoveredItems2 = [ hb_message:with_commitments( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, Item, Opts) || Item <- RecoveredItems], - ?assertEqual(lists:sort([Item1, Item3]), lists:sort(RecoveredItems2)), + WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, Item, Opts) + || Item <- [Item1, Item3]], + ?assertEqual(lists:sort(WrittenItems), lists:sort(RecoveredItems2)), ok. recover_respects_max_items_test() -> diff --git a/src/dev_bundler_cache.erl b/src/dev_bundler_cache.erl index a0320c679..3899b09d1 100644 --- a/src/dev_bundler_cache.erl +++ b/src/dev_bundler_cache.erl @@ -262,8 +262,14 @@ basic_cache_test() -> ?assertEqual(<<"posted">>, get_tx_status(TX, Opts)), ok = complete_tx(TX, Opts), ?assertEqual(<<"complete">>, get_tx_status(TX, Opts)), - ?assertEqual(TX, read_cache(TXID, <<"tx@1.0">>, Opts)), - ?assertEqual(Item, read_cache(ItemID, <<"ans104@1.0">>, Opts)), + ?assertEqual( + hb_message:with_commitments( + #{ <<"type">> => <<"rsa-pss-sha256">> }, TX, Opts), + read_cache(TXID, <<"tx@1.0">>, Opts)), + ?assertEqual( + hb_message:with_commitments( + #{ <<"type">> => <<"rsa-pss-sha256">> }, Item, Opts), + read_cache(ItemID, <<"ans104@1.0">>, Opts)), ok. load_unbundled_items_test() -> @@ -278,15 +284,22 @@ load_unbundled_items_test() -> % Link item2 to a bundle, leave others unbundled ok = write_tx(TX, [Item2], Opts), % Load unbundled items - UnbundledItems1 = load_unbundled_items(Opts), - UnbundledItems2 = [ + LoadedItems1 = load_unbundled_items(Opts), + LoadedItems2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, - Item, Opts) || Item <- UnbundledItems1 + Item, Opts) || Item <- LoadedItems1 ], - UnbundledItems3 = lists:sort(UnbundledItems2), - ?event(debug_test, {unbundled_items, UnbundledItems3}), - ?assertEqual(lists:sort([Item1, Item3]), UnbundledItems3), + LoadedItems3 = lists:sort(LoadedItems2), + WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, + Item, Opts) || Item <- [Item1, Item3]], + ?event(debug_test, {loaded_items, LoadedItems3}), + ?assertEqual(lists:sort(WrittenItems), LoadedItems3), ok. load_bundle_states_test() -> @@ -320,23 +333,37 @@ load_bundled_items_test() -> ok = write_tx(TX1, [Item1, Item2], Opts), ok = write_tx(TX2, [Item3], Opts), % Load items for bundle 1 - Bundle1Items1 = load_bundled_items(tx_id(TX1, Opts), Opts), - Bundle1Items2 = [ + Bundle1LoadedItems1 = load_bundled_items(tx_id(TX1, Opts), Opts), + Bundle1LoadedItems2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, - Item, Opts) || Item <- Bundle1Items1 + Item, Opts) || Item <- Bundle1LoadedItems1 ], - Bundle1Items3 = lists:sort(Bundle1Items2), - ?assertEqual(lists:sort([Item1, Item2]), Bundle1Items3), + Bundle1LoadedItems3 = lists:sort(Bundle1LoadedItems2), + Bundle1WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, + Item, Opts) || Item <- [Item1, Item2]], + ?assertEqual(lists:sort(Bundle1WrittenItems), Bundle1LoadedItems3), % Load items for bundle 2 - Bundle2Items1 = load_bundled_items(tx_id(TX2, Opts), Opts), - Bundle2Items2 = [ + Bundle2LoadedItems1 = load_bundled_items(tx_id(TX2, Opts), Opts), + Bundle2LoadedItems2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, - Item, Opts) || Item <- Bundle2Items1 + Item, Opts) || Item <- Bundle2LoadedItems1 ], - Bundle2Items3 = lists:sort(Bundle2Items2), - ?assertEqual(lists:sort([Item3]), Bundle2Items3), + Bundle2LoadedItems3 = lists:sort(Bundle2LoadedItems2), + Bundle2WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, + Item, Opts) || Item <- [Item3]], + ?assertEqual(lists:sort(Bundle2WrittenItems), Bundle2LoadedItems3), ok. new_data_item(Index, SizeOrData, Opts) -> @@ -365,7 +392,9 @@ new_tx(Index, Opts) -> hb_message:convert(TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts). read_cache(ID, Device, Opts) -> - {ok, Resolved} = hb_ao:resolve(#{ <<"path">> => ID }, Opts), + % {ok, Resolved} = hb_ao:resolve(#{ <<"path">> => ID }, Opts), + {ok, Resolved} = hb_cache:read(ID, Opts), Loaded = hb_cache:ensure_all_loaded(Resolved, Opts), + ?event(debug_test, {loaded, Loaded}), hb_message:with_commitments( #{ <<"commitment-device">> => Device }, Loaded, Opts). \ No newline at end of file diff --git a/src/dev_bundler_dispatch.erl b/src/dev_bundler_dispatch.erl index 078067827..3951434ae 100644 --- a/src/dev_bundler_dispatch.erl +++ b/src/dev_bundler_dispatch.erl @@ -1001,8 +1001,15 @@ recover_bundles_test() -> hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) || Item <- Bundle#bundle.items], + WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, + Item, Opts) || Item <- [Item1, Item2, Item3]], ?assertEqual( - lists:sort([Item1, Item2, Item3]), + lists:sort(WrittenItems), lists:sort(RecoveredItems)), ?assertEqual(tx_posted, Bundle#bundle.status), ?assert(hb_message:verify(Bundle#bundle.tx)), From e3d67224f091fe25a062387eb4241cef132a5e8e Mon Sep 17 00:00:00 2001 From: James Piechota Date: Fri, 21 Nov 2025 15:33:21 -0500 Subject: [PATCH 09/22] fix: trusted-keys logic doesn't discard additional commitments --- src/hb_gateway_client.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hb_gateway_client.erl b/src/hb_gateway_client.erl index 7f4edf061..3156162ae 100644 --- a/src/hb_gateway_client.erl +++ b/src/hb_gateway_client.erl @@ -327,7 +327,7 @@ result_to_message(ExpectedID, Item, Opts) -> AttName = hd(hb_maps:keys(Comms, Opts)), Comm = hb_maps:get(AttName, Comms, not_found, Opts), Structured#{ - <<"commitments">> => #{ + <<"commitments">> => Comms#{ AttName => Comm#{ <<"trusted-keys">> => From 8795fbc15e4476369febcb46be6302f3d002a4e7 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Fri, 21 Nov 2025 16:20:28 -0500 Subject: [PATCH 10/22] fix: add trusted-keys to all commitments --- src/hb_gateway_client.erl | 62 ++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/src/hb_gateway_client.erl b/src/hb_gateway_client.erl index 3156162ae..a89e562ee 100644 --- a/src/hb_gateway_client.erl +++ b/src/hb_gateway_client.erl @@ -317,38 +317,14 @@ result_to_message(ExpectedID, Item, Opts) -> true -> % The node trusts the GraphQL API, so we add the explicit % keys as committed fields. - ?event(warning, + ?event( + warning, {gql_verify_failed, adding_trusted_fields, {tags, Tags} } ), - Comms = hb_maps:get(<<"commitments">>, Structured, #{}, Opts), - AttName = hd(hb_maps:keys(Comms, Opts)), - Comm = hb_maps:get(AttName, Comms, not_found, Opts), - Structured#{ - <<"commitments">> => Comms#{ - AttName => - Comm#{ - <<"trusted-keys">> => - hb_ao:normalize_keys( - [ - hb_ao:normalize_key(Name) - || - #{ <<"name">> := Name } <- - hb_maps:values( - hb_ao:normalize_keys( - Tags, - Opts - ), - Opts - ) - ], - Opts - ) - } - } - } + add_trusted_keys_to_commitments(Structured, Tags, Opts) end end, {ok, Embedded}. @@ -367,6 +343,38 @@ decode_or_null(Bin) when is_binary(Bin) -> decode_or_null(_) -> <<>>. +add_trusted_keys_to_commitments(Structured, Tags, Opts) -> + Comms = hb_maps:get(<<"commitments">>, Structured, #{}, Opts), + TrustedKeys = trusted_keys_from_tags(Tags, Opts), + UpdatedComms = + maps:map( + fun(_, Comm) when is_map(Comm) -> + Comm#{ + <<"trusted-keys">> => TrustedKeys + }; + (_, Value) -> + Value + end, + Comms + ), + Structured#{ + <<"commitments">> => UpdatedComms + }. + +trusted_keys_from_tags(Tags, Opts) -> + hb_ao:normalize_keys( + [ + hb_ao:normalize_key(Name) + || + #{<<"name">> := Name} <- + hb_maps:values( + hb_ao:normalize_keys(Tags, Opts), + Opts + ) + ], + Opts + ). + %% @doc Takes a list of messages with `name' and `value' fields, and formats %% them as a GraphQL `tags' argument. subindex_to_tags(Subindex) -> From e36b7c0f1a3ec1a5d7a78f1bf10abdadfefeb7bc Mon Sep 17 00:00:00 2001 From: James Piechota Date: Fri, 21 Nov 2025 17:25:41 -0500 Subject: [PATCH 11/22] fix: dev_query_test_vectors test fix --- src/dev_query_test_vectors.erl | 182 +++++++++++---------------------- 1 file changed, 62 insertions(+), 120 deletions(-) diff --git a/src/dev_query_test_vectors.erl b/src/dev_query_test_vectors.erl index 98b4715ca..2004bdd19 100644 --- a/src/dev_query_test_vectors.erl +++ b/src/dev_query_test_vectors.erl @@ -170,6 +170,7 @@ simple_ans104_query_test() -> }, Node = hb_http_server:start_node(Opts), {ok, WrittenMsg} = write_test_message(Opts), + ?event({written_msg, WrittenMsg}), ?assertMatch( {ok, [_]}, hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) @@ -206,27 +207,8 @@ simple_ans104_query_test() -> }, Opts ), - ExpectedID = hb_message:id(WrittenMsg, all, Opts), - ?event({expected_id, ExpectedID}), ?event({simple_ans104_query_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + assert_query_match(WrittenMsg, Res, Opts). %% @doc Test transactions query with tags filter transactions_query_tags_test() -> @@ -269,27 +251,8 @@ transactions_query_tags_test() -> #{}, Opts ), - ExpectedID = hb_message:id(WrittenMsg, all, Opts), - ?event({expected_id, ExpectedID}), ?event({transactions_query_tags_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + assert_query_match(WrittenMsg, Res, Opts). %% @doc Test transactions query with owners filter transactions_query_owners_test() -> @@ -331,27 +294,8 @@ transactions_query_owners_test() -> }, Opts ), - ExpectedID = hb_message:id(WrittenMsg, all, Opts), - ?event({expected_id, ExpectedID}), ?event({transactions_query_owners_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + assert_query_match(WrittenMsg, Res, Opts). %% @doc Test transactions query with recipients filter transactions_query_recipients_test() -> @@ -396,27 +340,8 @@ transactions_query_recipients_test() -> }, Opts ), - ExpectedID = hb_message:id(WrittenMsg, all, Opts), - ?event({expected_id, ExpectedID}), ?event({transactions_query_recipients_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + assert_query_match(WrittenMsg, Res, Opts). %% @doc Test transactions query with ids filter transactions_query_ids_test() -> @@ -459,26 +384,8 @@ transactions_query_ids_test() -> }, Opts ), - ?event({expected_id, ExpectedID}), ?event({transactions_query_ids_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + assert_query_match(WrittenMsg, Res, Opts). %% @doc Test transactions query with combined filters transactions_query_combined_test() -> @@ -526,27 +433,8 @@ transactions_query_combined_test() -> }, Opts ), - ?event({expected_id, ExpectedID}), ?event({transactions_query_combined_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). - + assert_query_match(WrittenMsg, Res, Opts). %% @doc Test single transaction query by ID transaction_query_by_id_test() -> @@ -767,4 +655,58 @@ transaction_query_with_anchor_test() -> } }, Res - ). \ No newline at end of file + ). + +assert_query_match(WrittenMsg, Res, Opts) -> + SignedID = hb_message:id(WrittenMsg, signed, Opts), + UnsignedID = hb_message:id(WrittenMsg, unsigned, Opts), + ?event({signed_id, SignedID}), + ?event({unsigned_id, UnsignedID}), + % asset shape of response + ?assertMatch( + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := + [#{ + <<"node">> := + #{ + <<"id">> := _, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + }, + #{ + <<"node">> := + #{ + <<"id">> := _, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + } + ] + } + } + } when ?IS_ID(SignedID) andalso ?IS_ID(UnsignedID), + Res + ), + % assert that the correct IDs are in the response in any order + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := Edges + } + } + } = Res, + ActualIDs = + [ID + || + #{ + <<"node">> := + #{ + <<"id">> := ID + } + } <- Edges + ], + ExpectedIDs = lists:sort([SignedID, UnsignedID]), + ?assertEqual(ExpectedIDs, lists:sort(ActualIDs)). \ No newline at end of file From 4c28f236ba93e8e9f9c52a6c1186d4d8d1804adc Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 24 Nov 2025 12:58:50 -0500 Subject: [PATCH 12/22] fix: query-by-id --- src/dev_query_arweave.erl | 13 +++++++++---- src/dev_query_test_vectors.erl | 27 +++++++++++++-------------- src/hb_message.erl | 19 ++++++++++++------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/dev_query_arweave.erl b/src/dev_query_arweave.erl index 19ded4210..4fa3360ea 100644 --- a/src/dev_query_arweave.erl +++ b/src/dev_query_arweave.erl @@ -23,10 +23,15 @@ query(List, <<"edges">>, _Args, _Opts) -> {ok, [{ok, Msg} || Msg <- List]}; query(Msg, <<"node">>, _Args, _Opts) -> {ok, Msg}; -query(Obj, <<"transaction">>, Args, Opts) -> - case query(Obj, <<"transactions">>, Args, Opts) of - {ok, []} -> {ok, null}; - {ok, [Msg|_]} -> {ok, Msg} +query(Obj, <<"transaction">>, #{ <<"id">> := ID }, Opts) -> + case hb_cache:read(ID, Opts) of + {ok, Msg} -> + case hb_message:commitment(ID, Msg, Opts) of + {ok, ID, Comm} -> + {ok, Msg#{ <<"commitments">> => #{ ID => Comm } }}; + not_found -> {ok, null} + end; + not_found -> {ok, null} end; query(Obj, <<"transactions">>, Args, Opts) -> ?event({transactions_query, diff --git a/src/dev_query_test_vectors.erl b/src/dev_query_test_vectors.erl index 2004bdd19..6e7cf07c2 100644 --- a/src/dev_query_test_vectors.erl +++ b/src/dev_query_test_vectors.erl @@ -607,22 +607,21 @@ transaction_query_with_anchor_test() -> store => [hb_test_utils:test_store(hb_store_lmdb)] }, Node = hb_http_server:start_node(Opts), - {ok, ID} = - hb_cache:write( - hb_message:convert( - ar_bundles:sign_item( - #tx { - anchor = AnchorID = crypto:strong_rand_bytes(32), - data = <<"test-data">> - }, - hb:wallet() - ), - <<"structured@1.0">>, - <<"ans104@1.0">>, - Opts + Msg = + hb_message:convert( + ar_bundles:sign_item( + #tx { + anchor = AnchorID = crypto:strong_rand_bytes(32), + data = <<"test-data">> + }, + hb:wallet() ), + <<"structured@1.0">>, + <<"ans104@1.0">>, Opts ), + {ok, _UnsignedID} =hb_cache:write(Msg, Opts), + SignedID = hb_message:id(Msg, signed, Opts), EncodedAnchor = hb_util:encode(AnchorID), Query = <<""" @@ -641,7 +640,7 @@ transaction_query_with_anchor_test() -> Node, Query, #{ - <<"id">> => ID + <<"id">> => SignedID }, Opts ), diff --git a/src/hb_message.erl b/src/hb_message.erl index 2f4e96cb5..e71035f85 100644 --- a/src/hb_message.erl +++ b/src/hb_message.erl @@ -781,12 +781,16 @@ commitment(ID, Link, Opts) when ?IS_LINK(Link) -> commitment(ID, hb_cache:ensure_loaded(Link, Opts), Opts); commitment(ID, #{ <<"commitments">> := Commitments }, Opts) when is_binary(ID), is_map_key(ID, Commitments) -> - hb_maps:get( - ID, - Commitments, - not_found, - Opts - ); + FindRes = + hb_maps:find( + ID, + Commitments, + Opts + ), + case FindRes of + error -> not_found; + {ok, Comm} -> {ok, ID, Comm} + end; commitment(#{ <<"type">> := <<"unsigned">> }, Msg, Opts) -> Commitments = hb_maps:get(<<"commitments">>, Msg, #{}, Opts), UnsignedCommitments = @@ -804,7 +808,8 @@ commitment(#{ <<"type">> := <<"unsigned">> }, Msg, Opts) -> {ok, CommID, hb_util:ok(hb_maps:find(CommID, UnsignedCommitments, Opts))}; true -> ?event(commitment, {multiple_matches, {matches, UnsignedCommitments}}), - multiple_matches end; + multiple_matches + end; commitment(Spec, Msg, Opts) -> Matches = commitments(Spec, Msg, Opts), ?event(debug_commitment, {commitment, {spec, Spec}, {matches, Matches}}), From af30c92496be308a18243eca48c18ff9ea7a028a Mon Sep 17 00:00:00 2001 From: James Piechota Date: Mon, 24 Nov 2025 20:28:17 -0500 Subject: [PATCH 13/22] fix: add the default commitment (httpsig unsigned) to ans104 and tx --- src/dev_codec_ans104.erl | 22 ++++++----- src/dev_codec_ans104_from.erl | 43 +++++++++++---------- src/dev_codec_tx.erl | 30 ++++++++++++--- src/hb_message.erl | 71 ++++++++++++++++++++++------------- 4 files changed, 103 insertions(+), 63 deletions(-) diff --git a/src/dev_codec_ans104.erl b/src/dev_codec_ans104.erl index d8569084a..38e2e1332 100644 --- a/src/dev_codec_ans104.erl +++ b/src/dev_codec_ans104.erl @@ -69,15 +69,17 @@ commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> Opts ), ?event({without_existing_unsigned_encoded, WithoutExistingUnsignedEncoded}), - { - ok, - hb_message:convert( - WithoutExistingUnsignedEncoded, - <<"structured@1.0">>, - <<"ans104@1.0">>, - Opts - ) - }. + Committed = hb_message:convert( + WithoutExistingUnsignedEncoded, + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ), + ?event({committed, Committed}), + WithNormalizedCommitments = hb_message:normalize_commitments( + Committed, Opts, add), + ?event({with_normalized_commitments, WithNormalizedCommitments}), + {ok, WithNormalizedCommitments}. %% @doc Verify an ANS-104 commitment. verify(Msg, Req, Opts) -> @@ -127,7 +129,7 @@ do_from(RawTX, Req, Opts) -> FieldCommitments = dev_codec_ans104_from:fields(TX, ?FIELD_PREFIX, Opts), WithCommitments = dev_codec_ans104_from:with_commitments( TX, <<"ans104@1.0">>, FieldCommitments, Tags, Base, Keys, Opts), - ?event({from, {parsed_message, WithCommitments}}), + ?event({from, {result, WithCommitments}}), {ok, WithCommitments}. %% @doc Internal helper to translate a message to its #tx record representation, diff --git a/src/dev_codec_ans104_from.erl b/src/dev_codec_ans104_from.erl index 6642e3045..1a613cb2f 100644 --- a/src/dev_codec_ans104_from.erl +++ b/src/dev_codec_ans104_from.erl @@ -168,27 +168,30 @@ with_commitments( CommittedKeys, Opts ), - case Item#tx.signature of - ?DEFAULT_SIG -> - Base#{ <<"commitments">> => #{ UnsignedID => UnsignedCommitment } }; - _ -> - {SignedID, SignedCommitment} = - signed_commitment( - Item, - Device, - FieldCommitments, - Tags, - CommittedKeys, - Opts - ), - Base#{ - <<"commitments">> => - #{ - SignedID => SignedCommitment, - UnsignedID => UnsignedCommitment + WithCommitments = + case Item#tx.signature of + ?DEFAULT_SIG -> + Base#{ <<"commitments">> => + #{ UnsignedID => UnsignedCommitment } }; + _ -> + {SignedID, SignedCommitment} = + signed_commitment( + Item, + Device, + FieldCommitments, + Tags, + CommittedKeys, + Opts + ), + Base#{ + <<"commitments">> => + #{ + SignedID => SignedCommitment, + UnsignedID => UnsignedCommitment + } } - } - end. + end, + hb_message:normalize_commitments(WithCommitments, Opts, add). %% @doc Returns a commitments message for an item, containing an unsigned %% commitment. diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index 902e900ab..def7ed4cd 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -29,20 +29,38 @@ commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) -> <<"tx@1.0">>, Opts ), - {ok, SignedStructured}; + ?event({commit, {signed_structured, SignedStructured}}), + commit(SignedStructured, Req#{ <<"type">> => <<"unsigned">> }, Opts); commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> % Remove the commitments from the message, convert it to an L1 TX, % then back. This forces the message to be normalized and the unsigned ID % to be recalculated. - { - ok, + ?event({adding_unsigned_commitment, Msg}), + WithoutExistingUnsigned = + hb_message:without_commitments( + #{ <<"type">> => <<"unsigned-sha256">> }, + Msg, + Opts + ), + ?event({without_existing_unsigned, WithoutExistingUnsigned}), + WithoutExistingUnsignedEncoded = hb_message:convert( - hb_maps:without([<<"commitments">>], Msg, Opts), + WithoutExistingUnsigned, <<"tx@1.0">>, <<"structured@1.0">>, Opts - ) - }. + ), + ?event({without_existing_unsigned_encoded, WithoutExistingUnsignedEncoded}), + Committed = hb_message:convert( + WithoutExistingUnsignedEncoded, + <<"structured@1.0">>, + <<"tx@1.0">>, + Opts + ), + ?event({committed, Committed}), + WithNormalizedCommitments = hb_message:normalize_commitments(Committed, Opts), + ?event({with_normalized_commitments, WithNormalizedCommitments}), + {ok, WithNormalizedCommitments}. %% @doc Verify an L1 TX commitment. verify(Msg, Req, Opts) -> diff --git a/src/hb_message.erl b/src/hb_message.erl index e71035f85..2f0d2c8c0 100644 --- a/src/hb_message.erl +++ b/src/hb_message.erl @@ -188,7 +188,7 @@ id(Msg, RawCommitters, Opts) -> signed -> #{ <<"committers">> => <<"all">> }; List when is_list(List) -> #{ <<"committers">> => List } end, - ?event({getting_id, {msg, Msg}, {spec, CommSpec}}), + ?event(debug_test, {getting_id, {msg, Msg}, {spec, CommSpec}}), {ok, ID} = dev_message:id( Msg, @@ -220,6 +220,17 @@ normalize_commitments(Msg, Opts, Mode) when is_list(Msg) -> normalize_commitments(Msg, _Opts, _Mode) -> Msg. +%% @doc Normalize commitments on a message depending on `Mode': +%% - `passive`: ensure there is at least one unsigned commitment, +%% preserving any signed ones. If no unsigned commitment is present a new +%% one is added using the node-default commitment device +%% (aka NormCommitment). +%% - `add`: always add a NormCommitment if it's not present. +%% - `verify`: same as `passive`, but replace all existing commitments if +%% the committed keys found in the NormCommitment are different. +%% - `fast`: use a cached `phash2` in `priv.last-phash2` to short‑circuit full +%% verification when possible; if it mismatches, recompute and fall back to +%% `verify`. do_normalize_commitments(Msg, _Opts, _Mode) when ?IS_EMPTY_MESSAGE(Msg) -> Msg; do_normalize_commitments(Msg, Opts, passive) -> @@ -231,28 +242,31 @@ do_normalize_commitments(Msg, Opts, passive) -> end, hb_maps:to_list(Commitments) ), - ?event({do_normalize_commitments, + ?event(normalization, { + {normalizing_commitments, passive}, {unsigned_commitments, UnsignedCommitments}, {maybe_signed_commitment, SignedCommitments} }), case {UnsignedCommitments, SignedCommitments} of {[], _} -> - {ok, #{ <<"commitments">> := NewCommitments }} = - dev_message:commit( - uncommitted(Msg), - #{ - <<"type">> => <<"unsigned">> - }, - Opts - ), + NormCommitments = generate_norm_commitments(Msg, #{}, Opts), MergedCommitments = hb_maps:merge( - NewCommitments, + NormCommitments, hb_maps:from_list(SignedCommitments), Opts ), Msg#{ <<"commitments">> => MergedCommitments }; _ -> Msg end; +do_normalize_commitments(Msg, Opts, add) -> + NormCommitments = generate_norm_commitments(Msg, #{}, Opts), + Msg#{ + <<"commitments">> => + hb_maps:merge( + NormCommitments, + hb_maps:get(<<"commitments">>, Msg, #{}, Opts) + ) + }; do_normalize_commitments(Msg, Opts, verify) -> UnsignedCommitment = commitment(#{ <<"type">> => <<"unsigned">> }, Msg, Opts), {MaybeUnsignedID, MaybeCommittedSpec} = @@ -261,16 +275,13 @@ do_normalize_commitments(Msg, Opts, verify) -> {ID, #{ <<"committed">> => Committed }}; _ -> {undefined, #{}} end, - {ok, #{ <<"commitments">> := NormCommitments }} = - dev_message:commit( - uncommitted(Msg), - MaybeCommittedSpec#{ - <<"type">> => <<"unsigned">> - }, - Opts - ), - ?event(normalization, {normalizing_commitments, verify}), + NormCommitments = generate_norm_commitments(Msg, MaybeCommittedSpec, Opts), [NormID] = hb_maps:keys(NormCommitments, Opts), + ?event(normalization, { + {normalizing_commitments, verify}, + {maybe_unsigned_id, MaybeUnsignedID}, + {norm_id, NormID} + }), case {MaybeUnsignedID, NormID} of {MatchedID, MatchedID} -> Msg; @@ -287,16 +298,11 @@ do_normalize_commitments(Msg, Opts, verify) -> Opts ); {_OldID, _NewID} -> - {ok, #{ <<"commitments">> := NewCommitments }} = - dev_message:commit( - uncommitted(Msg), - #{ <<"type">> => <<"unsigned">> }, - Opts - ), + NormCommitments = generate_norm_commitments(Msg, #{}, Opts), % We had an unsigned ID to begin with and the new one is different. % This means that the committed keys have changed, so we drop any % other commitments and return only the new unsigned one. - attach_phash2(Msg#{ <<"commitments">> => NewCommitments }, Opts) + attach_phash2(Msg#{ <<"commitments">> => NormCommitments }, Opts) end; do_normalize_commitments(Msg, Opts, fast) when is_map(Msg) -> ExpectedHash = erlang:phash2(hb_private:reset(Msg)), @@ -316,6 +322,17 @@ do_normalize_commitments(Msg, Opts, fast) when is_map(Msg) -> do_normalize_commitments(MsgWithHash, Opts, verify) end. +%% @doc Generate unsigned commitments using the node-default commitment +%% device. (aka NormCommitments) +generate_norm_commitments(Msg, BaseSpec, Opts) -> + {ok, #{ <<"commitments">> := NormCommitments }} = + dev_message:commit( + uncommitted(Msg), + BaseSpec#{ <<"type">> => <<"unsigned">> }, + Opts + ), + NormCommitments. + %% @doc Annotate a message with its phash2 value in the `priv' sub-map, %% calculating it if necessary. attach_phash2(Msg, Opts) -> From 354bcdd82643b4654337f1bcff8d9f527e39ef56 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Mon, 24 Nov 2025 20:28:52 -0500 Subject: [PATCH 14/22] wip: graphql query only returns matching ids --- src/dev_query_arweave.erl | 61 ++++++++++++++++++--------------------- src/dev_query_graphql.erl | 5 ++-- 2 files changed, 31 insertions(+), 35 deletions(-) diff --git a/src/dev_query_arweave.erl b/src/dev_query_arweave.erl index 4fa3360ea..255eaa062 100644 --- a/src/dev_query_arweave.erl +++ b/src/dev_query_arweave.erl @@ -51,6 +51,7 @@ query(Obj, <<"transactions">>, Args, Opts) -> end, Matches ), + ?event({transactions_messages, Messages}), {ok, Messages}; query(Obj, <<"block">>, Args, Opts) -> case query(Obj, <<"blocks">>, Args, Opts) of @@ -194,20 +195,14 @@ match_args([], Results, Opts) -> Matches = lists:foldl( fun(Result, Acc) -> - hb_util:list_with(resolve_ids(Result, Opts), Acc) + hb_util:list_with(Result, Acc) end, - resolve_ids(hd(Results), Opts), + hd(Results), tl(Results) ), - hb_util:unique( - lists:flatten( - [ - all_ids(ID, Opts) - || - ID <- Matches - ] - ) - ); + ?event({match_args_results, + {results, Results}, {matches, Matches}}), + Matches; match_args([{Field, X} | Rest], Acc, Opts) -> MatchRes = match(Field, X, Opts), ?event({match, {field, Field}, {arg, X}, {match_res, MatchRes}}), @@ -247,13 +242,13 @@ match(<<"id">>, ID, _Opts) -> match(<<"ids">>, IDs, _Opts) -> {ok, IDs}; match(<<"tags">>, Tags, Opts) -> - hb_cache:match(dev_query_graphql:keys_to_template(Tags), Opts); + {ok, Matches} = + hb_cache:match(dev_query_graphql:keys_to_template(Tags), Opts), + {ok, all_ids(Matches, Opts)}; match(<<"owners">>, Owners, Opts) -> {ok, matching_commitments(<<"committer">>, Owners, Opts)}; match(<<"owner">>, Owner, Opts) -> - Res = matching_commitments(<<"committer">>, Owner, Opts), - ?event({match_owner, Owner, Res}), - {ok, Res}; + {ok, matching_commitments(<<"committer">>, Owner, Opts)}; match(<<"recipients">>, Recipients, Opts) -> {ok, matching_commitments(<<"field-target">>, Recipients, Opts)}; match(UnsupportedFilter, _, _) -> @@ -275,14 +270,18 @@ matching_commitments(Field, Values, Opts) when is_list(Values) -> matching_commitments(Field, Value, Opts) when is_binary(Value) -> case hb_cache:match(#{ Field => Value }, Opts) of {ok, IDs} -> + BaseIDs = + lists:map( + fun(ID) -> commitment_id_to_base_id(ID, Opts) end, IDs), ?event( {found_matching_commitments, {field, Field}, {value, Value}, - {ids, IDs} + {ids, IDs}, + {base_ids, BaseIDs} } ), - lists:map(fun(ID) -> commitment_id_to_base_id(ID, Opts) end, IDs); + BaseIDs; not_found -> not_found end. @@ -299,29 +298,25 @@ commitment_id_to_base_id(ID, Opts) -> end. %% @doc Find all IDs for a message, by any of its other IDs. -all_ids(ID, Opts) -> +all_ids(IDs, Opts) -> + all_ids(IDs, [], Opts). +all_ids([], Results, _Opts) -> + hb_util:unique( + lists:flatten( + Results + ) + ); +all_ids([ID | Rest], Results, Opts) -> Store = hb_opts:get(store, no_store, Opts), - case hb_store:list(Store, << ID/binary, "/commitments">>) of + IDs = case hb_store:list(Store, << ID/binary, "/commitments">>) of {ok, []} -> [ID]; {ok, CommitmentIDs} -> CommitmentIDs; _ -> [] - end. + end, + all_ids(Rest, [IDs | Results], Opts). %% @doc Scope the stores used for block matching. The searched stores can be %% scoped by setting the `query_arweave_scope' option. scope(Opts) -> Scope = hb_opts:get(query_arweave_scope, [local], Opts), hb_store:scope(Opts, Scope). - -%% @doc Resolve a list of IDs to their store paths, using the stores provided. -resolve_ids(IDs, Opts) -> - Scoped = scope(Opts), - lists:map( - fun(ID) -> - case hb_cache:read(ID, Opts) of - {ok, Msg} -> hb_message:id(Msg, uncommitted, Scoped); - not_found -> ID - end - end, - IDs - ). \ No newline at end of file diff --git a/src/dev_query_graphql.erl b/src/dev_query_graphql.erl index 0b8ea5017..74c9a1ef1 100644 --- a/src/dev_query_graphql.erl +++ b/src/dev_query_graphql.erl @@ -219,8 +219,9 @@ message_query(Msg, Field, _Args, Opts) message_query(Msg = #{ <<"independent_hash">> := _ }, <<"id">>, _Args, Opts) -> {ok, hb_maps:get(<<"independent_hash">>, Msg, null, Opts)}; message_query(Msg, <<"id">>, _Args, Opts) -> - ?event({message_query_id, {object, Msg}}), - {ok, hb_message:id(Msg, all, Opts)}; + ID = hb_message:id(Msg, all, Opts), + ?event({message_query_id, {object, Msg}, {id, ID}}), + {ok, ID}; message_query(_Msg, <<"cursor">>, _Args, _Opts) -> {ok, <<"">>}; message_query(_Obj, _Field, _, _) -> From 4e21a82ed208864ec7677cfa109276a50cebbed5 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Tue, 25 Nov 2025 10:42:36 -0500 Subject: [PATCH 15/22] wip: add `committer` field to unsigned ans104 and tx commitments so they get picked up by hb_message:id --- src/dev_codec_ans104_from.erl | 1 + src/dev_message.erl | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/dev_codec_ans104_from.erl b/src/dev_codec_ans104_from.erl index 1a613cb2f..74a552364 100644 --- a/src/dev_codec_ans104_from.erl +++ b/src/dev_codec_ans104_from.erl @@ -207,6 +207,7 @@ unsigned_commitment(Item, Device, CommittedFields, Tags, CommittedKeys, Opts) -> <<"committed">> => CommittedKeys, <<"type">> => <<"unsigned-sha256">>, <<"signature">> => ID, + <<"committer">> => ID, <<"bundle">> => bundle_commitment_key(Tags, Opts), <<"original-tags">> => original_tags(Item, Opts) }, diff --git a/src/dev_message.erl b/src/dev_message.erl index 1cb9cc917..1f653cf76 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -474,11 +474,6 @@ commitment_ids_from_request(Base, Req, Opts) -> X2 when is_list(X2) -> X2; CommitmentDescriptor -> hb_ao:normalize_key(CommitmentDescriptor) end, - ?event(debug_commitments, - {commitment_ids_from_request, - {req_commitments, ReqCommitments}, - {req_committers, ReqCommitters}} - ), % Get the commitments to verify. FromCommitmentIDs = case ReqCommitments of @@ -526,7 +521,15 @@ commitment_ids_from_request(Base, Req, Opts) -> ); FinalCommitmentIDs -> FinalCommitmentIDs end, - ?event({commitment_ids_from_request, {base, Base}, {req, Req}, {res, Res}}), + ?event(debug_commitments, + {commitment_ids_from_request, + {base, Base}, {req, Req}, {res, Res}, + {req_commitments, ReqCommitments}, + {req_committers, ReqCommitters}, + {from_commitment_ids, FromCommitmentIDs}, + {from_committer_addrs, FromCommitterAddrs} + } + ), Res. %% @doc Ensure that the `commitments` submessage of a base message is fully From 10b1d824e6cde7c8907e7e24f5e36e52da459944 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Tue, 25 Nov 2025 15:38:04 -0500 Subject: [PATCH 16/22] fix: revert earlier `committer => ID` change, remove unsigned IDs from graphql response --- src/dev_codec_ans104_from.erl | 1 - src/dev_query_arweave.erl | 42 +++++++------ src/dev_query_test_vectors.erl | 105 +++++++++++++++++++++++++-------- 3 files changed, 104 insertions(+), 44 deletions(-) diff --git a/src/dev_codec_ans104_from.erl b/src/dev_codec_ans104_from.erl index 74a552364..1a613cb2f 100644 --- a/src/dev_codec_ans104_from.erl +++ b/src/dev_codec_ans104_from.erl @@ -207,7 +207,6 @@ unsigned_commitment(Item, Device, CommittedFields, Tags, CommittedKeys, Opts) -> <<"committed">> => CommittedKeys, <<"type">> => <<"unsigned-sha256">>, <<"signature">> => ID, - <<"committer">> => ID, <<"bundle">> => bundle_commitment_key(Tags, Opts), <<"original-tags">> => original_tags(Item, Opts) }, diff --git a/src/dev_query_arweave.erl b/src/dev_query_arweave.erl index 255eaa062..f063d26a2 100644 --- a/src/dev_query_arweave.erl +++ b/src/dev_query_arweave.erl @@ -24,14 +24,10 @@ query(List, <<"edges">>, _Args, _Opts) -> query(Msg, <<"node">>, _Args, _Opts) -> {ok, Msg}; query(Obj, <<"transaction">>, #{ <<"id">> := ID }, Opts) -> - case hb_cache:read(ID, Opts) of - {ok, Msg} -> - case hb_message:commitment(ID, Msg, Opts) of - {ok, ID, Comm} -> - {ok, Msg#{ <<"commitments">> => #{ ID => Comm } }}; - not_found -> {ok, null} - end; - not_found -> {ok, null} + Messages = read_filtered_messages([ID], Opts), + case Messages of + [Msg] -> {ok, Msg}; + [] -> {ok, null} end; query(Obj, <<"transactions">>, Args, Opts) -> ?event({transactions_query, @@ -41,16 +37,7 @@ query(Obj, <<"transactions">>, Args, Opts) -> }), Matches = match_args(Args, Opts), ?event({transactions_matches, Matches}), - Messages = - lists:filtermap( - fun(Match) -> - case hb_cache:read(Match, Opts) of - {ok, Msg} -> {true, Msg}; - not_found -> false - end - end, - Matches - ), + Messages = read_filtered_messages(Matches, Opts), ?event({transactions_messages, Messages}), {ok, Messages}; query(Obj, <<"block">>, Args, Opts) -> @@ -315,6 +302,25 @@ all_ids([ID | Rest], Results, Opts) -> end, all_ids(Rest, [IDs | Results], Opts). +%% @doc Read the provided IDs and remove any messages that don't exist or +%% are unsigned. +read_filtered_messages(IDs, Opts) -> + lists:filtermap( + fun(ID) -> + case hb_cache:read(ID, Opts) of + {ok, Msg} -> + case hb_message:commitment(ID, Msg, Opts) of + {ok, ID, #{ <<"committer">> := _ } = Comm} -> + {true, Msg#{ <<"commitments">> => #{ ID => Comm } }}; + {ok, _ID, _CommWithoutCommitter} -> false; + not_found -> false + end; + not_found -> false + end + end, + IDs + ). + %% @doc Scope the stores used for block matching. The searched stores can be %% scoped by setting the `query_arweave_scope' option. scope(Opts) -> diff --git a/src/dev_query_test_vectors.erl b/src/dev_query_test_vectors.erl index 6e7cf07c2..5456d4936 100644 --- a/src/dev_query_test_vectors.erl +++ b/src/dev_query_test_vectors.erl @@ -208,7 +208,10 @@ simple_ans104_query_test() -> Opts ), ?event({simple_ans104_query_test, Res}), - assert_query_match(WrittenMsg, Res, Opts). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with tags filter transactions_query_tags_test() -> @@ -252,7 +255,10 @@ transactions_query_tags_test() -> Opts ), ?event({transactions_query_tags_test, Res}), - assert_query_match(WrittenMsg, Res, Opts). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with owners filter transactions_query_owners_test() -> @@ -295,7 +301,10 @@ transactions_query_owners_test() -> Opts ), ?event({transactions_query_owners_test, Res}), - assert_query_match(WrittenMsg, Res, Opts). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with recipients filter transactions_query_recipients_test() -> @@ -341,7 +350,10 @@ transactions_query_recipients_test() -> Opts ), ?event({transactions_query_recipients_test, Res}), - assert_query_match(WrittenMsg, Res, Opts). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with ids filter transactions_query_ids_test() -> @@ -385,7 +397,10 @@ transactions_query_ids_test() -> Opts ), ?event({transactions_query_ids_test, Res}), - assert_query_match(WrittenMsg, Res, Opts). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with combined filters transactions_query_combined_test() -> @@ -434,10 +449,13 @@ transactions_query_combined_test() -> Opts ), ?event({transactions_query_combined_test, Res}), - assert_query_match(WrittenMsg, Res, Opts). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). -%% @doc Test single transaction query by ID -transaction_query_by_id_test() -> +%% @doc Test single transaction query by signed ID +transaction_query_by_signed_id_test() -> Opts = #{ priv_wallet => hb:wallet(), @@ -471,6 +489,7 @@ transaction_query_by_id_test() -> }, Opts ), + ?event({written_msg, WrittenMsg}), ?event({expected_id, ExpectedID}), ?event({transaction_query_by_id_test, Res}), ?assertMatch( @@ -486,6 +505,54 @@ transaction_query_by_id_test() -> Res ). +%% @doc Arweave grapqhql filters out unsigned IDs +transaction_query_by_unsigned_id_test() -> + Opts = + #{ + priv_wallet => hb:wallet(), + store => [hb_test_utils:test_store(hb_store_lmdb)] + }, + Node = hb_http_server:start_node(Opts), + {ok, WrittenMsg} = write_test_message(Opts), + ExpectedID = hb_message:id(WrittenMsg, unsigned, Opts), + ?assertMatch( + {ok, [_]}, + hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) + ), + Query = + <<""" + query($id: ID!) { + transaction(id: $id) { + id + tags { + name + value + } + } + } + """>>, + Res = + dev_query_graphql:test_query( + Node, + Query, + #{ + <<"id">> => ExpectedID + }, + Opts + ), + ?event({written_msg, WrittenMsg}), + ?event({expected_id, ExpectedID}), + ?event({transaction_query_by_id_test, Res}), + ?assertMatch( + #{ + <<"data">> := #{ + <<"transaction">> := null + } + }, + Res + ). + + %% @doc Test single transaction query with more fields transaction_query_full_test() -> Opts = @@ -656,11 +723,8 @@ transaction_query_with_anchor_test() -> Res ). -assert_query_match(WrittenMsg, Res, Opts) -> - SignedID = hb_message:id(WrittenMsg, signed, Opts), - UnsignedID = hb_message:id(WrittenMsg, unsigned, Opts), - ?event({signed_id, SignedID}), - ?event({unsigned_id, UnsignedID}), +assert_query_match(ExpectedIDs, Res, _Opts) -> + ?event({expected_ids, ExpectedIDs}), % asset shape of response ?assertMatch( #{ @@ -674,19 +738,11 @@ assert_query_match(WrittenMsg, Res, Opts) -> <<"tags">> := [#{ <<"name">> := _, <<"value">> := _ }|_] } - }, - #{ - <<"node">> := - #{ - <<"id">> := _, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } } - ] + | _] } } - } when ?IS_ID(SignedID) andalso ?IS_ID(UnsignedID), + }, Res ), % assert that the correct IDs are in the response in any order @@ -707,5 +763,4 @@ assert_query_match(WrittenMsg, Res, Opts) -> } } <- Edges ], - ExpectedIDs = lists:sort([SignedID, UnsignedID]), - ?assertEqual(ExpectedIDs, lists:sort(ActualIDs)). \ No newline at end of file + ?assertEqual(lists:sort(ExpectedIDs), lists:sort(ActualIDs)). \ No newline at end of file From 64d52ec2df430bfea1bbca6dc10ef6255bfc27ed Mon Sep 17 00:00:00 2001 From: James Piechota Date: Tue, 25 Nov 2025 17:26:09 -0500 Subject: [PATCH 17/22] fix: revert change to add default commitment --- src/dev_codec_ans104.erl | 15 ++++++--------- src/dev_codec_ans104_from.erl | 2 +- src/dev_codec_tx.erl | 4 +--- src/hb_message.erl | 6 +++--- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/dev_codec_ans104.erl b/src/dev_codec_ans104.erl index 38e2e1332..332797478 100644 --- a/src/dev_codec_ans104.erl +++ b/src/dev_codec_ans104.erl @@ -76,10 +76,7 @@ commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> Opts ), ?event({committed, Committed}), - WithNormalizedCommitments = hb_message:normalize_commitments( - Committed, Opts, add), - ?event({with_normalized_commitments, WithNormalizedCommitments}), - {ok, WithNormalizedCommitments}. + {ok, Committed}. %% @doc Verify an ANS-104 commitment. verify(Msg, Req, Opts) -> @@ -552,7 +549,7 @@ unsigned_mixedcase_bundle_list_tags_1_test() -> ], UnsignedTX#tx.tags), {ok, UnsignedTABM} = dev_codec_ans104:from(UnsignedTX, #{}, #{}), ?event(debug_test, {tabm, UnsignedTABM}), - Commitment = hb_message:commitment( + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(UnsignedTX#tx.unsigned_id), UnsignedTABM), ?event(debug_test, {commitment, Commitment}), ExpectedCommitment = #{ @@ -600,7 +597,7 @@ unsigned_mixedcase_bundle_list_tags_2_test() -> ], UnsignedTX#tx.tags), {ok, UnsignedTABM} = dev_codec_ans104:from(UnsignedTX, #{}, #{}), ?event(debug_test, {tabm, UnsignedTABM}), - Commitment = hb_message:commitment( + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(UnsignedTX#tx.unsigned_id), UnsignedTABM), ?event(debug_test, {commitment, Commitment}), ExpectedCommitment = #{ @@ -649,7 +646,7 @@ unsigned_mixedcase_bundle_map_tags_test() -> ], UnsignedTX#tx.tags), {ok, UnsignedTABM} = dev_codec_ans104:from(UnsignedTX, #{}, #{}), ?event(debug_test, {tabm, UnsignedTABM}), - Commitment = hb_message:commitment( + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(UnsignedTX#tx.unsigned_id), UnsignedTABM), ?event(debug_test, {commitment, Commitment}), ExpectedCommitment = #{ @@ -696,7 +693,7 @@ signed_lowercase_bundle_map_tags_test() -> ?event({signed_tabm, SignedTABM}), % Recursively exclude commitments from the SignedTABM for the match test. ?assert(hb_message:match(UnsignedTABM, SignedTABM, only_present, #{})), - Commitment = hb_message:commitment( + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(SignedTX#tx.id), SignedTABM), ?event({commitment, Commitment}), ExpectedCommitment = #{ @@ -755,7 +752,7 @@ signed_mixedcase_bundle_map_tags_test() -> ?event(debug_test, {signed_tabm, SignedTABM}), % Recursively exclude commitments from the SignedTABM for the match test. ?assert(hb_message:match(UnsignedTABM, SignedTABM, only_present, #{})), - Commitment = hb_message:commitment( + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(SignedTX#tx.id), SignedTABM), ?event(debug_test, {commitment, Commitment}), ExpectedCommitment = #{ diff --git a/src/dev_codec_ans104_from.erl b/src/dev_codec_ans104_from.erl index 1a613cb2f..1e4a6cd8f 100644 --- a/src/dev_codec_ans104_from.erl +++ b/src/dev_codec_ans104_from.erl @@ -191,7 +191,7 @@ with_commitments( } } end, - hb_message:normalize_commitments(WithCommitments, Opts, add). + WithCommitments. %% @doc Returns a commitments message for an item, containing an unsigned %% commitment. diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index def7ed4cd..77e2859c1 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -58,9 +58,7 @@ commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> Opts ), ?event({committed, Committed}), - WithNormalizedCommitments = hb_message:normalize_commitments(Committed, Opts), - ?event({with_normalized_commitments, WithNormalizedCommitments}), - {ok, WithNormalizedCommitments}. + {ok, Committed}. %% @doc Verify an L1 TX commitment. verify(Msg, Req, Opts) -> diff --git a/src/hb_message.erl b/src/hb_message.erl index 2f0d2c8c0..eca0ee559 100644 --- a/src/hb_message.erl +++ b/src/hb_message.erl @@ -188,7 +188,7 @@ id(Msg, RawCommitters, Opts) -> signed -> #{ <<"committers">> => <<"all">> }; List when is_list(List) -> #{ <<"committers">> => List } end, - ?event(debug_test, {getting_id, {msg, Msg}, {spec, CommSpec}}), + ?event({getting_id, {msg, Msg}, {spec, CommSpec}}), {ok, ID} = dev_message:id( Msg, @@ -298,11 +298,11 @@ do_normalize_commitments(Msg, Opts, verify) -> Opts ); {_OldID, _NewID} -> - NormCommitments = generate_norm_commitments(Msg, #{}, Opts), + NewCommitments = generate_norm_commitments(Msg, #{}, Opts), % We had an unsigned ID to begin with and the new one is different. % This means that the committed keys have changed, so we drop any % other commitments and return only the new unsigned one. - attach_phash2(Msg#{ <<"commitments">> => NormCommitments }, Opts) + attach_phash2(Msg#{ <<"commitments">> => NewCommitments }, Opts) end; do_normalize_commitments(Msg, Opts, fast) when is_map(Msg) -> ExpectedHash = erlang:phash2(hb_private:reset(Msg)), From f930bd62fc599ac1c9beed21c50f6a2b0bff09ad Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 2 Dec 2025 12:08:14 -0500 Subject: [PATCH 18/22] wip: use all known unsigned IDs when `committers: none` --- src/dev_message.erl | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/dev_message.erl b/src/dev_message.erl index 1f653cf76..c2f80b8e9 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -463,18 +463,23 @@ with_relevant_commitments(Base, Req, Opts) -> %% the default is `all' for commitments -- also implying `all' for committers. commitment_ids_from_request(Base, Req, Opts) -> Commitments = maps:get(<<"commitments">>, Base, #{}), - ReqCommitters = - case maps:get(<<"committers">>, Req, <<"none">>) of - X when is_list(X) -> X; - CommitterDescriptor -> hb_ao:normalize_key(CommitterDescriptor) - end, RawReqCommitments = maps:get(<<"commitment-ids">>, Req, <<"none">>), ReqCommitments = case RawReqCommitments of X2 when is_list(X2) -> X2; CommitmentDescriptor -> hb_ao:normalize_key(CommitmentDescriptor) end, - % Get the commitments to verify. + ReqCommitters = + case maps:get(<<"committers">>, Req, <<"none">>) of + X when is_list(X) -> X; + CommitterDescriptor -> hb_ao:normalize_key(CommitterDescriptor) + end, + ?event(debug_commitments, + {commitment_ids_from_request, + {req_commitments, ReqCommitments}, + {req_committers, ReqCommitters}} + ), + % Get the commitments to return from explicit commitment IDs. FromCommitmentIDs = case ReqCommitments of <<"none">> -> []; @@ -509,13 +514,10 @@ commitment_ids_from_request(Base, Req, Opts) -> % commitment device, if it exists. lists:filter( fun(CommitmentID) -> - Comm = maps:get(CommitmentID, Commitments), - Dev = maps:get(<<"commitment-device">>, Comm, undefined), - case Dev of - ?DEFAULT_ATT_DEVICE -> - not hb_maps:is_key(<<"committer">>, Comm); - _ -> false - end + not maps:is_key( + <<"committer">>, + maps:get(CommitmentID, Commitments) + ) end, maps:keys(Commitments) ); From 7a88d63d8f3a2f3867737714c6b5c96c227889a1 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Tue, 2 Dec 2025 13:40:18 -0500 Subject: [PATCH 19/22] fix: revert change to filter unsigned messages from graphql --- src/dev_query_arweave.erl | 44 +++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/src/dev_query_arweave.erl b/src/dev_query_arweave.erl index f063d26a2..4201a6723 100644 --- a/src/dev_query_arweave.erl +++ b/src/dev_query_arweave.erl @@ -24,10 +24,14 @@ query(List, <<"edges">>, _Args, _Opts) -> query(Msg, <<"node">>, _Args, _Opts) -> {ok, Msg}; query(Obj, <<"transaction">>, #{ <<"id">> := ID }, Opts) -> - Messages = read_filtered_messages([ID], Opts), - case Messages of - [Msg] -> {ok, Msg}; - [] -> {ok, null} + case hb_cache:read(ID, Opts) of + {ok, Msg} -> + case hb_message:commitment(ID, Msg, Opts) of + {ok, ID, Comm} -> + {ok, Msg#{ <<"commitments">> => #{ ID => Comm } }}; + not_found -> {ok, null} + end; + not_found -> {ok, null} end; query(Obj, <<"transactions">>, Args, Opts) -> ?event({transactions_query, @@ -37,7 +41,16 @@ query(Obj, <<"transactions">>, Args, Opts) -> }), Matches = match_args(Args, Opts), ?event({transactions_matches, Matches}), - Messages = read_filtered_messages(Matches, Opts), + Messages = + lists:filtermap( + fun(Match) -> + case hb_cache:read(Match, Opts) of + {ok, Msg} -> {true, Msg}; + not_found -> false + end + end, + Matches + ), ?event({transactions_messages, Messages}), {ok, Messages}; query(Obj, <<"block">>, Args, Opts) -> @@ -302,27 +315,8 @@ all_ids([ID | Rest], Results, Opts) -> end, all_ids(Rest, [IDs | Results], Opts). -%% @doc Read the provided IDs and remove any messages that don't exist or -%% are unsigned. -read_filtered_messages(IDs, Opts) -> - lists:filtermap( - fun(ID) -> - case hb_cache:read(ID, Opts) of - {ok, Msg} -> - case hb_message:commitment(ID, Msg, Opts) of - {ok, ID, #{ <<"committer">> := _ } = Comm} -> - {true, Msg#{ <<"commitments">> => #{ ID => Comm } }}; - {ok, _ID, _CommWithoutCommitter} -> false; - not_found -> false - end; - not_found -> false - end - end, - IDs - ). - %% @doc Scope the stores used for block matching. The searched stores can be %% scoped by setting the `query_arweave_scope' option. scope(Opts) -> Scope = hb_opts:get(query_arweave_scope, [local], Opts), - hb_store:scope(Opts, Scope). + hb_store:scope(Opts, Scope). \ No newline at end of file From 6dbafdde283a443d43ac988ec7e51a8a8c4e8a56 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Tue, 2 Dec 2025 14:48:53 -0500 Subject: [PATCH 20/22] fix: fix hb_store_gateway:remote_hyperbeam_node_ans104_test Update the test, and implement verify for unsigned ans104 --- src/dev_codec_ans104.erl | 10 ++++++++-- src/hb_store_gateway.erl | 19 ++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/dev_codec_ans104.erl b/src/dev_codec_ans104.erl index 332797478..281279d31 100644 --- a/src/dev_codec_ans104.erl +++ b/src/dev_codec_ans104.erl @@ -92,8 +92,14 @@ verify(Msg, Req, Opts) -> ?event({verify, {only_with_commitment, OnlyWithCommitment}}), {ok, TX} = to(OnlyWithCommitment, Req, Opts), ?event({verify, {encoded, TX}}), - Res = ar_bundles:verify_item(TX), - {ok, Res}. + case maps:get(<<"type">>, Req) of + <<"rsa-pss-sha256">> -> + {ok, ar_bundles:verify_item(TX)}; + <<"unsigned-sha256">> -> + ID = hb_util:human_id(TX#tx.unsigned_id), + Signature = maps:get(<<"signature">>, Req), + {ok, Signature =:= ID } + end. %% @doc Convert a #tx record into a message map recursively. from(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, Binary}; diff --git a/src/hb_store_gateway.erl b/src/hb_store_gateway.erl index 57f8e3083..8dd20271b 100644 --- a/src/hb_store_gateway.erl +++ b/src/hb_store_gateway.erl @@ -382,9 +382,19 @@ remote_hyperbeam_node_ans104_test() -> ServerOpts, #{ <<"commitment-device">> => <<"ans104@1.0">> } ), + UnsignedID = hb_message:id(Msg, none, ServerOpts), + SignedID = hb_message:id(Msg, all, ServerOpts), + ?assertNotEqual(UnsignedID, SignedID), + ?event(debug_test, {{unsigned_id, UnsignedID}, {signed_id, SignedID}}), {ok, ID} = hb_cache:write(Msg, ServerOpts), - {ok, ReadMsg} = hb_cache:read(ID, ServerOpts), - ?assert(hb_message:verify(ReadMsg)), + ?assertEqual(UnsignedID, ID), + {ok, UnsignedMsg} = hb_cache:read(UnsignedID, ServerOpts), + {ok, SignedMsg} = hb_cache:read(SignedID, ServerOpts), + ?event(debug_test, { + {written_id, ID}, {written_msg, Msg}, + {unsigned_msg, UnsignedMsg}, {signed_msg, SignedMsg}}), + ?assert(hb_message:verify(UnsignedMsg)), + ?assert(hb_message:verify(SignedMsg)), LocalStore = hb_test_utils:test_store(), ClientOpts = #{ @@ -398,6 +408,9 @@ remote_hyperbeam_node_ans104_test() -> } ] }, - {ok, Req} = hb_cache:read(ID, ClientOpts), + % Unsigned dataitems can not be synced from the remote node via graphql. + ?assertEqual(not_found, hb_cache:read(UnsignedID, ClientOpts)), + % But signed dataitems can + {ok, Req} = hb_cache:read(SignedID, ClientOpts), ?assert(hb_message:verify(Req)), ?assert(hb_message:match(Msg, Req)). \ No newline at end of file From 42c1e3dd85efe5a7c2da1871309f6c423d7062c9 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Tue, 2 Dec 2025 15:31:36 -0500 Subject: [PATCH 21/22] fix: revert "use all knonw unsigned IDs", update tests --- src/dev_message.erl | 15 ++++++++--- src/dev_query_test_vectors.erl | 49 +--------------------------------- src/hb_store_gateway.erl | 2 +- 3 files changed, 13 insertions(+), 53 deletions(-) diff --git a/src/dev_message.erl b/src/dev_message.erl index c2f80b8e9..c654db1e5 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -514,10 +514,17 @@ commitment_ids_from_request(Base, Req, Opts) -> % commitment device, if it exists. lists:filter( fun(CommitmentID) -> - not maps:is_key( - <<"committer">>, - maps:get(CommitmentID, Commitments) - ) + Comm = maps:get(CommitmentID, Commitments), + Dev = maps:get(<<"commitment-device">>, Comm, undefined), + case Dev of + ?DEFAULT_ATT_DEVICE -> + not hb_maps:is_key(<<"committer">>, Comm); + _ -> false + end + % not maps:is_key( + % <<"committer">>, + % maps:get(CommitmentID, Commitments) + % ) end, maps:keys(Commitments) ); diff --git a/src/dev_query_test_vectors.erl b/src/dev_query_test_vectors.erl index 5456d4936..59f7404f4 100644 --- a/src/dev_query_test_vectors.erl +++ b/src/dev_query_test_vectors.erl @@ -256,6 +256,7 @@ transactions_query_tags_test() -> ), ?event({transactions_query_tags_test, Res}), ExpectedIDs = [ + hb_message:id(WrittenMsg, none, Opts), hb_message:id(WrittenMsg, signed, Opts) ], assert_query_match(ExpectedIDs, Res, Opts). @@ -505,54 +506,6 @@ transaction_query_by_signed_id_test() -> Res ). -%% @doc Arweave grapqhql filters out unsigned IDs -transaction_query_by_unsigned_id_test() -> - Opts = - #{ - priv_wallet => hb:wallet(), - store => [hb_test_utils:test_store(hb_store_lmdb)] - }, - Node = hb_http_server:start_node(Opts), - {ok, WrittenMsg} = write_test_message(Opts), - ExpectedID = hb_message:id(WrittenMsg, unsigned, Opts), - ?assertMatch( - {ok, [_]}, - hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) - ), - Query = - <<""" - query($id: ID!) { - transaction(id: $id) { - id - tags { - name - value - } - } - } - """>>, - Res = - dev_query_graphql:test_query( - Node, - Query, - #{ - <<"id">> => ExpectedID - }, - Opts - ), - ?event({written_msg, WrittenMsg}), - ?event({expected_id, ExpectedID}), - ?event({transaction_query_by_id_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transaction">> := null - } - }, - Res - ). - - %% @doc Test single transaction query with more fields transaction_query_full_test() -> Opts = diff --git a/src/hb_store_gateway.erl b/src/hb_store_gateway.erl index 8dd20271b..de1b3dc07 100644 --- a/src/hb_store_gateway.erl +++ b/src/hb_store_gateway.erl @@ -408,7 +408,7 @@ remote_hyperbeam_node_ans104_test() -> } ] }, - % Unsigned dataitems can not be synced from the remote node via graphql. + % Unsigned dataitems can not be synced from the remote node via graphql ?assertEqual(not_found, hb_cache:read(UnsignedID, ClientOpts)), % But signed dataitems can {ok, Req} = hb_cache:read(SignedID, ClientOpts), From 576e9b2248cc9464f73d610fc02141d0a4677a83 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Tue, 2 Dec 2025 16:13:48 -0500 Subject: [PATCH 22/22] fix: speed up dev_bundler tests --- src/dev_bundler.erl | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index 3035f11e5..96e37f979 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -202,9 +202,9 @@ tx_error_test() -> ?assertMatch({ok, _}, post_data_item(Node, Item1, ClientOpts)), % After a tx request fails it should be retried indefinitely. We'll % wait for a few retries then continue. - TXs = hb_mock_server:get_requests(tx, 4, ServerHandle), - ?assert(length(TXs) >= 4), - Chunks = hb_mock_server:get_requests(chunk, 1, ServerHandle), + TXs = hb_mock_server:get_requests(tx, 2, ServerHandle), + ?assert(length(TXs) >= 2), + Chunks = hb_mock_server:get_requests(chunk, 1, ServerHandle, 500), ?assertEqual([], Chunks), ok after @@ -256,7 +256,7 @@ idle_test() -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - bundler_max_idle_time => 10000, + bundler_max_idle_time => 2000, priv_wallet => hb:wallet(), store => hb_test_utils:test_store(hb_store_lmdb) }), @@ -265,12 +265,12 @@ idle_test() -> ?assertMatch({ok, _}, post_data_item(Node, Item1, ClientOpts)), % Wait just to give the server a chance to post a transaction % (but it shouldn't) - timer:sleep(2000), + timer:sleep(1000), ?assertEqual(0, length(hb_mock_server:get_requests(tx, 0, ServerHandle))), ?assertEqual(0, length(hb_mock_server:get_requests(chunk, 0, ServerHandle))), % Wait gain to give the server a chance to trip the max idle time. % It should *now* post a transaction. - timer:sleep(8000), + timer:sleep(1000), TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), ?assertEqual(1, length(TXs)), %% Wait for expected chunks @@ -284,7 +284,7 @@ idle_test() -> end. dispatch_blocking_test() -> - BlockTime = 10000, + BlockTime = 2000, Anchor = rand:bytes(32), Price = 12345, % NodeOpts redirects arweave gateway requests to the mock server. @@ -327,7 +327,6 @@ dispatch_blocking_test() -> {slowest, Slowest}, {max_allowed, 2 * Slowest} }), ?assert(Time4 =< 2 * Slowest), - timer:sleep(BlockTime), TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), ?assertEqual(1, length(TXs)), %% Wait for expected chunks @@ -475,11 +474,11 @@ test_api_error(Responses) -> }), Item1 = new_data_item(1, floor(2.5 * ?DATA_CHUNK_SIZE)), ?assertMatch({ok, _}, post_data_item(Node, Item1, ClientOpts)), - % Since thre was an error either before or while posting the tx, + % Since there was an error either before or while posting the tx, % no bundles should be posted and no chunks should be posted. - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle, 1000), ?assertEqual([], TXs), - Chunks = hb_mock_server:get_requests(chunk, 1, ServerHandle), + Chunks = hb_mock_server:get_requests(chunk, 1, ServerHandle, 1000), ?assertEqual([], Chunks), % Now that we dispatch asynchronously, an error won't cause the % Item to remain in the queue. Instead we'll rely on the retry