diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index 7b2bc0d77..96e37f979 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -202,9 +202,9 @@ tx_error_test() -> ?assertMatch({ok, _}, post_data_item(Node, Item1, ClientOpts)), % After a tx request fails it should be retried indefinitely. We'll % wait for a few retries then continue. - TXs = hb_mock_server:get_requests(tx, 4, ServerHandle), - ?assert(length(TXs) >= 4), - Chunks = hb_mock_server:get_requests(chunk, 1, ServerHandle), + TXs = hb_mock_server:get_requests(tx, 2, ServerHandle), + ?assert(length(TXs) >= 2), + Chunks = hb_mock_server:get_requests(chunk, 1, ServerHandle, 500), ?assertEqual([], Chunks), ok after @@ -256,7 +256,7 @@ idle_test() -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - bundler_max_idle_time => 10000, + bundler_max_idle_time => 2000, priv_wallet => hb:wallet(), store => hb_test_utils:test_store(hb_store_lmdb) }), @@ -265,12 +265,12 @@ idle_test() -> ?assertMatch({ok, _}, post_data_item(Node, Item1, ClientOpts)), % Wait just to give the server a chance to post a transaction % (but it shouldn't) - timer:sleep(2000), + timer:sleep(1000), ?assertEqual(0, length(hb_mock_server:get_requests(tx, 0, ServerHandle))), ?assertEqual(0, length(hb_mock_server:get_requests(chunk, 0, ServerHandle))), % Wait gain to give the server a chance to trip the max idle time. % It should *now* post a transaction. - timer:sleep(8000), + timer:sleep(1000), TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), ?assertEqual(1, length(TXs)), %% Wait for expected chunks @@ -284,7 +284,7 @@ idle_test() -> end. dispatch_blocking_test() -> - BlockTime = 10000, + BlockTime = 2000, Anchor = rand:bytes(32), Price = 12345, % NodeOpts redirects arweave gateway requests to the mock server. @@ -327,7 +327,6 @@ dispatch_blocking_test() -> {slowest, Slowest}, {max_allowed, 2 * Slowest} }), ?assert(Time4 =< 2 * Slowest), - timer:sleep(BlockTime), TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), ?assertEqual(1, length(TXs)), %% Wait for expected chunks @@ -346,24 +345,39 @@ dispatch_blocking_test() -> recover_unbundled_items_test() -> Opts = #{store => hb_test_utils:test_store(hb_store_lmdb)}, % Create and cache some items - Item1 = hb_message:convert(new_data_item(1, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - Item2 = hb_message:convert(new_data_item(2, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - Item3 = hb_message:convert(new_data_item(3, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), + Item1 = hb_message:convert( + new_data_item(1, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), + Item2 = hb_message:convert( + new_data_item(2, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), + Item3 = hb_message:convert( + new_data_item(3, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), ok = dev_bundler_cache:write_item(Item1, Opts), ok = dev_bundler_cache:write_item(Item2, Opts), ok = dev_bundler_cache:write_item(Item3, Opts), % Bundle Item2 with a fake TX - FakeTX = ar_tx:sign(#tx{format = 2, tags = [{<<"test">>, <<"tx">>}]}, hb:wallet()), - StructuredTX = hb_message:convert(FakeTX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), + FakeTX = ar_tx:sign( + #tx{format = 2, tags = [{<<"test">>, <<"tx">>}]}, hb:wallet()), + StructuredTX = hb_message:convert( + FakeTX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), ok = dev_bundler_cache:write_tx(StructuredTX, [Item2], Opts), % Now recover unbundled items {RecoveredItems, RecoveredBytes} = recover_unbundled_items(Opts), ?assertEqual(3924, RecoveredBytes), RecoveredItems2 = [ hb_message:with_commitments( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, Item, Opts) || Item <- RecoveredItems], - ?assertEqual(lists:sort([Item1, Item3]), lists:sort(RecoveredItems2)), + WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, Item, Opts) + || Item <- [Item1, Item3]], + ?assertEqual(lists:sort(WrittenItems), lists:sort(RecoveredItems2)), ok. recover_respects_max_items_test() -> @@ -460,11 +474,11 @@ test_api_error(Responses) -> }), Item1 = new_data_item(1, floor(2.5 * ?DATA_CHUNK_SIZE)), ?assertMatch({ok, _}, post_data_item(Node, Item1, ClientOpts)), - % Since thre was an error either before or while posting the tx, + % Since there was an error either before or while posting the tx, % no bundles should be posted and no chunks should be posted. - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle, 1000), ?assertEqual([], TXs), - Chunks = hb_mock_server:get_requests(chunk, 1, ServerHandle), + Chunks = hb_mock_server:get_requests(chunk, 1, ServerHandle, 1000), ?assertEqual([], Chunks), % Now that we dispatch asynchronously, an error won't cause the % Item to remain in the queue. Instead we'll rely on the retry diff --git a/src/dev_bundler_cache.erl b/src/dev_bundler_cache.erl index a0320c679..3899b09d1 100644 --- a/src/dev_bundler_cache.erl +++ b/src/dev_bundler_cache.erl @@ -262,8 +262,14 @@ basic_cache_test() -> ?assertEqual(<<"posted">>, get_tx_status(TX, Opts)), ok = complete_tx(TX, Opts), ?assertEqual(<<"complete">>, get_tx_status(TX, Opts)), - ?assertEqual(TX, read_cache(TXID, <<"tx@1.0">>, Opts)), - ?assertEqual(Item, read_cache(ItemID, <<"ans104@1.0">>, Opts)), + ?assertEqual( + hb_message:with_commitments( + #{ <<"type">> => <<"rsa-pss-sha256">> }, TX, Opts), + read_cache(TXID, <<"tx@1.0">>, Opts)), + ?assertEqual( + hb_message:with_commitments( + #{ <<"type">> => <<"rsa-pss-sha256">> }, Item, Opts), + read_cache(ItemID, <<"ans104@1.0">>, Opts)), ok. load_unbundled_items_test() -> @@ -278,15 +284,22 @@ load_unbundled_items_test() -> % Link item2 to a bundle, leave others unbundled ok = write_tx(TX, [Item2], Opts), % Load unbundled items - UnbundledItems1 = load_unbundled_items(Opts), - UnbundledItems2 = [ + LoadedItems1 = load_unbundled_items(Opts), + LoadedItems2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, - Item, Opts) || Item <- UnbundledItems1 + Item, Opts) || Item <- LoadedItems1 ], - UnbundledItems3 = lists:sort(UnbundledItems2), - ?event(debug_test, {unbundled_items, UnbundledItems3}), - ?assertEqual(lists:sort([Item1, Item3]), UnbundledItems3), + LoadedItems3 = lists:sort(LoadedItems2), + WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, + Item, Opts) || Item <- [Item1, Item3]], + ?event(debug_test, {loaded_items, LoadedItems3}), + ?assertEqual(lists:sort(WrittenItems), LoadedItems3), ok. load_bundle_states_test() -> @@ -320,23 +333,37 @@ load_bundled_items_test() -> ok = write_tx(TX1, [Item1, Item2], Opts), ok = write_tx(TX2, [Item3], Opts), % Load items for bundle 1 - Bundle1Items1 = load_bundled_items(tx_id(TX1, Opts), Opts), - Bundle1Items2 = [ + Bundle1LoadedItems1 = load_bundled_items(tx_id(TX1, Opts), Opts), + Bundle1LoadedItems2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, - Item, Opts) || Item <- Bundle1Items1 + Item, Opts) || Item <- Bundle1LoadedItems1 ], - Bundle1Items3 = lists:sort(Bundle1Items2), - ?assertEqual(lists:sort([Item1, Item2]), Bundle1Items3), + Bundle1LoadedItems3 = lists:sort(Bundle1LoadedItems2), + Bundle1WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, + Item, Opts) || Item <- [Item1, Item2]], + ?assertEqual(lists:sort(Bundle1WrittenItems), Bundle1LoadedItems3), % Load items for bundle 2 - Bundle2Items1 = load_bundled_items(tx_id(TX2, Opts), Opts), - Bundle2Items2 = [ + Bundle2LoadedItems1 = load_bundled_items(tx_id(TX2, Opts), Opts), + Bundle2LoadedItems2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, - Item, Opts) || Item <- Bundle2Items1 + Item, Opts) || Item <- Bundle2LoadedItems1 ], - Bundle2Items3 = lists:sort(Bundle2Items2), - ?assertEqual(lists:sort([Item3]), Bundle2Items3), + Bundle2LoadedItems3 = lists:sort(Bundle2LoadedItems2), + Bundle2WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, + Item, Opts) || Item <- [Item3]], + ?assertEqual(lists:sort(Bundle2WrittenItems), Bundle2LoadedItems3), ok. new_data_item(Index, SizeOrData, Opts) -> @@ -365,7 +392,9 @@ new_tx(Index, Opts) -> hb_message:convert(TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts). read_cache(ID, Device, Opts) -> - {ok, Resolved} = hb_ao:resolve(#{ <<"path">> => ID }, Opts), + % {ok, Resolved} = hb_ao:resolve(#{ <<"path">> => ID }, Opts), + {ok, Resolved} = hb_cache:read(ID, Opts), Loaded = hb_cache:ensure_all_loaded(Resolved, Opts), + ?event(debug_test, {loaded, Loaded}), hb_message:with_commitments( #{ <<"commitment-device">> => Device }, Loaded, Opts). \ No newline at end of file diff --git a/src/dev_bundler_dispatch.erl b/src/dev_bundler_dispatch.erl index 078067827..3951434ae 100644 --- a/src/dev_bundler_dispatch.erl +++ b/src/dev_bundler_dispatch.erl @@ -1001,8 +1001,15 @@ recover_bundles_test() -> hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) || Item <- Bundle#bundle.items], + WrittenItems = [ + hb_message:with_commitments( + #{ + <<"commitment-device">> => <<"ans104@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, + Item, Opts) || Item <- [Item1, Item2, Item3]], ?assertEqual( - lists:sort([Item1, Item2, Item3]), + lists:sort(WrittenItems), lists:sort(RecoveredItems)), ?assertEqual(tx_posted, Bundle#bundle.status), ?assert(hb_message:verify(Bundle#bundle.tx)), diff --git a/src/dev_codec_ans104.erl b/src/dev_codec_ans104.erl index 5b90ad154..281279d31 100644 --- a/src/dev_codec_ans104.erl +++ b/src/dev_codec_ans104.erl @@ -35,9 +35,11 @@ commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) -> % Convert the given message to an ANS-104 TX record, sign it, and convert % it back to a structured message. + ?event({commit, {input_message, Msg}}), {ok, TX} = to(hb_private:reset(Msg), Req, Opts), Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), Signed = ar_bundles:sign_item(TX, Wallet), + ?event({commit, {signed_item, Signed}}), SignedStructured = hb_message:convert( Signed, @@ -45,20 +47,36 @@ commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) -> <<"ans104@1.0">>, Opts ), - {ok, SignedStructured}; + ?event({commit, {signed_structured, SignedStructured}}), + commit(SignedStructured, Req#{ <<"type">> => <<"unsigned">> }, Opts); commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> % Remove the commitments from the message, convert it to ANS-104, then back. % This forces the message to be normalized and the unsigned ID to be % recalculated. - { - ok, + ?event({adding_unsigned_commitment, Msg}), + WithoutExistingUnsigned = + hb_message:without_commitments( + #{ <<"type">> => <<"unsigned-sha256">> }, + Msg, + Opts + ), + ?event({without_existing_unsigned, WithoutExistingUnsigned}), + WithoutExistingUnsignedEncoded = hb_message:convert( - hb_maps:without([<<"commitments">>], Msg, Opts), + WithoutExistingUnsigned, <<"ans104@1.0">>, <<"structured@1.0">>, Opts - ) - }. + ), + ?event({without_existing_unsigned_encoded, WithoutExistingUnsignedEncoded}), + Committed = hb_message:convert( + WithoutExistingUnsignedEncoded, + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ), + ?event({committed, Committed}), + {ok, Committed}. %% @doc Verify an ANS-104 commitment. verify(Msg, Req, Opts) -> @@ -74,8 +92,14 @@ verify(Msg, Req, Opts) -> ?event({verify, {only_with_commitment, OnlyWithCommitment}}), {ok, TX} = to(OnlyWithCommitment, Req, Opts), ?event({verify, {encoded, TX}}), - Res = ar_bundles:verify_item(TX), - {ok, Res}. + case maps:get(<<"type">>, Req) of + <<"rsa-pss-sha256">> -> + {ok, ar_bundles:verify_item(TX)}; + <<"unsigned-sha256">> -> + ID = hb_util:human_id(TX#tx.unsigned_id), + Signature = maps:get(<<"signature">>, Req), + {ok, Signature =:= ID } + end. %% @doc Convert a #tx record into a message map recursively. from(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, Binary}; @@ -108,7 +132,7 @@ do_from(RawTX, Req, Opts) -> FieldCommitments = dev_codec_ans104_from:fields(TX, ?FIELD_PREFIX, Opts), WithCommitments = dev_codec_ans104_from:with_commitments( TX, <<"ans104@1.0">>, FieldCommitments, Tags, Base, Keys, Opts), - ?event({from, {parsed_message, WithCommitments}}), + ?event({from, {result, WithCommitments}}), {ok, WithCommitments}. %% @doc Internal helper to translate a message to its #tx record representation, @@ -130,11 +154,8 @@ to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX}; to(RawTABM, Req, Opts) when is_map(RawTABM) -> % Ensure that the TABM is fully loaded if the `bundle` key is set to true. ?event({to, {inbound, RawTABM}, {req, Req}}), - MaybeCommitment = hb_message:commitment( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, - RawTABM, - Opts - ), + MaybeCommitment = dev_codec_ans104_to:commitment( + <<"ans104@1.0">>, RawTABM, Opts), IsBundle = dev_codec_ans104_to:is_bundle(MaybeCommitment, Req, Opts), MaybeBundle = dev_codec_ans104_to:maybe_load(RawTABM, IsBundle, Opts), ?event({to, {maybe_bundle, MaybeBundle}}), @@ -238,9 +259,9 @@ unsigned_duplicated_tag_name_test() -> ] }), Msg = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}), - ?event({msg, Msg}), + ?event(debug_test, {msg, Msg}), TX2 = hb_message:convert(Msg, <<"ans104@1.0">>, <<"structured@1.0">>, #{}), - ?event({tx2, TX2}), + ?event(debug_test, {tx2, TX2}), ?assertEqual(TX, TX2). signed_duplicated_tag_name_test() -> @@ -469,7 +490,7 @@ unsigned_lowercase_bundle_map_tags_test() -> } }, {ok, UnsignedTX} = dev_codec_ans104:to(UnsignedTABM, #{}, #{}), - ?event({tx, UnsignedTX}), + ?event(debug_test, {tx, UnsignedTX}), ?assertEqual([ {<<"bundle-format">>, <<"binary">>}, {<<"bundle-version">>, <<"2.0.0">>}, @@ -479,8 +500,34 @@ unsigned_lowercase_bundle_map_tags_test() -> ], UnsignedTX#tx.tags), ?assert(UnsignedTX#tx.manifest =/= undefined), {ok, TABM} = dev_codec_ans104:from(UnsignedTX, #{}, #{}), - ?event({tabm, TABM}), - ?assertEqual(UnsignedTABM, TABM). + ExpectedTABM = UnsignedTABM#{ + <<"commitments">> => #{ + <<"q6hcdZlyNre_X3L5Z7zeSkjOKUP6L88BcQ_D0JOjrLs">> => #{ + <<"bundle">> => <<"true">>, + <<"commitment-device">> => <<"ans104@1.0">>, + <<"committed">> => [<<"data">>, <<"a1">>, <<"c1">>], + <<"signature">> => <<"q6hcdZlyNre_X3L5Z7zeSkjOKUP6L88BcQ_D0JOjrLs">>, + <<"type">> => <<"unsigned-sha256">> + } + }, + <<"data">> => #{ + <<"commitments">> => #{ + <<"IkDi2KTYxvbyeJ3t0JR1wuN9vJunYI1hj3wQjFloSG8">> => #{ + <<"bundle">> => <<"false">>, + <<"commitment-device">> => <<"ans104@1.0">>, + <<"committed">> => [<<"data">>, <<"a2">>, <<"c2">>], + <<"signature">> => <<"IkDi2KTYxvbyeJ3t0JR1wuN9vJunYI1hj3wQjFloSG8">>, + <<"type">> => <<"unsigned-sha256">> + } + }, + <<"data">> => <<"testdata">>, + <<"a2">> => <<"value2">>, + <<"c2">> => <<"value3">> + } + }, + ?event(debug_test, {expected_tabm, {explicit, ExpectedTABM}}), + ?event(debug_test, {tabm, {explicit, TABM}}), + ?assertEqual(ExpectedTABM, TABM). unsigned_mixedcase_bundle_list_tags_1_test() -> UnsignedTX = dev_arweave_common:normalize(#tx{ @@ -500,7 +547,6 @@ unsigned_mixedcase_bundle_list_tags_1_test() -> } ] }), - ?event(debug_test, {unsigned_tx, UnsignedTX}), ?assertEqual([ {<<"TagA1">>, <<"value1">>}, {<<"TagA2">>, <<"value2">>}, @@ -509,7 +555,7 @@ unsigned_mixedcase_bundle_list_tags_1_test() -> ], UnsignedTX#tx.tags), {ok, UnsignedTABM} = dev_codec_ans104:from(UnsignedTX, #{}, #{}), ?event(debug_test, {tabm, UnsignedTABM}), - Commitment = hb_message:commitment( + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(UnsignedTX#tx.unsigned_id), UnsignedTABM), ?event(debug_test, {commitment, Commitment}), ExpectedCommitment = #{ @@ -525,6 +571,7 @@ unsigned_mixedcase_bundle_list_tags_1_test() -> ExpectedCommitment, hb_maps:with([<<"committed">>, <<"original-tags">>], Commitment, #{})), {ok, TX} = dev_codec_ans104:to(UnsignedTABM, #{}, #{}), + ?event(debug_test, {expected_tx, UnsignedTX}), ?event(debug_test, {tx, TX}), ?assertEqual(UnsignedTX, TX), ok. @@ -556,7 +603,7 @@ unsigned_mixedcase_bundle_list_tags_2_test() -> ], UnsignedTX#tx.tags), {ok, UnsignedTABM} = dev_codec_ans104:from(UnsignedTX, #{}, #{}), ?event(debug_test, {tabm, UnsignedTABM}), - Commitment = hb_message:commitment( + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(UnsignedTX#tx.unsigned_id), UnsignedTABM), ?event(debug_test, {commitment, Commitment}), ExpectedCommitment = #{ @@ -605,7 +652,7 @@ unsigned_mixedcase_bundle_map_tags_test() -> ], UnsignedTX#tx.tags), {ok, UnsignedTABM} = dev_codec_ans104:from(UnsignedTX, #{}, #{}), ?event(debug_test, {tabm, UnsignedTABM}), - Commitment = hb_message:commitment( + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(UnsignedTX#tx.unsigned_id), UnsignedTABM), ?event(debug_test, {commitment, Commitment}), ExpectedCommitment = #{ @@ -650,8 +697,9 @@ signed_lowercase_bundle_map_tags_test() -> ?assert(SignedTX#tx.manifest =/= undefined), {ok, SignedTABM} = dev_codec_ans104:from(SignedTX, #{}, #{}), ?event({signed_tabm, SignedTABM}), - ?assertEqual(UnsignedTABM, hb_maps:without([<<"commitments">>], SignedTABM)), - Commitment = hb_message:commitment( + % Recursively exclude commitments from the SignedTABM for the match test. + ?assert(hb_message:match(UnsignedTABM, SignedTABM, only_present, #{})), + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(SignedTX#tx.id), SignedTABM), ?event({commitment, Commitment}), ExpectedCommitment = #{ @@ -708,8 +756,9 @@ signed_mixedcase_bundle_map_tags_test() -> ?assert(SignedTX#tx.manifest =/= undefined), {ok, SignedTABM} = dev_codec_ans104:from(SignedTX, #{}, #{}), ?event(debug_test, {signed_tabm, SignedTABM}), - ?assertEqual(UnsignedTABM, hb_maps:without([<<"commitments">>], SignedTABM)), - Commitment = hb_message:commitment( + % Recursively exclude commitments from the SignedTABM for the match test. + ?assert(hb_message:match(UnsignedTABM, SignedTABM, only_present, #{})), + {ok, _, Commitment} = hb_message:commitment( hb_util:human_id(SignedTX#tx.id), SignedTABM), ?event(debug_test, {commitment, Commitment}), ExpectedCommitment = #{ @@ -762,7 +811,8 @@ test_bundle_commitment(Commit, Encode, Decode) -> #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => ToBool(Commit) }), ?event(debug_test, {committed, Label, {explicit, Committed}}), ?assert(hb_message:verify(Committed, all, Opts), Label), - {ok, _, CommittedCommitment} = hb_message:commitment(#{}, Committed, Opts), + {ok, _, CommittedCommitment} = hb_message:commitment( + #{ <<"type">> => <<"rsa-pss-sha256">> }, Committed, Opts), ?assertEqual( [<<"list">>], hb_maps:get(<<"committed">>, CommittedCommitment, Opts), Label), @@ -784,7 +834,8 @@ test_bundle_commitment(Commit, Encode, Decode) -> Opts), ?event(debug_test, {decoded, Label, {explicit, Decoded}}), ?assert(hb_message:verify(Decoded, all, Opts), Label), - {ok, _, DecodedCommitment} = hb_message:commitment(#{}, Decoded, Opts), + {ok, _, DecodedCommitment} = hb_message:commitment( + #{ <<"type">> => <<"rsa-pss-sha256">> }, Decoded, Opts), ?assertEqual( [<<"list">>], hb_maps:get(<<"committed">>, DecodedCommitment, Opts), Label), diff --git a/src/dev_codec_ans104_from.erl b/src/dev_codec_ans104_from.erl index a60e90505..1e4a6cd8f 100644 --- a/src/dev_codec_ans104_from.erl +++ b/src/dev_codec_ans104_from.erl @@ -159,50 +159,66 @@ base(CommittedKeys, Fields, Tags, Data, Opts) -> %% @doc Return a message with the appropriate commitments added to it. with_commitments( Item, Device, FieldCommitments, Tags, Base, CommittedKeys, Opts) -> - case Item#tx.signature of - ?DEFAULT_SIG -> - case normal_tags(Item#tx.tags) of - true -> Base; - false -> - with_unsigned_commitment( - Item, Device, FieldCommitments, Tags, Base, - CommittedKeys, Opts) - end; - _ -> with_signed_commitment( - Item, Device, FieldCommitments, Tags, Base, CommittedKeys, Opts) - end. + {UnsignedID, UnsignedCommitment} = + unsigned_commitment( + Item, + Device, + FieldCommitments, + Tags, + CommittedKeys, + Opts + ), + WithCommitments = + case Item#tx.signature of + ?DEFAULT_SIG -> + Base#{ <<"commitments">> => + #{ UnsignedID => UnsignedCommitment } }; + _ -> + {SignedID, SignedCommitment} = + signed_commitment( + Item, + Device, + FieldCommitments, + Tags, + CommittedKeys, + Opts + ), + Base#{ + <<"commitments">> => + #{ + SignedID => SignedCommitment, + UnsignedID => UnsignedCommitment + } + } + end, + WithCommitments. %% @doc Returns a commitments message for an item, containing an unsigned %% commitment. -with_unsigned_commitment( - Item, Device, CommittedFields, Tags, - UncommittedMessage, CommittedKeys, Opts) -> +unsigned_commitment(Item, Device, CommittedFields, Tags, CommittedKeys, Opts) -> ID = hb_util:human_id(Item#tx.unsigned_id), - UncommittedMessage#{ - <<"commitments">> => #{ - ID => - filter_unset( - hb_maps:merge( - CommittedFields, - #{ - <<"commitment-device">> => Device, - <<"committed">> => CommittedKeys, - <<"type">> => <<"unsigned-sha256">>, - <<"bundle">> => bundle_commitment_key(Tags, Opts), - <<"original-tags">> => original_tags(Item, Opts) - }, - Opts - ), - Opts - ) - } + { + ID, + filter_unset( + hb_maps:merge( + CommittedFields, + #{ + <<"commitment-device">> => Device, + <<"committed">> => CommittedKeys, + <<"type">> => <<"unsigned-sha256">>, + <<"signature">> => ID, + <<"bundle">> => bundle_commitment_key(Tags, Opts), + <<"original-tags">> => original_tags(Item, Opts) + }, + Opts + ), + Opts + ) }. %% @doc Returns a commitments message for an item, containing a signed %% commitment. -with_signed_commitment( - Item, Device, FieldCommitments, Tags, - UncommittedMessage, CommittedKeys, Opts) -> +signed_commitment(Item, Device, FieldCommitments, Tags, CommittedKeys, Opts) -> Address = hb_util:human_id(ar_wallet:to_address(Item#tx.owner)), ID = hb_util:human_id(Item#tx.id), ExtraCommitments = hb_maps:merge( @@ -229,11 +245,7 @@ with_signed_commitment( ), Opts ), - UncommittedMessage#{ - <<"commitments">> => #{ - ID => Commitment - } - }. + {ID, Commitment}. %% @doc Return the bundle key for an item. bundle_commitment_key(Tags, Opts) -> diff --git a/src/dev_codec_ans104_to.erl b/src/dev_codec_ans104_to.erl index 68a6ba033..2be8fc963 100644 --- a/src/dev_codec_ans104_to.erl +++ b/src/dev_codec_ans104_to.erl @@ -1,7 +1,7 @@ %%% @doc Library functions for encoding messages to the ANS-104 format. -module(dev_codec_ans104_to). -export([is_bundle/3, maybe_load/3, data/3, tags/5, excluded_tags/3]). --export([siginfo/4, fields_to_tx/4]). +-export([commitment/3, siginfo/4, fields_to_tx/4]). -include("include/hb.hrl"). is_bundle({ok, _, Commitment}, _Req, Opts) -> @@ -31,12 +31,61 @@ maybe_load(RawTABM, true, Opts) -> Opts ), % Ensure the commitments from the original message are the only - % ones in the fully loaded message. - LoadedComms = maps:get(<<"commitments">>, RawTABM, #{}), - LoadedTABM#{ <<"commitments">> => LoadedComms }; + % ones in the fully loaded message, recursively for nested maps. + replace_commitments_recursive(LoadedTABM, RawTABM); maybe_load(RawTABM, false, _Opts) -> RawTABM. +%% @doc Recursively replace commitments from RawTABM into LoadedTABM. +%% For each nested map in LoadedTABM, if there's a corresponding map in RawTABM, +%% recursively apply commitment replacement. +replace_commitments_recursive(LoadedTABM, RawTABM) + when is_map(LoadedTABM), is_map(RawTABM) -> + % First, replace commitments at this level + RawCommitments = maps:get(<<"commitments">>, RawTABM, #{}), + LoadedTABM2 = LoadedTABM#{ <<"commitments">> => RawCommitments }, + % Then recursively process nested maps, but do not recurse into the + % `commitments' map itself (we only replace it at each level). + maps:map( + fun(<<"commitments">>, Value) -> + Value; + (Key, Value) when is_map(Value) -> + case maps:get(Key, RawTABM, undefined) of + RawValue when is_map(RawValue) -> + replace_commitments_recursive(Value, RawValue); + _ -> + Value + end; + (_Key, Value) -> + Value + end, + LoadedTABM2 + ); +replace_commitments_recursive(LoadedTABM, _RawTABM) -> + LoadedTABM. + +commitment(Device, TABM, Opts) -> + SignedCommitment = hb_message:commitment( + #{ + <<"commitment-device">> => Device, + <<"type">> => <<"rsa-pss-sha256">> + }, + TABM, + Opts + ), + case SignedCommitment of + {ok, _, _} -> SignedCommitment; + _ -> + hb_message:commitment( + #{ + <<"commitment-device">> => Device, + <<"type">> => <<"unsigned-sha256">> + }, + TABM, + Opts + ) + end. + %% @doc Calculate the fields for a message, returning an initial TX record. %% One of the nuances here is that the `target' field must be set correctly. %% If the message has a commitment, we extract the `field-target' if found and @@ -55,10 +104,7 @@ siginfo(Message, multiple_matches, _FieldsFun, _Opts) -> %% tags, and last TX from the commitment. If the value is not present, the %% default value is used. commitment_to_tx(Commitment, FieldsFun, Opts) -> - Signature = - hb_util:decode( - maps:get(<<"signature">>, Commitment, hb_util:encode(?DEFAULT_SIG)) - ), + Signature = signature_from_commitment(Commitment), Owner = case hb_maps:find(<<"keyid">>, Commitment, Opts) of {ok, KeyID} -> @@ -82,6 +128,11 @@ commitment_to_tx(Commitment, FieldsFun, Opts) -> }, FieldsFun(TX, ?FIELD_PREFIX, Commitment, Opts). +signature_from_commitment(#{ <<"type">> := <<"unsigned-sha256">> }) -> + ?DEFAULT_SIG; +signature_from_commitment(Commitment) -> + hb_util:decode(maps:get(<<"signature">>, Commitment)). + %% @doc Convert a HyperBEAM-compatible map into an ANS-104 encoded tag list, %% recreating the original order of the tags. diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index 3363969d2..77e2859c1 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -29,20 +29,36 @@ commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) -> <<"tx@1.0">>, Opts ), - {ok, SignedStructured}; + ?event({commit, {signed_structured, SignedStructured}}), + commit(SignedStructured, Req#{ <<"type">> => <<"unsigned">> }, Opts); commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> % Remove the commitments from the message, convert it to an L1 TX, % then back. This forces the message to be normalized and the unsigned ID % to be recalculated. - { - ok, + ?event({adding_unsigned_commitment, Msg}), + WithoutExistingUnsigned = + hb_message:without_commitments( + #{ <<"type">> => <<"unsigned-sha256">> }, + Msg, + Opts + ), + ?event({without_existing_unsigned, WithoutExistingUnsigned}), + WithoutExistingUnsignedEncoded = hb_message:convert( - hb_maps:without([<<"commitments">>], Msg, Opts), + WithoutExistingUnsigned, <<"tx@1.0">>, <<"structured@1.0">>, Opts - ) - }. + ), + ?event({without_existing_unsigned_encoded, WithoutExistingUnsignedEncoded}), + Committed = hb_message:convert( + WithoutExistingUnsignedEncoded, + <<"structured@1.0">>, + <<"tx@1.0">>, + Opts + ), + ?event({committed, Committed}), + {ok, Committed}. %% @doc Verify an L1 TX commitment. verify(Msg, Req, Opts) -> @@ -117,11 +133,8 @@ to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX}; to(RawTABM, Req, Opts) when is_map(RawTABM) -> % Ensure that the TABM is fully loaded if the `bundle` key is set to true. ?event({to, {inbound, RawTABM}, {req, Req}}), - MaybeCommitment = hb_message:commitment( - #{ <<"commitment-device">> => <<"tx@1.0">> }, - RawTABM, - Opts - ), + MaybeCommitment = dev_codec_ans104_to:commitment( + <<"tx@1.0">>, RawTABM, Opts), IsBundle = dev_codec_ans104_to:is_bundle(MaybeCommitment, Req, Opts), MaybeBundle = dev_codec_ans104_to:maybe_load(RawTABM, IsBundle, Opts), ?event({to, {raw_tabm, RawTABM}, {is_bundle, IsBundle}, {maybe_bundle, MaybeBundle}, {req, Req}, {opts, Opts}}), @@ -860,7 +873,7 @@ do_unsigned_tx_roundtrip(UnsignedTX, UnsignedTABM, Req) -> TABM = hb_util:ok(from(DeserializedTX, Req, #{})), ?event(debug_test, {unsigned_tx_roundtrip, {expected_tabm, UnsignedTABM}, {actual_tabm, TABM}}), - ?assertEqual(UnsignedTABM, TABM, unsigned_tx_roundtrip), + ?assert(hb_message:match(UnsignedTABM, TABM), unsigned_tx_roundtrip), % TABM -> TX TX = hb_util:ok(to(TABM, Req, #{})), ExpectedTX = UnsignedTX#tx{ unsigned_id = ar_tx:id(UnsignedTX, unsigned) }, @@ -890,7 +903,7 @@ do_signed_tx_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Req) -> #{ hb_util:human_id(SignedTX#tx.id) => SignedCommitment }}, ?event(debug_test, {signed_tx_roundtrip, {expected_tabm, SignedTABM}, {actual_tabm, TABM}}), - ?assertEqual(SignedTABM, TABM, signed_tx_roundtrip), + ?assert(hb_message:match(SignedTABM, TABM), signed_tx_roundtrip), % TABM -> TX TX = hb_util:ok(to(TABM, Req, #{})), ExpectedTX = SignedTX#tx{ @@ -924,7 +937,7 @@ do_unsigned_tabm_roundtrip(UnsignedTX0, UnsignedTABM, Req) -> TABM = hb_util:ok(from(DeserializedTX, Req, #{})), ?event(debug_test, {unsigned_tabm_roundtrip, {expected_tabm, UnsignedTABM}, {actual_tabm, TABM}}), - ?assertEqual(UnsignedTABM, TABM, unsigned_tabm_roundtrip). + ?assert(hb_message:match(UnsignedTABM, TABM), unsigned_tabm_roundtrip). do_signed_tabm_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Device, Req) -> % Commit TABM @@ -934,7 +947,10 @@ do_signed_tabm_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Device, Req) -> ?event(debug_test, {signed_tabm_roundtrip, {signed_tabm, SignedTABM}}), ?assert(hb_message:verify(SignedTABM), signed_tabm_roundtrip), {ok, _, SignedCommitment} = hb_message:commitment( - #{ <<"commitment-device">> => <<"tx@1.0">> }, + #{ + <<"commitment-device">> => <<"tx@1.0">>, + <<"type">> => <<"rsa-pss-sha256">> + }, SignedTABM, #{} ), @@ -963,7 +979,7 @@ do_signed_tabm_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Device, Req) -> }, SignedTX, signed_tabm_roundtrip), % TX -> TABM FinalTABM = hb_util:ok(from(SignedTX, Req, #{})), - ?assertEqual(SignedTABM, FinalTABM, signed_tabm_roundtrip). + ?assert(hb_message:match(SignedTABM, FinalTABM), signed_tabm_roundtrip). bundle_commitment_test() -> test_bundle_commitment(unbundled, unbundled, unbundled), @@ -989,7 +1005,14 @@ test_bundle_commitment(Commit, Encode, Decode) -> #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => ToBool(Commit) }), ?event(debug_test, {committed, Label, {explicit, Committed}}), ?assert(hb_message:verify(Committed, all, Opts), Label), - {ok, _, CommittedCommitment} = hb_message:commitment(#{}, Committed, Opts), + {ok, _, CommittedCommitment} = + hb_message:commitment( + #{ + <<"type">> => <<"rsa-pss-sha256">> + }, + Committed, + Opts + ), ?assertEqual( [<<"list">>], hb_maps:get(<<"committed">>, CommittedCommitment, Opts), Label), @@ -1011,7 +1034,14 @@ test_bundle_commitment(Commit, Encode, Decode) -> Opts), ?event(debug_test, {decoded, Label, {explicit, Decoded}}), ?assert(hb_message:verify(Decoded, all, Opts), Label), - {ok, _, DecodedCommitment} = hb_message:commitment(#{}, Decoded, Opts), + {ok, _, DecodedCommitment} = + hb_message:commitment( + #{ + <<"type">> => <<"rsa-pss-sha256">> + }, + Decoded, + Opts + ), ?assertEqual( [<<"list">>], hb_maps:get(<<"committed">>, DecodedCommitment, Opts), Label), diff --git a/src/dev_message.erl b/src/dev_message.erl index 1cb9cc917..c654db1e5 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -463,23 +463,23 @@ with_relevant_commitments(Base, Req, Opts) -> %% the default is `all' for commitments -- also implying `all' for committers. commitment_ids_from_request(Base, Req, Opts) -> Commitments = maps:get(<<"commitments">>, Base, #{}), - ReqCommitters = - case maps:get(<<"committers">>, Req, <<"none">>) of - X when is_list(X) -> X; - CommitterDescriptor -> hb_ao:normalize_key(CommitterDescriptor) - end, RawReqCommitments = maps:get(<<"commitment-ids">>, Req, <<"none">>), ReqCommitments = case RawReqCommitments of X2 when is_list(X2) -> X2; CommitmentDescriptor -> hb_ao:normalize_key(CommitmentDescriptor) end, + ReqCommitters = + case maps:get(<<"committers">>, Req, <<"none">>) of + X when is_list(X) -> X; + CommitterDescriptor -> hb_ao:normalize_key(CommitterDescriptor) + end, ?event(debug_commitments, {commitment_ids_from_request, {req_commitments, ReqCommitments}, {req_committers, ReqCommitters}} ), - % Get the commitments to verify. + % Get the commitments to return from explicit commitment IDs. FromCommitmentIDs = case ReqCommitments of <<"none">> -> []; @@ -521,12 +521,24 @@ commitment_ids_from_request(Base, Req, Opts) -> not hb_maps:is_key(<<"committer">>, Comm); _ -> false end + % not maps:is_key( + % <<"committer">>, + % maps:get(CommitmentID, Commitments) + % ) end, maps:keys(Commitments) ); FinalCommitmentIDs -> FinalCommitmentIDs end, - ?event({commitment_ids_from_request, {base, Base}, {req, Req}, {res, Res}}), + ?event(debug_commitments, + {commitment_ids_from_request, + {base, Base}, {req, Req}, {res, Res}, + {req_commitments, ReqCommitments}, + {req_committers, ReqCommitters}, + {from_commitment_ids, FromCommitmentIDs}, + {from_committer_addrs, FromCommitterAddrs} + } + ), Res. %% @doc Ensure that the `commitments` submessage of a base message is fully diff --git a/src/dev_query_arweave.erl b/src/dev_query_arweave.erl index 19ded4210..4201a6723 100644 --- a/src/dev_query_arweave.erl +++ b/src/dev_query_arweave.erl @@ -23,10 +23,15 @@ query(List, <<"edges">>, _Args, _Opts) -> {ok, [{ok, Msg} || Msg <- List]}; query(Msg, <<"node">>, _Args, _Opts) -> {ok, Msg}; -query(Obj, <<"transaction">>, Args, Opts) -> - case query(Obj, <<"transactions">>, Args, Opts) of - {ok, []} -> {ok, null}; - {ok, [Msg|_]} -> {ok, Msg} +query(Obj, <<"transaction">>, #{ <<"id">> := ID }, Opts) -> + case hb_cache:read(ID, Opts) of + {ok, Msg} -> + case hb_message:commitment(ID, Msg, Opts) of + {ok, ID, Comm} -> + {ok, Msg#{ <<"commitments">> => #{ ID => Comm } }}; + not_found -> {ok, null} + end; + not_found -> {ok, null} end; query(Obj, <<"transactions">>, Args, Opts) -> ?event({transactions_query, @@ -46,6 +51,7 @@ query(Obj, <<"transactions">>, Args, Opts) -> end, Matches ), + ?event({transactions_messages, Messages}), {ok, Messages}; query(Obj, <<"block">>, Args, Opts) -> case query(Obj, <<"blocks">>, Args, Opts) of @@ -189,20 +195,14 @@ match_args([], Results, Opts) -> Matches = lists:foldl( fun(Result, Acc) -> - hb_util:list_with(resolve_ids(Result, Opts), Acc) + hb_util:list_with(Result, Acc) end, - resolve_ids(hd(Results), Opts), + hd(Results), tl(Results) ), - hb_util:unique( - lists:flatten( - [ - all_ids(ID, Opts) - || - ID <- Matches - ] - ) - ); + ?event({match_args_results, + {results, Results}, {matches, Matches}}), + Matches; match_args([{Field, X} | Rest], Acc, Opts) -> MatchRes = match(Field, X, Opts), ?event({match, {field, Field}, {arg, X}, {match_res, MatchRes}}), @@ -242,13 +242,13 @@ match(<<"id">>, ID, _Opts) -> match(<<"ids">>, IDs, _Opts) -> {ok, IDs}; match(<<"tags">>, Tags, Opts) -> - hb_cache:match(dev_query_graphql:keys_to_template(Tags), Opts); + {ok, Matches} = + hb_cache:match(dev_query_graphql:keys_to_template(Tags), Opts), + {ok, all_ids(Matches, Opts)}; match(<<"owners">>, Owners, Opts) -> {ok, matching_commitments(<<"committer">>, Owners, Opts)}; match(<<"owner">>, Owner, Opts) -> - Res = matching_commitments(<<"committer">>, Owner, Opts), - ?event({match_owner, Owner, Res}), - {ok, Res}; + {ok, matching_commitments(<<"committer">>, Owner, Opts)}; match(<<"recipients">>, Recipients, Opts) -> {ok, matching_commitments(<<"field-target">>, Recipients, Opts)}; match(UnsupportedFilter, _, _) -> @@ -270,14 +270,18 @@ matching_commitments(Field, Values, Opts) when is_list(Values) -> matching_commitments(Field, Value, Opts) when is_binary(Value) -> case hb_cache:match(#{ Field => Value }, Opts) of {ok, IDs} -> + BaseIDs = + lists:map( + fun(ID) -> commitment_id_to_base_id(ID, Opts) end, IDs), ?event( {found_matching_commitments, {field, Field}, {value, Value}, - {ids, IDs} + {ids, IDs}, + {base_ids, BaseIDs} } ), - lists:map(fun(ID) -> commitment_id_to_base_id(ID, Opts) end, IDs); + BaseIDs; not_found -> not_found end. @@ -294,29 +298,25 @@ commitment_id_to_base_id(ID, Opts) -> end. %% @doc Find all IDs for a message, by any of its other IDs. -all_ids(ID, Opts) -> +all_ids(IDs, Opts) -> + all_ids(IDs, [], Opts). +all_ids([], Results, _Opts) -> + hb_util:unique( + lists:flatten( + Results + ) + ); +all_ids([ID | Rest], Results, Opts) -> Store = hb_opts:get(store, no_store, Opts), - case hb_store:list(Store, << ID/binary, "/commitments">>) of + IDs = case hb_store:list(Store, << ID/binary, "/commitments">>) of {ok, []} -> [ID]; {ok, CommitmentIDs} -> CommitmentIDs; _ -> [] - end. + end, + all_ids(Rest, [IDs | Results], Opts). %% @doc Scope the stores used for block matching. The searched stores can be %% scoped by setting the `query_arweave_scope' option. scope(Opts) -> Scope = hb_opts:get(query_arweave_scope, [local], Opts), - hb_store:scope(Opts, Scope). - -%% @doc Resolve a list of IDs to their store paths, using the stores provided. -resolve_ids(IDs, Opts) -> - Scoped = scope(Opts), - lists:map( - fun(ID) -> - case hb_cache:read(ID, Opts) of - {ok, Msg} -> hb_message:id(Msg, uncommitted, Scoped); - not_found -> ID - end - end, - IDs - ). \ No newline at end of file + hb_store:scope(Opts, Scope). \ No newline at end of file diff --git a/src/dev_query_graphql.erl b/src/dev_query_graphql.erl index 0b8ea5017..74c9a1ef1 100644 --- a/src/dev_query_graphql.erl +++ b/src/dev_query_graphql.erl @@ -219,8 +219,9 @@ message_query(Msg, Field, _Args, Opts) message_query(Msg = #{ <<"independent_hash">> := _ }, <<"id">>, _Args, Opts) -> {ok, hb_maps:get(<<"independent_hash">>, Msg, null, Opts)}; message_query(Msg, <<"id">>, _Args, Opts) -> - ?event({message_query_id, {object, Msg}}), - {ok, hb_message:id(Msg, all, Opts)}; + ID = hb_message:id(Msg, all, Opts), + ?event({message_query_id, {object, Msg}, {id, ID}}), + {ok, ID}; message_query(_Msg, <<"cursor">>, _Args, _Opts) -> {ok, <<"">>}; message_query(_Obj, _Field, _, _) -> diff --git a/src/dev_query_test_vectors.erl b/src/dev_query_test_vectors.erl index 98b4715ca..59f7404f4 100644 --- a/src/dev_query_test_vectors.erl +++ b/src/dev_query_test_vectors.erl @@ -170,6 +170,7 @@ simple_ans104_query_test() -> }, Node = hb_http_server:start_node(Opts), {ok, WrittenMsg} = write_test_message(Opts), + ?event({written_msg, WrittenMsg}), ?assertMatch( {ok, [_]}, hb_cache:match(#{<<"type">> => <<"Message">>}, Opts) @@ -206,27 +207,11 @@ simple_ans104_query_test() -> }, Opts ), - ExpectedID = hb_message:id(WrittenMsg, all, Opts), - ?event({expected_id, ExpectedID}), ?event({simple_ans104_query_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with tags filter transactions_query_tags_test() -> @@ -269,27 +254,12 @@ transactions_query_tags_test() -> #{}, Opts ), - ExpectedID = hb_message:id(WrittenMsg, all, Opts), - ?event({expected_id, ExpectedID}), ?event({transactions_query_tags_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + ExpectedIDs = [ + hb_message:id(WrittenMsg, none, Opts), + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with owners filter transactions_query_owners_test() -> @@ -331,27 +301,11 @@ transactions_query_owners_test() -> }, Opts ), - ExpectedID = hb_message:id(WrittenMsg, all, Opts), - ?event({expected_id, ExpectedID}), ?event({transactions_query_owners_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with recipients filter transactions_query_recipients_test() -> @@ -396,27 +350,11 @@ transactions_query_recipients_test() -> }, Opts ), - ExpectedID = hb_message:id(WrittenMsg, all, Opts), - ?event({expected_id, ExpectedID}), ?event({transactions_query_recipients_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with ids filter transactions_query_ids_test() -> @@ -459,26 +397,11 @@ transactions_query_ids_test() -> }, Opts ), - ?event({expected_id, ExpectedID}), ?event({transactions_query_ids_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). %% @doc Test transactions query with combined filters transactions_query_combined_test() -> @@ -526,30 +449,14 @@ transactions_query_combined_test() -> }, Opts ), - ?event({expected_id, ExpectedID}), ?event({transactions_query_combined_test, Res}), - ?assertMatch( - #{ - <<"data">> := #{ - <<"transactions">> := #{ - <<"edges">> := - [#{ - <<"node">> := - #{ - <<"id">> := ExpectedID, - <<"tags">> := - [#{ <<"name">> := _, <<"value">> := _ }|_] - } - }] - } - } - } when ?IS_ID(ExpectedID), - Res - ). - + ExpectedIDs = [ + hb_message:id(WrittenMsg, signed, Opts) + ], + assert_query_match(ExpectedIDs, Res, Opts). -%% @doc Test single transaction query by ID -transaction_query_by_id_test() -> +%% @doc Test single transaction query by signed ID +transaction_query_by_signed_id_test() -> Opts = #{ priv_wallet => hb:wallet(), @@ -583,6 +490,7 @@ transaction_query_by_id_test() -> }, Opts ), + ?event({written_msg, WrittenMsg}), ?event({expected_id, ExpectedID}), ?event({transaction_query_by_id_test, Res}), ?assertMatch( @@ -719,22 +627,21 @@ transaction_query_with_anchor_test() -> store => [hb_test_utils:test_store(hb_store_lmdb)] }, Node = hb_http_server:start_node(Opts), - {ok, ID} = - hb_cache:write( - hb_message:convert( - ar_bundles:sign_item( - #tx { - anchor = AnchorID = crypto:strong_rand_bytes(32), - data = <<"test-data">> - }, - hb:wallet() - ), - <<"structured@1.0">>, - <<"ans104@1.0">>, - Opts + Msg = + hb_message:convert( + ar_bundles:sign_item( + #tx { + anchor = AnchorID = crypto:strong_rand_bytes(32), + data = <<"test-data">> + }, + hb:wallet() ), + <<"structured@1.0">>, + <<"ans104@1.0">>, Opts ), + {ok, _UnsignedID} =hb_cache:write(Msg, Opts), + SignedID = hb_message:id(Msg, signed, Opts), EncodedAnchor = hb_util:encode(AnchorID), Query = <<""" @@ -753,7 +660,7 @@ transaction_query_with_anchor_test() -> Node, Query, #{ - <<"id">> => ID + <<"id">> => SignedID }, Opts ), @@ -767,4 +674,46 @@ transaction_query_with_anchor_test() -> } }, Res - ). \ No newline at end of file + ). + +assert_query_match(ExpectedIDs, Res, _Opts) -> + ?event({expected_ids, ExpectedIDs}), + % asset shape of response + ?assertMatch( + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := + [#{ + <<"node">> := + #{ + <<"id">> := _, + <<"tags">> := + [#{ <<"name">> := _, <<"value">> := _ }|_] + } + } + | _] + } + } + }, + Res + ), + % assert that the correct IDs are in the response in any order + #{ + <<"data">> := #{ + <<"transactions">> := #{ + <<"edges">> := Edges + } + } + } = Res, + ActualIDs = + [ID + || + #{ + <<"node">> := + #{ + <<"id">> := ID + } + } <- Edges + ], + ?assertEqual(lists:sort(ExpectedIDs), lists:sort(ActualIDs)). \ No newline at end of file diff --git a/src/hb_gateway_client.erl b/src/hb_gateway_client.erl index 7f4edf061..a89e562ee 100644 --- a/src/hb_gateway_client.erl +++ b/src/hb_gateway_client.erl @@ -317,38 +317,14 @@ result_to_message(ExpectedID, Item, Opts) -> true -> % The node trusts the GraphQL API, so we add the explicit % keys as committed fields. - ?event(warning, + ?event( + warning, {gql_verify_failed, adding_trusted_fields, {tags, Tags} } ), - Comms = hb_maps:get(<<"commitments">>, Structured, #{}, Opts), - AttName = hd(hb_maps:keys(Comms, Opts)), - Comm = hb_maps:get(AttName, Comms, not_found, Opts), - Structured#{ - <<"commitments">> => #{ - AttName => - Comm#{ - <<"trusted-keys">> => - hb_ao:normalize_keys( - [ - hb_ao:normalize_key(Name) - || - #{ <<"name">> := Name } <- - hb_maps:values( - hb_ao:normalize_keys( - Tags, - Opts - ), - Opts - ) - ], - Opts - ) - } - } - } + add_trusted_keys_to_commitments(Structured, Tags, Opts) end end, {ok, Embedded}. @@ -367,6 +343,38 @@ decode_or_null(Bin) when is_binary(Bin) -> decode_or_null(_) -> <<>>. +add_trusted_keys_to_commitments(Structured, Tags, Opts) -> + Comms = hb_maps:get(<<"commitments">>, Structured, #{}, Opts), + TrustedKeys = trusted_keys_from_tags(Tags, Opts), + UpdatedComms = + maps:map( + fun(_, Comm) when is_map(Comm) -> + Comm#{ + <<"trusted-keys">> => TrustedKeys + }; + (_, Value) -> + Value + end, + Comms + ), + Structured#{ + <<"commitments">> => UpdatedComms + }. + +trusted_keys_from_tags(Tags, Opts) -> + hb_ao:normalize_keys( + [ + hb_ao:normalize_key(Name) + || + #{<<"name">> := Name} <- + hb_maps:values( + hb_ao:normalize_keys(Tags, Opts), + Opts + ) + ], + Opts + ). + %% @doc Takes a list of messages with `name' and `value' fields, and formats %% them as a GraphQL `tags' argument. subindex_to_tags(Subindex) -> diff --git a/src/hb_message.erl b/src/hb_message.erl index 21ed45ddf..eca0ee559 100644 --- a/src/hb_message.erl +++ b/src/hb_message.erl @@ -220,6 +220,17 @@ normalize_commitments(Msg, Opts, Mode) when is_list(Msg) -> normalize_commitments(Msg, _Opts, _Mode) -> Msg. +%% @doc Normalize commitments on a message depending on `Mode': +%% - `passive`: ensure there is at least one unsigned commitment, +%% preserving any signed ones. If no unsigned commitment is present a new +%% one is added using the node-default commitment device +%% (aka NormCommitment). +%% - `add`: always add a NormCommitment if it's not present. +%% - `verify`: same as `passive`, but replace all existing commitments if +%% the committed keys found in the NormCommitment are different. +%% - `fast`: use a cached `phash2` in `priv.last-phash2` to short‑circuit full +%% verification when possible; if it mismatches, recompute and fall back to +%% `verify`. do_normalize_commitments(Msg, _Opts, _Mode) when ?IS_EMPTY_MESSAGE(Msg) -> Msg; do_normalize_commitments(Msg, Opts, passive) -> @@ -231,28 +242,31 @@ do_normalize_commitments(Msg, Opts, passive) -> end, hb_maps:to_list(Commitments) ), - ?event({do_normalize_commitments, + ?event(normalization, { + {normalizing_commitments, passive}, {unsigned_commitments, UnsignedCommitments}, {maybe_signed_commitment, SignedCommitments} }), case {UnsignedCommitments, SignedCommitments} of {[], _} -> - {ok, #{ <<"commitments">> := NewCommitments }} = - dev_message:commit( - uncommitted(Msg), - #{ - <<"type">> => <<"unsigned">> - }, - Opts - ), + NormCommitments = generate_norm_commitments(Msg, #{}, Opts), MergedCommitments = hb_maps:merge( - NewCommitments, + NormCommitments, hb_maps:from_list(SignedCommitments), Opts ), Msg#{ <<"commitments">> => MergedCommitments }; _ -> Msg end; +do_normalize_commitments(Msg, Opts, add) -> + NormCommitments = generate_norm_commitments(Msg, #{}, Opts), + Msg#{ + <<"commitments">> => + hb_maps:merge( + NormCommitments, + hb_maps:get(<<"commitments">>, Msg, #{}, Opts) + ) + }; do_normalize_commitments(Msg, Opts, verify) -> UnsignedCommitment = commitment(#{ <<"type">> => <<"unsigned">> }, Msg, Opts), {MaybeUnsignedID, MaybeCommittedSpec} = @@ -261,16 +275,13 @@ do_normalize_commitments(Msg, Opts, verify) -> {ID, #{ <<"committed">> => Committed }}; _ -> {undefined, #{}} end, - {ok, #{ <<"commitments">> := NormCommitments }} = - dev_message:commit( - uncommitted(Msg), - MaybeCommittedSpec#{ - <<"type">> => <<"unsigned">> - }, - Opts - ), - ?event(normalization, {normalizing_commitments, verify}), + NormCommitments = generate_norm_commitments(Msg, MaybeCommittedSpec, Opts), [NormID] = hb_maps:keys(NormCommitments, Opts), + ?event(normalization, { + {normalizing_commitments, verify}, + {maybe_unsigned_id, MaybeUnsignedID}, + {norm_id, NormID} + }), case {MaybeUnsignedID, NormID} of {MatchedID, MatchedID} -> Msg; @@ -287,12 +298,7 @@ do_normalize_commitments(Msg, Opts, verify) -> Opts ); {_OldID, _NewID} -> - {ok, #{ <<"commitments">> := NewCommitments }} = - dev_message:commit( - uncommitted(Msg), - #{ <<"type">> => <<"unsigned">> }, - Opts - ), + NewCommitments = generate_norm_commitments(Msg, #{}, Opts), % We had an unsigned ID to begin with and the new one is different. % This means that the committed keys have changed, so we drop any % other commitments and return only the new unsigned one. @@ -316,6 +322,17 @@ do_normalize_commitments(Msg, Opts, fast) when is_map(Msg) -> do_normalize_commitments(MsgWithHash, Opts, verify) end. +%% @doc Generate unsigned commitments using the node-default commitment +%% device. (aka NormCommitments) +generate_norm_commitments(Msg, BaseSpec, Opts) -> + {ok, #{ <<"commitments">> := NormCommitments }} = + dev_message:commit( + uncommitted(Msg), + BaseSpec#{ <<"type">> => <<"unsigned">> }, + Opts + ), + NormCommitments. + %% @doc Annotate a message with its phash2 value in the `priv' sub-map, %% calculating it if necessary. attach_phash2(Msg, Opts) -> @@ -781,12 +798,16 @@ commitment(ID, Link, Opts) when ?IS_LINK(Link) -> commitment(ID, hb_cache:ensure_loaded(Link, Opts), Opts); commitment(ID, #{ <<"commitments">> := Commitments }, Opts) when is_binary(ID), is_map_key(ID, Commitments) -> - hb_maps:get( - ID, - Commitments, - not_found, - Opts - ); + FindRes = + hb_maps:find( + ID, + Commitments, + Opts + ), + case FindRes of + error -> not_found; + {ok, Comm} -> {ok, ID, Comm} + end; commitment(#{ <<"type">> := <<"unsigned">> }, Msg, Opts) -> Commitments = hb_maps:get(<<"commitments">>, Msg, #{}, Opts), UnsignedCommitments = @@ -804,7 +825,8 @@ commitment(#{ <<"type">> := <<"unsigned">> }, Msg, Opts) -> {ok, CommID, hb_util:ok(hb_maps:find(CommID, UnsignedCommitments, Opts))}; true -> ?event(commitment, {multiple_matches, {matches, UnsignedCommitments}}), - multiple_matches end; + multiple_matches + end; commitment(Spec, Msg, Opts) -> Matches = commitments(Spec, Msg, Opts), ?event(debug_commitment, {commitment, {spec, Spec}, {matches, Matches}}), @@ -842,11 +864,13 @@ commitments(_Spec, _Msg, _Opts) -> %% @doc Return the devices for which there are commitments on a message. commitment_devices(#{ <<"commitments">> := Commitments }, Opts) -> - lists:map( - fun(CommMsg) -> - hb_ao:get(<<"commitment-device">>, CommMsg, Opts) - end, - maps:values(Commitments) + hb_util:unique( + lists:map( + fun(CommMsg) -> + hb_ao:get(<<"commitment-device">>, CommMsg, Opts) + end, + maps:values(Commitments) + ) ); commitment_devices(_Msg, _Opts) -> []. diff --git a/src/hb_message_test_vectors.erl b/src/hb_message_test_vectors.erl index bd3a3f28d..194963d69 100644 --- a/src/hb_message_test_vectors.erl +++ b/src/hb_message_test_vectors.erl @@ -9,8 +9,8 @@ %% Disable/enable as needed. run_test() -> hb:init(), - nested_empty_map_test( - <<"structured@1.0">>, + signed_message_encode_decode_verify_test( + <<"ans104@1.0">>, test_opts(normal) ). diff --git a/src/hb_store_gateway.erl b/src/hb_store_gateway.erl index 57f8e3083..de1b3dc07 100644 --- a/src/hb_store_gateway.erl +++ b/src/hb_store_gateway.erl @@ -382,9 +382,19 @@ remote_hyperbeam_node_ans104_test() -> ServerOpts, #{ <<"commitment-device">> => <<"ans104@1.0">> } ), + UnsignedID = hb_message:id(Msg, none, ServerOpts), + SignedID = hb_message:id(Msg, all, ServerOpts), + ?assertNotEqual(UnsignedID, SignedID), + ?event(debug_test, {{unsigned_id, UnsignedID}, {signed_id, SignedID}}), {ok, ID} = hb_cache:write(Msg, ServerOpts), - {ok, ReadMsg} = hb_cache:read(ID, ServerOpts), - ?assert(hb_message:verify(ReadMsg)), + ?assertEqual(UnsignedID, ID), + {ok, UnsignedMsg} = hb_cache:read(UnsignedID, ServerOpts), + {ok, SignedMsg} = hb_cache:read(SignedID, ServerOpts), + ?event(debug_test, { + {written_id, ID}, {written_msg, Msg}, + {unsigned_msg, UnsignedMsg}, {signed_msg, SignedMsg}}), + ?assert(hb_message:verify(UnsignedMsg)), + ?assert(hb_message:verify(SignedMsg)), LocalStore = hb_test_utils:test_store(), ClientOpts = #{ @@ -398,6 +408,9 @@ remote_hyperbeam_node_ans104_test() -> } ] }, - {ok, Req} = hb_cache:read(ID, ClientOpts), + % Unsigned dataitems can not be synced from the remote node via graphql + ?assertEqual(not_found, hb_cache:read(UnsignedID, ClientOpts)), + % But signed dataitems can + {ok, Req} = hb_cache:read(SignedID, ClientOpts), ?assert(hb_message:verify(Req)), ?assert(hb_message:match(Msg, Req)). \ No newline at end of file diff --git a/src/hb_util.erl b/src/hb_util.erl index 5a3502014..998876f2d 100644 --- a/src/hb_util.erl +++ b/src/hb_util.erl @@ -449,7 +449,15 @@ message_to_ordered_list(Message, Opts) -> fun hb_ao:normalize_key/1, lists:sort(lists:map(fun int/1, Keys)) ), - message_to_ordered_list(NormMessage, SortedKeys, erlang:hd(SortedKeys), Opts). + case SortedKeys of + [] -> []; + _ -> + message_to_ordered_list( + NormMessage, + SortedKeys, + erlang:hd(SortedKeys), + Opts) + end. message_to_ordered_list(_Message, [], _Key, _Opts) -> []; message_to_ordered_list(Message, [Key|Keys], Key, Opts) ->