Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 47 additions & 5 deletions src/ar_bundles.erl
Original file line number Diff line number Diff line change
Expand Up @@ -129,15 +129,15 @@ enforce_valid_tx(TX) ->
{invalid_field, anchor, TX#tx.anchor}
),
hb_util:ok_or_throw(TX,
hb_util:check_size(TX#tx.owner, [0, byte_size(?DEFAULT_OWNER)]),
hb_util:check_size(TX#tx.owner, [0, 32, byte_size(?DEFAULT_OWNER)]),
{invalid_field, owner, TX#tx.owner}
),
hb_util:ok_or_throw(TX,
hb_util:check_size(TX#tx.target, [0, 32]),
{invalid_field, target, TX#tx.target}
),
hb_util:ok_or_throw(TX,
hb_util:check_size(TX#tx.signature, [0, 65, byte_size(?DEFAULT_SIG)]),
hb_util:check_size(TX#tx.signature, [0, 64, 65, byte_size(?DEFAULT_SIG)]),
{invalid_field, signature, TX#tx.signature}
),
hb_util:ok_or_throw(TX,
Expand Down Expand Up @@ -184,15 +184,17 @@ data_item_signature_data(RawItem) ->
ar_deep_hash:hash([
utf8_encoded("dataitem"),
utf8_encoded("1"),
%% Only SignatureType 1 is supported for now (RSA 4096)
utf8_encoded("1"),
utf8_encoded(get_signature_type(Item#tx.signature_type)),
<<(Item#tx.owner)/binary>>,
<<(Item#tx.target)/binary>>,
<<(Item#tx.anchor)/binary>>,
encode_tags(Item#tx.tags),
<<(Item#tx.data)/binary>>
]).

get_signature_type({rsa, 65537}) -> "1";
get_signature_type({eddsa, ed25519}) -> "2".

%% @doc Verify the data item's ID matches the signature.
verify_data_item_id(DataItem) ->
ExpectedID = crypto:hash(sha256, DataItem#tx.signature),
Expand Down Expand Up @@ -316,6 +318,8 @@ to_serialized_pair(Item, false, Signed) ->
%% little-endian format which is why we encode to `<<1, 0>>'.
encode_signature_type({rsa, 65537}) ->
<<1, 0>>;
encode_signature_type({eddsa, ed25519}) ->
<<2, 0>>;
encode_signature_type(_) ->
unsupported_tx_format.

Expand Down Expand Up @@ -498,6 +502,8 @@ decode_bundle_header(Count, <<Size:256/little-integer, ID:32/binary, Rest/binary
%% little-endian format which is why we match on `<<1, 0>>'.
decode_signature(<<1, 0, Signature:512/binary, Owner:512/binary, Rest/binary>>) ->
{{rsa, 65537}, Signature, Owner, Rest};
decode_signature(<<2, 0, Signature:64/binary, Owner:32/binary, Rest/binary>>) ->
{{eddsa, ed25519}, Signature, Owner, Rest};
decode_signature(Other) ->
?event({error_decoding_signature,
{sig_type, {explicit, binary:part(Other, 0, 2)}},
Expand Down Expand Up @@ -749,6 +755,32 @@ bundle_map_test() ->
?assertEqual(Item1#tx.data, (maps:get(<<"key1">>, BundleItem#tx.data))#tx.data),
?assert(verify_item(BundleItem)).

eddsa_cases_test() ->
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it might be worth adding a test which deserializes and verifies a real eddsa data item pulled from the weave. You can download the raw dataitem and stick it in the test directory and then read/deserialize/verify in a test.

Earlier in our ans104 support we had only these sort of circular tests (use HB to generate and verify dataitems) and they all passed. But as soon as we started pulling real data from the weave everything broke (e.g. due to the little- vs big- encoding issue, or due to slight discrepancies in how the specs are handled). We still have an open issue due to HB and Arweave implementing ECDSA differently and incompatibly. Adding one test on real data should help avoid many of these issues or future regressions.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unless maybe you already have that in the hb_gateway_client test? I see that it is querying an ID - does it also verify the data?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The first test I created, it used 1rTy7gQuK9lJydlKqCEhtGLp2WWG-GOrVo5JdiCmaxs information to verify the signature, but because the anchor isn't available, I provided it manually. But decided that since I was focus on testing the verify_item, I could just create a mock transaction signed with EDDSA, and it should be the same.

The only reason I removed it was because of the anchor, but I can add it back by providing the anchor and calling verify_item.

The test on the hb_gateway_client only checks other parameters of an ED25519 transaction, but doesn't verify it.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, I think this is what is being done in l2_dataitem_ed25519_test, @JamesPiechota . A sample (or a few) in the test/ directory and an associated eunit test sounds like a good idea in general, though.

@speeddragon : I don't see a hb_message:verify call on that test, I think? We should verify that a few validate correctly and that at least one form of invalid signature (ex: take a valid one, modify a bit in the body) fails.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, I think I focus too much on the transaction verify function and forgot to check the support for the hb_message. I will take a look into it.

The tests on hb_gateway_client don't validate the message, so I didn't add it. There are tests under dev_message that do the validations, I will look into them.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are tests under dev_message that do the validations, I will look into them.

The main codec tests are in the hb_message_test_vectors module. The dev_message ones are only really to verify that the calls to the codecs are hooked up correctly. Could you setup a new codec definition (with type: NEW_SIG_SCHEME?) in the test vectors, then we can just check that all of those tests pass? The suite is quite exhaustive at this point.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've modified the dev_message:verify_test to support both wallets. I don't think modifying hb_message_test_vectors would make sense for this case.

Also added the data item of 1rTy7gQuK9lJydlKqCEhtGLp2WWG-GOrVo5JdiCmaxs which I extracted from the bundle. This has the anchor information, and we can deserialize and verify it.

Key = ar_wallet:new(?EDDSA_KEY_TYPE),
%% Owner and SignatureType defined during signing process.
Item1 = sign_item(#tx{
format = ans104,
target = crypto:strong_rand_bytes(32),
anchor = crypto:strong_rand_bytes(32),
tags = [{<<"tag1">>, <<"value1">>}, {<<"tag2">>, <<"value2">>}],
data = <<"item1_data">>
}, Key),
Bundle = serialize(dev_arweave_common:normalize(Item1)),
BundleItem = deserialize(Bundle),
%% Sign a valid transaction and verify it
?assert(verify_item(BundleItem)),
%% Missing Anchor should fail
?assertNot(verify_item(BundleItem#tx{anchor = <<>>})),
%% Missing Tags should fail
?assertNot(verify_item(BundleItem#tx{tags = []})),
%% Missing Owner should fail
?assertNot(verify_item(BundleItem#tx{owner = crypto:strong_rand_bytes(32)})),
%% Missing Target should fail
?assertNot(verify_item(BundleItem#tx{target = <<>>})),
%% Missing Data should fail
?assertNot(verify_item(BundleItem#tx{data = <<>>})),
ok.

extremely_large_bundle_test() ->
W = ar_wallet:new(),
Data = crypto:strong_rand_bytes(100_000_000),
Expand Down Expand Up @@ -991,4 +1023,14 @@ generate_and_write_map_bundle_test_disabled() ->
?event(debug_test, {deserialized, {explicit, Deserialized}}),
?assert(verify_item(Deserialized)),
ok = file:write_file(
<<"test/arbundles.js/ans104-map-bundle-erlang.bundle">>, Serialized).
<<"test/arbundles.js/ans104-map-bundle-erlang.bundle">>, Serialized).

deserialize_ed25519_transaction_test() ->
% ans104-item-ed25519.bin is dataitem 1rTy7gQuK9lJydlKqCEhtGLp2WWG-GOrVo5JdiCmaxs
{ok, Serialized} = file:read_file(<<"test/arbundles.js/ans104-item-ed25519.bin">>),
Deserialized = deserialize(Serialized),
?assertEqual([{<<"Content-Type">>,<<"image/png">>}], Deserialized#tx.tags),
?assertEqual(<<"ZbExyvGrJKOJTJcHMtKzoOZVCQBkjZ+5">>, Deserialized#tx.anchor),
?assertEqual(<<"ejhYD9Cw9VCsVik6yGLoclo3CLRvAITHTZamLY_6ro4">>,
hb_util:human_id(ar_wallet:to_address(Deserialized#tx.owner, Deserialized#tx.signature_type))),
?assert(verify_item(Deserialized)).
19 changes: 16 additions & 3 deletions src/ar_wallet.erl
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ new() ->
new(KeyType = {KeyAlg, PublicExpnt}) when KeyType =:= {rsa, 65537} ->
{[_, Pub], [_, Pub, Priv|_]} = {[_, Pub], [_, Pub, Priv|_]}
= crypto:generate_key(KeyAlg, {4096, PublicExpnt}),
{{KeyType, Priv, Pub}, {KeyType, Pub}};
new(KeyType = {KeyAlg, Curve}) when KeyType =:= {?EDDSA_SIGN_ALG, ed25519} ->
{Pub, Priv} = crypto:generate_key(KeyAlg, Curve),
{{KeyType, Priv, Pub}, {KeyType, Pub}}.


%% @doc Sign some data with a private key.
sign(Key, Data) ->
sign(Key, Data, sha256).
Expand All @@ -36,6 +38,8 @@ sign({{rsa, PublicExpnt}, Priv, Pub}, Data, DigestType) when PublicExpnt =:= 655
privateExponent = binary:decode_unsigned(Priv)
}
);
sign({KeyType = {KeyAlg, Curve}, Priv, _Pub}, Data, _DigestType) when KeyType =:= {?EDDSA_SIGN_ALG, ed25519} ->
crypto:sign(KeyAlg, none, Data, [Priv, Curve]);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not super familiar with crypto:sign, do you know why we aren't passing DigestType into crypto:sign?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great question. ed25519 digest the message in its raw form. Providing none or sha256 doesn't affect the output.

From Google AI:

EdDSA uses two main digest types: PureEdDSA and HashEdDSA. PureEdDSA signs the message directly without a prior hash, while HashEdDSA first hashes the message using a collision-resistant hash function (like SHA-512).

HashEdDSA can be ed25519ph.

sign({{KeyType, Priv, Pub}, {KeyType, Pub}}, Data, DigestType) ->
sign({KeyType, Priv, Pub}, Data, DigestType).

Expand All @@ -57,7 +61,11 @@ verify({{rsa, PublicExpnt}, Pub}, Data, Sig, DigestType) when PublicExpnt =:= 65
publicExponent = PublicExpnt,
modulus = binary:decode_unsigned(Pub)
}
).
);
verify({{eddsa, Curve}, Pub}, Data, Sig, _DigestType) when
byte_size(Pub) == 32 andalso byte_size(Sig) == 64 andalso Curve =:= ed25519 ->
crypto:verify(eddsa, none, Data, Sig, [Pub, Curve]).


%% @doc Find a public key from a wallet.
to_pubkey(Pubkey) ->
Expand All @@ -81,7 +89,9 @@ to_address({{_, _, PubKey}, {_, PubKey}}, _) ->
to_address(PubKey, {rsa, 65537}) ->
to_rsa_address(PubKey);
to_address(PubKey, {ecdsa, 256}) ->
to_ecdsa_address(PubKey).
to_ecdsa_address(PubKey);
to_address(PubKey, {eddsa, ed25519}) ->
to_eddsa_address(PubKey).

%% @doc Generate a new wallet public and private key, with a corresponding keyfile.
%% The provided key is used as part of the file name.
Expand Down Expand Up @@ -230,6 +240,9 @@ hash_address(PubKey) ->
to_ecdsa_address(PubKey) ->
hb_keccak:key_to_ethereum_address(PubKey).

to_eddsa_address(PubKey) ->
hash_address(PubKey).

%%%===================================================================
%%% Private functions.
%%%===================================================================
Expand Down
8 changes: 6 additions & 2 deletions src/dev_codec_ans104_from.erl
Original file line number Diff line number Diff line change
Expand Up @@ -203,13 +203,17 @@ with_unsigned_commitment(
with_signed_commitment(
Item, Device, FieldCommitments, Tags,
UncommittedMessage, CommittedKeys, Opts) ->
Address = hb_util:human_id(ar_wallet:to_address(Item#tx.owner)),
Address = hb_util:human_id(ar_wallet:to_address(Item#tx.owner, Item#tx.signature_type)),
ID = hb_util:human_id(Item#tx.id),
ExtraCommitments = hb_maps:merge(
FieldCommitments,
hb_maps:with(?BUNDLE_KEYS, Tags),
Opts
),
Type = case Item#tx.signature_type of
?RSA_KEY_TYPE -> <<"rsa-pss-sha256">>;
?EDDSA_KEY_TYPE -> <<"ed25519">>
end,
Commitment =
filter_unset(
hb_maps:merge(
Expand All @@ -221,7 +225,7 @@ with_signed_commitment(
<<"signature">> => hb_util:encode(Item#tx.signature),
<<"keyid">> =>
<<"publickey:", (hb_util:encode(Item#tx.owner))/binary>>,
<<"type">> => <<"rsa-pss-sha256">>,
<<"type">> => Type,
<<"bundle">> => bundle_commitment_key(Tags, Opts),
<<"original-tags">> => original_tags(Item, Opts)
},
Expand Down
1 change: 0 additions & 1 deletion src/dev_codec_tx.erl
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,6 @@ enforce_valid_tx_test() ->
SigInvalidSize66 = crypto:strong_rand_bytes(66),
SigInvalidSize511 = crypto:strong_rand_bytes(511),
SigTooLong513 = crypto:strong_rand_bytes(byte_size(?DEFAULT_SIG)+1),


FailureCases = [
{not_a_tx_record, not_a_tx_record_atom, {invalid_tx, not_a_tx_record_atom}},
Expand Down
11 changes: 9 additions & 2 deletions src/dev_message.erl
Original file line number Diff line number Diff line change
Expand Up @@ -937,9 +937,16 @@ set_ignore_undefined_test() ->
?assertEqual(#{ <<"test-key">> => <<"Value1">> },
hb_private:reset(hb_util:ok(set(Base, Req, #{ hashpath => ignore })))).

verify_test() ->
verify_test_() ->
{foreach, fun () -> ok end, fun (_) -> ok end, [
{"RSA", fun () -> test_verify(?RSA_KEY_TYPE) end},
{"EDSSA", fun () -> test_verify(?EDDSA_KEY_TYPE) end}
]}.

test_verify(KeyType) ->
Unsigned = #{ <<"a">> => <<"b">> },
Signed = hb_message:commit(Unsigned, #{ priv_wallet => hb:wallet() }),
Wallet = ar_wallet:new(KeyType),
Signed = hb_message:commit(Unsigned, #{ priv_wallet => Wallet }),
?event({signed, Signed}),
BadSigned = Signed#{ <<"a">> => <<"c">> },
?event({bad_signed, BadSigned}),
Expand Down
36 changes: 35 additions & 1 deletion src/hb_gateway_client.erl
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ result_to_message(ExpectedID, Item, Opts) ->
),
SignatureType =
case byte_size(Signature) of
64 -> {eddsa, ed25519};
65 -> {ecdsa, 256};
512 -> {rsa, 65537};
_ -> unsupported_tx_signature_type
Expand Down Expand Up @@ -424,11 +425,44 @@ l1_transaction_test() ->
%% @doc Test l2 message from graphql
l2_dataitem_test() ->
_Node = hb_http_server:start_node(#{}),
{ok, Res} = read(<<"oyo3_hCczcU7uYhfByFZ3h0ELfeMMzNacT-KpRoJK6g">>, #{}),
{ok, Res} = read(ID = <<"oyo3_hCczcU7uYhfByFZ3h0ELfeMMzNacT-KpRoJK6g">>, #{}),
?event(gateway, {l2_dataitem, Res}),
Opts = #{},
CommitmentType = hb_util:deep_get(
[<<"commitments">>, ID, <<"type">>],
Res,
not_found,
Opts
),
?assertEqual(<<"rsa-pss-sha256">>, CommitmentType),
Data = maps:get(<<"data">>, Res),
?assertEqual(<<"Hello World">>, Data).

%% @doc ed25519 L2 Transaction test
l2_dataitem_ed25519_test() ->
_Node = hb_http_server:start_node(#{}),
ID = <<"AwrAs-HaBlc8xeI8sw6Wpbi7A0weQWeXYwW20CpX5oM">>,
{ok, Res} = read(ID, #{}),
?event(gateway, {l2_dataitem, Res}),
Opts = #{},
CommitmentType = hb_util:deep_get(
[<<"commitments">>, ID, <<"type">>],
Res,
not_found,
Opts
),
?assertEqual(<<"ed25519">>, CommitmentType),
CommitmentCommitter = hb_util:deep_get(
[<<"commitments">>, ID, <<"committer">>],
Res,
not_found,
Opts
),
?assertEqual(<<"ejhYD9Cw9VCsVik6yGLoclo3CLRvAITHTZamLY_6ro4">>, CommitmentCommitter),
%% Check Data
Data = maps:get(<<"data">>, Res),
?assertEqual(<<"{\"displayName\":\"Test Hub\",\"description\":\"This is a test hub created in the test suite\",\"externalurl\":\"\",\"image\":\"\"}">>, Data).

%% @doc Test optimistic index
ao_dataitem_test() ->
_Node = hb_http_server:start_node(#{}),
Expand Down
3 changes: 2 additions & 1 deletion src/include/ar.hrl
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,11 @@

-define(ECDSA_SIGN_ALG, ecdsa).
-define(ECDSA_TYPE_BYTE, <<2>>).
-define(ECDSA_KEY_TYPE, {?ECDSA_SIGN_ALG, secp256k1}).

-define(EDDSA_SIGN_ALG, eddsa).
-define(EDDSA_TYPE_BYTE, <<3>>).
-define(ECDSA_KEY_TYPE, {?ECDSA_SIGN_ALG, secp256k1}).
-define(EDDSA_KEY_TYPE, {?EDDSA_SIGN_ALG, ed25519}).

%% The default key type used by transactions that do not specify a signature type.
-define(DEFAULT_KEY_TYPE, ?RSA_KEY_TYPE).
Expand Down