+ `generate` (optional): A key that generates a secret based on a + user's request. May return either the secret + directly, or a message with a `secret` key. If + a message is returned, it is assumed to be a + modified version of the user's request and is + used for further processing. + `finalize` (optional): A key that takes the message sequence after this + device has processed it and returns it in a + modified form. ++At present, the `~cookie-secret@1.0` and `~http-auth@1.0` devices implement +the `generator` interface. For example, the following hook definition will +use the `~cookie-secret@1.0` device to generate and manage wallets for +users, with authentication details stored in cookies: +
+ "on": {
+ "request": {
+ "device": "auth-hook@1.0",
+ "secret-provider": {
+ "device": "cookie-secret@1.0"
+ }
+ }
+ }
+
+`~auth-hook@1.0` expects to receive a `secret-provider` key in the hook
+base message. It may optionally also take a `generate-path` and
+`finalize-path`, which are used to generate the secret and post-process the
+response. If either `X-path` keys are not present, the `generate` and
+`finalize` paths are used upon the `secret-provider` message. If the secret
+provider's device does not implement these keys, the operations are skipped.
+Node operators may also specify a `when` message inside their hook definition
+which is used to determine when messages should be signed. The supported keys
+are:
++ `committers`: always | uncommitted | [committer1, or committer2, or ...] + `keys`: always | [key1, or key2, or ...] ++Both keys are optional and can be combined to form 'and' conditions. For +example, the following hook definition will sign all uncommitted requests +that have the `Authorization` header: +
+ "on": {
+ "request": {
+ "device": "auth-hook@1.0",
+ "when": {
+ "keys": ["authorization"],
+ "committers": "uncommitted"
+ }
+ }
+ }
+
+
+---
+
+## Exported Functions
+
+- `request/3`
+
+---
+
+### request
+
+A device offering an on-request hook that signs incoming messages with
+Process an incoming request through a key provider. The key provider
+
+```erlang
+request(Base, HookReq, Opts) ->
+ ?event({auth_hook_request, {base, Base}, {hook_req, HookReq}}),
+ maybe
+ % Get the key provider from options and short-circuit if none is
+ % provided.
+```
+
+### is_relevant
+
+Check if the request is relevant to the hook base. Node operators may
+
+```erlang
+is_relevant(Base, Request, MessageSequence, Opts) ->
+ Committers = is_relevant_from_committers(Base, Request, Opts),
+ Keys =
+ lists:any(
+ fun(Msg) -> is_relevant_from_keys(Base, Msg, Opts) end,
+ [Request | MessageSequence]
+ ),
+ ?event({auth_hook_is_relevant, {committers, Committers}, {keys, Keys}}),
+ if Committers andalso Keys -> true;
+ true -> {skip, {committers, Committers}, {keys, Keys}}
+ end.
+```
+
+### is_relevant_from_committers
+
+Check if the request is relevant to the hook base based on the committers
+
+```erlang
+is_relevant_from_committers(Base, Request, Opts) ->
+ Config =
+ hb_util:deep_get(
+ [<<"when">>, <<"committers">>],
+ Base,
+ <<"uncommitted">>,
+ Opts
+ ),
+ ?event({auth_hook_is_relevant_from_committers, {config, Config}, {base, Base}}),
+ case Config of
+ <<"always">> -> true;
+ <<"uncommitted">> -> hb_message:signers(Request, Opts) == [];
+ RelevantCommitters ->
+ lists:any(
+ fun(Signer) ->
+ lists:member(Signer, RelevantCommitters)
+ end,
+ hb_message:signers(Request, Opts)
+ )
+ end.
+```
+
+### is_relevant_from_keys
+
+Check if the request is relevant to the hook base based on the presence
+
+```erlang
+is_relevant_from_keys(_Base, ID, _Opts) when is_binary(ID) ->
+ false;
+```
+
+### is_relevant_from_keys
+
+Check if the request is relevant to the hook base based on the presence
+
+```erlang
+is_relevant_from_keys(Base, {as, _, Msg}, Opts) ->
+ is_relevant_from_keys(Base, Msg, Opts);
+```
+
+### is_relevant_from_keys
+
+Check if the request is relevant to the hook base based on the presence
+
+```erlang
+is_relevant_from_keys(Base, {resolve, Msg}, Opts) ->
+ is_relevant_from_keys(Base, Msg, Opts);
+```
+
+### is_relevant_from_keys
+
+Check if the request is relevant to the hook base based on the presence
+
+```erlang
+is_relevant_from_keys(Base, Request, Opts) ->
+ Config = hb_util:deep_get([<<"when">>, <<"keys">>], Base, <<"always">>, Opts),
+ ?event(
+ {
+ auth_hook_is_relevant_from_keys,
+ {config, Config},
+ {base, Base},
+ {request, Request}
+ }
+ ),
+ case Config of
+ <<"always">> -> true;
+ RelevantKeys ->
+ lists:any(
+ fun(Key) ->
+ case hb_maps:find(Key, Request, Opts) of
+ {ok, _} -> true;
+ error -> false
+ end
+ end,
+ RelevantKeys
+ )
+ end.
+```
+
+### generate_secret
+
+Normalize authentication credentials, generating new ones if needed.
+
+```erlang
+generate_secret(Provider, Request, Opts) ->
+ case call_provider(<<"generate">>, Provider, Request, Opts) of
+ {error, not_found} ->
+ ?event({no_generate_handler, Provider}),
+ {ok, Provider, strip_sensitive(Request, Opts)};
+ {error, Err} ->
+ % Forward the error. The main handler will fail to match this and
+ % return the error to the user.
+```
+
+### strip_sensitive
+
+Strip the `secret` field from a request.
+Generate a wallet with the key if the `wallet` field is not present in
+
+```erlang
+strip_sensitive(Request, Opts) ->
+ hb_maps:without([<<"secret">>], Request, Opts).
+```
+
+### generate_wallet
+
+Strip the `secret` field from a request.
+Generate a wallet with the key if the `wallet` field is not present in
+
+```erlang
+generate_wallet(Provider, Request, Opts) ->
+ {ok, #{ <<"body">> := WalletID }} =
+ dev_secret:generate(Provider, Request, Opts),
+ ?event({generated_wallet, WalletID}),
+ {ok, Provider, refresh_opts(Opts)}.
+```
+
+### sign_request
+
+Sign a request using the configured key provider
+
+```erlang
+sign_request(Provider, Msg, Opts) ->
+ case hb_maps:get(<<"skip-commit">>, Provider, true, Opts) of
+ false ->
+ % Skip signing and return the normalized message.
+```
+
+### maybe_sign_messages
+
+Process a sequence of messages, signing those marked for signing
+
+```erlang
+maybe_sign_messages(Provider, SignedReq, Opts) ->
+ Parsed = hb_singleton:from(SignedReq, Opts),
+ ?event({auth_hook_parsed_messages, {sequence_length, length(Parsed)}}),
+ SignKey = hb_opts:get(auth_hook_commit_key, ?DEFAULT_COMMIT_KEY, Opts),
+ Processed = maybe_sign_messages(Provider, SignKey, Parsed, Opts),
+ {ok, Processed}.
+```
+
+### maybe_sign_messages
+
+```erlang
+maybe_sign_messages(_Provider, _Key, [], _Opts) -> [];
+```
+
+### maybe_sign_messages
+
+```erlang
+maybe_sign_messages(Provider, Key, [Msg | Rest], Opts) when is_map(Msg) ->
+ case hb_util:atom(hb_maps:get(Key, Msg, false, Opts)) of
+ true ->
+ Uncommitted = hb_message:uncommitted(Msg, Opts),
+ ?event({auth_hook_signing_message, {uncommitted, Msg}}),
+ case sign_request(Provider, Uncommitted, Opts) of
+ {ok, Signed} ->
+ [
+ Signed
+ |
+ maybe_sign_messages(Provider, Key, Rest, Opts)
+ ];
+ {error, Err} ->
+ ?event({auth_hook_sign_error, Err}),
+ [{error, Err}]
+ end;
+ _ ->
+ [Msg | maybe_sign_messages(Provider, Key, Rest, Opts)]
+ end;
+```
+
+### maybe_sign_messages
+
+```erlang
+maybe_sign_messages(Provider, Key, [Msg | Rest], Opts) ->
+ [Msg | maybe_sign_messages(Provider, Key, Rest, Opts)].
+```
+
+### finalize
+
+Finalize the response by adding authentication state
+
+```erlang
+finalize(KeyProvider, SignedReq, MessageSequence, Opts) ->
+ % Add the signed request and message sequence to the response, mirroring the
+ % structure of a normal `~hook@1.0' on-request hook.
+```
+
+### refresh_opts
+
+Refresh the options and log an event if they have changed.
+
+```erlang
+refresh_opts(Opts) ->
+ NewOpts = hb_http_server:get_opts(Opts),
+ case NewOpts of
+ Opts -> ?event(auth_hook_no_opts_change);
+ _ ->
+ ?event(
+ {auth_hook_opts_changed,
+ {size_diff,
+ erlang:external_size(NewOpts) -
+ erlang:external_size(Opts)
+ }
+ }
+ )
+ end,
+ NewOpts.
+```
+
+### find_provider
+
+Get the key provider from the base message or the defaults.
+
+```erlang
+find_provider(Base, Opts) ->
+ case hb_maps:get(<<"secret-provider">>, Base, no_key_provider, Opts) of
+ no_key_provider ->
+ case hb_opts:get(hook_secret_provider, no_key_provider, Opts) of
+ no_key_provider -> {error, no_key_provider};
+ SecretProvider -> SecretProvider
+ end;
+ SecretProvider when is_binary(SecretProvider) ->
+ {ok, #{ <<"device">> => SecretProvider }};
+ SecretProvider when is_map(SecretProvider) ->
+ {ok, SecretProvider};
+ _ ->
+ {error, invalid_auth_provider}
+ end.
+```
+
+### call_provider
+
+Find the appropriate handler for a key in the key provider.
+
+```erlang
+call_provider(Key, Provider, Request, Opts) ->
+ ?event({call_provider, {key, Key}, {provider, Provider}, {req, Request}}),
+ ExecKey = hb_maps:get(<< Key/binary, "-path">>, Provider, Key, Opts),
+ ?event({call_provider, {exec_key, ExecKey}}),
+ case hb_ao:resolve(Provider, Request#{ <<"path">> => ExecKey }, Opts) of
+ {ok, Msg} when is_map(Msg) ->
+ % The result is a message. We revert the path to its original value.
+```
+
+### ignored_keys
+
+Default keys to ignore when signing
+
+```erlang
+ignored_keys(Msg, Opts) ->
+ hb_maps:get(
+ <<"ignored-keys">>,
+ Msg,
+ hb_opts:get(
+ hook_auth_ignored_keys,
+ ?DEFAULT_IGNORED_KEYS,
+ Opts
+ )
+ ).
+```
+
+### cookie_test
+
+```erlang
+cookie_test() ->
+ % Start a node with a secret-provider that uses the cookie device.
+```
+
+### http_auth_test
+
+```erlang
+http_auth_test() ->
+ % Start a node with the `~http-auth@1.0' device as the secret-provider.
+```
+
+### chained_preprocess_test
+
+```erlang
+chained_preprocess_test() ->
+ % Start a node with the `~http-auth@1.0' device as the secret-provider, with
+ % a router chained afterwards in the request hook.
+```
+
+### when_test
+
+```erlang
+when_test() ->
+ % Start a node with the `~http-auth@1.0' device as the secret-provider. Only
+ % request commitment with the hook if the `Authorization' header is present.
+```
+
+### signers_from_commitments_response
+
+The cookie hook test(s) call `GET /commitments`, which returns the
+
+```erlang
+signers_from_commitments_response(Response, ServerWallet) ->
+ ServerAddress = ar_wallet:to_address(ServerWallet),
+ hb_maps:values(hb_maps:filtermap(
+ fun(Key, Value) when ?IS_ID(Key) ->
+ Type = hb_maps:get(<<"type">>, Value, not_found, #{}),
+ Committer = hb_maps:get(<<"committer">>, Value, not_found, #{}),
+ case {Type, Committer} of
+ {<<"rsa-pss-sha512">>, ServerAddress} -> false;
+ {<<"rsa-pss-sha512">>, _} -> {true, Committer};
+ _ -> false
+ end;
+ (_Key, _Value) ->
+ false
+ end,
+ Response,
+ #{}
+```
+
+---
+
+*Generated from [dev_auth_hook.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_auth_hook.erl)*
diff --git a/docs/book/src/dev_cache.erl.md b/docs/book/src/dev_cache.erl.md
new file mode 100644
index 000000000..a3eae0380
--- /dev/null
+++ b/docs/book/src/dev_cache.erl.md
@@ -0,0 +1,310 @@
+# dev_cache
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_cache.erl)
+
+A device that looks up an ID from a local store and returns it,
+honoring the `accept` key to return the correct format. The cache also
+supports writing messages to the store, if the node message has the
+writer's address in its `cache_writers` key.
+
+---
+
+## Exported Functions
+
+- `link/3`
+- `read/3`
+- `write/3`
+
+---
+
+### read
+
+A device that looks up an ID from a local store and returns it,
+Read data from the cache.
+
+```erlang
+read(_M1, M2, Opts) ->
+ Location = hb_ao:get(<<"target">>, M2, Opts),
+ ?event({read, {key_extracted, Location}}),
+ ?event(debug_gateway, cache_read),
+ case hb_cache:read(Location, Opts) of
+ {ok, Res} ->
+ ?event({read, {cache_result, ok, Res}}),
+ case hb_ao:get(<<"accept">>, M2, Opts) of
+ <<"application/aos-2">> ->
+ ?event(dev_cache,
+ {read,
+ {accept_header, <<"application/aos-2">>}
+ }
+ ),
+ JSONMsg = dev_json_iface:message_to_json_struct(Res, Opts),
+ ?event(dev_cache, {read, {json_message, JSONMsg}}),
+ {ok,
+ #{
+ <<"body">> => hb_json:encode(JSONMsg),
+ <<"content-type">> => <<"application/aos-2">>
+ }
+ };
+ _ ->
+ {ok, Res}
+ end;
+ not_found ->
+ % The cache does not have this ID,but it may still be an explicit
+ % `data/' path.
+```
+
+### write
+
+Write data to the cache.
+
+```erlang
+write(_M1, M2, Opts) ->
+ case is_trusted_writer(M2, Opts) of
+ true ->
+ ?event(dev_cache, {write, {trusted_writer, true}}),
+ Type = hb_ao:get(<<"type">>, M2, <<"single">>, Opts),
+ ?event(dev_cache, {write, {write_type, Type}}),
+ case Type of
+ <<"single">> ->
+ ?event(dev_cache, {write, {write_single_called}}),
+ write_single(M2, Opts);
+ <<"batch">> ->
+ ?event(dev_cache, {write, {write_batch_called}}),
+ hb_maps:map(
+ fun(_, Value) ->
+ ?event(dev_cache, {write, {batch_item, Value}}),
+ write_single(Value, Opts)
+ end,
+ hb_ao:get(<<"body">>, M2, Opts),
+ Opts
+ );
+ _ ->
+ ?event(dev_cache, {write, {invalid_write_type, Type}}),
+ {error,
+ #{
+ <<"status">> => 400,
+ <<"body">> => <<"Invalid write type.">>
+ }
+ }
+ end;
+ false ->
+ ?event(dev_cache, {write, {trusted_writer, false}}),
+ {error,
+ #{
+ <<"status">> => 403,
+ <<"body">> => <<"Not authorized to write to the cache.">>
+ }
+ }
+ end.
+```
+
+### link
+
+Link a source to a destination in the cache.
+
+```erlang
+link(_Base, Req, Opts) ->
+ case is_trusted_writer(Req, Opts) of
+ true ->
+ Source = hb_ao:get(<<"source">>, Req, Opts),
+ Destination = hb_ao:get(<<"destination">>, Req, Opts),
+ write_single(#{
+ <<"operation">> => <<"link">>,
+ <<"source">> => Source,
+ <<"destination">> => Destination
+ }, Opts);
+ false ->
+ {error, not_authorized}
+ end.
+```
+
+### write_single
+
+Helper function to write a single data item to the cache.
+
+```erlang
+write_single(Msg, Opts) ->
+ Body = hb_ao:get(<<"body">>, Msg, Opts),
+ ?event(dev_cache, {write_single, {body_extracted, Body}}),
+ Location = hb_ao:get(<<"location">>, Msg, Opts),
+ ?event(dev_cache, {write_single, {location_extracted, Location}}),
+ Operation = hb_ao:get(<<"operation">>, Msg, <<"write">>, Opts),
+ ?event(dev_cache, {write_single, {operation, Operation}}),
+ case {Operation, Body, Location} of
+ {<<"write">>, not_found, _} ->
+ ?event(dev_cache, {write_single, {error, "No body to write"}}),
+ {error,
+ #{
+ <<"status">> => 400,
+ <<"body">> => <<"No body to write.">>
+ }
+ };
+ {<<"write">>, Binary, not_found} when is_binary(Binary) ->
+ % When asked to write only a binary, we do not calculate any
+ % alternative IDs.
+```
+
+### is_trusted_writer
+
+Verify that the request originates from a trusted writer.
+
+```erlang
+is_trusted_writer(Req, Opts) ->
+ Signers = hb_message:signers(Req, Opts),
+ ?event(dev_cache, {is_trusted_writer, {signers, Signers}, {req, Req}}),
+ CacheWriters = hb_opts:get(cache_writers, [], Opts),
+ ?event(dev_cache, {is_trusted_writer, {cache_writers, CacheWriters}}),
+ AnyTrusted = lists:any(fun(Signer) -> lists:member(Signer, CacheWriters) end, Signers),
+ case AnyTrusted of
+ true ->
+ ?event(dev_cache, {is_trusted_writer, {trusted, true}}),
+ true;
+ _ ->
+ ?event(dev_cache, {is_trusted_writer, {trusted, false}}),
+ false
+ end.
+```
+
+### setup_test_env
+
+Create a test environment with a local store and node.
+
+```erlang
+setup_test_env() ->
+ Timestamp = integer_to_binary(os:system_time(millisecond)),
+ StorePrefix = <<"cache-TEST/remote-", Timestamp/binary>>,
+ ?event(dev_cache, {setup_test_env, {start, StorePrefix}}),
+ application:ensure_all_started(hb),
+ ?event(dev_cache, {setup_test_env, {hb_started}}),
+ LocalStore =
+ #{ <<"store-module">> => hb_store_fs, <<"name">> => StorePrefix },
+ ?event(dev_cache, {setup_test_env, {local_store_configured, LocalStore}}),
+ hb_store:reset(LocalStore),
+ ?event(dev_cache, {setup_test_env, {store_reset}}),
+ Wallet = ar_wallet:new(),
+ Address = hb_util:human_id(ar_wallet:to_address(Wallet)),
+ ?event(dev_cache, {setup_test_env, {address, Address}}),
+ Node = hb_http_server:start_node(#{
+ cache_control => [<<"no-cache">>, <<"no-store">>],
+ store => LocalStore,
+ cache_writers => [
+ Address,
+ hb_util:human_id(ar_wallet:to_address(hb:wallet()))
+ ],
+ store_all_signed => false
+ }),
+ ?event(dev_cache, {setup_test_env, {node_started, Node}}),
+ TestOpts = #{
+ cache_control => [<<"no-cache">>, <<"no-store">>],
+ store_all_signed => false,
+ store => [
+ #{
+ <<"store-module">> => hb_store_remote_node,
+ <<"node">> => Node,
+ priv_wallet => Wallet
+ }
+ ]
+ },
+ {ok, TestOpts, [LocalStore, Wallet, Address, Node]}.
+```
+
+### write_to_cache
+
+Write data to the cache via HTTP.
+
+```erlang
+write_to_cache(Node, Data, Wallet) ->
+ ?event(dev_cache, {write_to_cache, {start, Node}}),
+ WriteMsg = #{
+ <<"path">> => <<"/~cache@1.0/write">>,
+ <<"method">> => <<"POST">>,
+ <<"body">> => Data
+ },
+ ?event(dev_cache, {write_to_cache, {message_created, WriteMsg}}),
+ SignedMsg = hb_message:commit(WriteMsg, Wallet),
+ ?event(dev_cache, {write_to_cache, {message_signed}}),
+ WriteResult = hb_http:post(Node, SignedMsg, #{}),
+ ?event(dev_cache, {write_to_cache, {http_post, WriteResult}}),
+ {ok, WriteResponse} = WriteResult,
+ ?event(dev_cache, {write_to_cache, {response_received, WriteResponse}}),
+ Status = hb_ao:get(<<"status">>, WriteResponse, 0, #{}),
+ ?assertEqual(200, Status),
+ Path = hb_ao:get(<<"path">>, WriteResponse, not_found, #{}),
+ ?assertNotEqual(not_found, Path),
+ ?event(dev_cache, {write_to_cache, {write_success, Path}}),
+ {WriteResponse, Path}.
+```
+
+### read_from_cache
+
+Read data from the cache via HTTP.
+
+```erlang
+read_from_cache(Node, Path) ->
+ ?event(dev_cache, {read_from_cache, {start, Node, Path}}),
+ ReadMsg = #{
+ <<"path">> => <<"/~cache@1.0/read">>,
+ <<"method">> => <<"GET">>,
+ <<"target">> => Path
+ },
+ ?event(dev_cache, {read_from_cache, {request_created, ReadMsg}}),
+ ?event({test_read, request, ReadMsg}),
+ ReadResult = hb_http:get(Node, ReadMsg, #{}),
+ ?event(dev_cache, {read_from_cache, {http_get, ReadResult}}),
+ case ReadResult of
+ ReadResponse when is_binary(ReadResponse) ->
+ ?event(dev_cache,
+ {read_from_cache,
+ {response_binary, ReadResponse}
+ }
+ ),
+ ReadResponse;
+ {ok, ReadResponse} ->
+ ?event(dev_cache, {read_from_cache, {response_ok, ReadResponse}}),
+ ReadResponse;
+ {error, Reason} ->
+ ?event(dev_cache, {read_from_cache, {response_error, Reason}}),
+ {error, Reason}
+ end.
+```
+
+### cache_write_message_test
+
+Test that the cache can be written to and read from using the hb_cache
+
+```erlang
+cache_write_message_test() ->
+ ?event(dev_cache, {cache_api_test, {start}}),
+ {ok, Opts, _} = setup_test_env(),
+ TestData = #{
+ <<"test_key">> => <<"test_value">>
+ },
+ ?event(dev_cache, {cache_api_test, {opts, Opts}}),
+ {ok, Path} = hb_cache:write(TestData, Opts),
+ ?event(dev_cache, {cache_api_test, {data_written, Path}}),
+ {ok, ReadData} = hb_cache:read(Path, Opts),
+ ?event(dev_cache, {cache_api_test, {data_read, ReadData}}),
+ ?assert(hb_message:match(TestData, ReadData, only_present, Opts)),
+ ?event(dev_cache, {cache_api_test}),
+ ok.
+```
+
+### cache_write_binary_test
+
+Ensure that we can write direct binaries to the cache.
+
+```erlang
+cache_write_binary_test() ->
+ ?event(dev_cache, {cache_api_test, {start}}),
+ {ok, Opts, _} = setup_test_env(),
+ TestData = <<"test_binary">>,
+ {ok, Path} = hb_cache:write(TestData, Opts),
+ {ok, ReadData} = hb_cache:read(Path, Opts),
+ ?event(dev_cache, {cache_api_test, {data_read, ReadData}}),
+ ?assertEqual(TestData, ReadData),
+ ?event(dev_cache, {cache_api_test}),
+```
+
+---
+
+*Generated from [dev_cache.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_cache.erl)*
diff --git a/docs/book/src/dev_cacheviz.erl.md b/docs/book/src/dev_cacheviz.erl.md
new file mode 100644
index 000000000..331f2a566
--- /dev/null
+++ b/docs/book/src/dev_cacheviz.erl.md
@@ -0,0 +1,115 @@
+# dev_cacheviz
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_cacheviz.erl)
+
+A device that generates renders (or renderable dot output) of a node's
+cache.
+
+---
+
+## Exported Functions
+
+- `dot/3`
+- `index/3`
+- `js/3`
+- `json/3`
+- `svg/3`
+
+---
+
+### dot
+
+A device that generates renders (or renderable dot output) of a node's
+Output the dot representation of the cache, or a specific path within
+Output the SVG representation of the cache, or a specific path within
+
+```erlang
+dot(_, Req, Opts) ->
+ Target = hb_ao:get(<<"target">>, Req, all, Opts),
+ Dot =
+ hb_cache_render:cache_path_to_dot(
+ Target,
+ #{
+ render_data =>
+ hb_util:atom(
+ hb_ao:get(<<"render-data">>, Req, false, Opts)
+ )
+ },
+ Opts
+ ),
+ {ok, #{ <<"content-type">> => <<"text/vnd.graphviz">>, <<"body">> => Dot }}.
+```
+
+### svg
+
+A device that generates renders (or renderable dot output) of a node's
+Output the dot representation of the cache, or a specific path within
+Output the SVG representation of the cache, or a specific path within
+Return a JSON representation of the cache graph, suitable for use with
+
+```erlang
+svg(Base, Req, Opts) ->
+ {ok, #{ <<"body">> := Dot }} = dot(Base, Req, Opts),
+ ?event(cacheviz, {dot, Dot}),
+ Svg = hb_cache_render:dot_to_svg(Dot),
+ {ok, #{ <<"content-type">> => <<"image/svg+xml">>, <<"body">> => Svg }}.
+```
+
+### json
+
+A device that generates renders (or renderable dot output) of a node's
+Output the dot representation of the cache, or a specific path within
+Output the SVG representation of the cache, or a specific path within
+Return a JSON representation of the cache graph, suitable for use with
+
+```erlang
+json(Base, Req, Opts) ->
+ ?event({json, {base, Base}, {req, Req}}),
+ Target =
+ case hb_ao:get(<<"target">>, Req, Opts) of
+ not_found ->
+ case map_size(maps:without([<<"device">>], hb_private:reset(Base))) of
+ 0 ->
+ all;
+ _ ->
+ ?event({writing_base_for_rendering, Base}),
+ {ok, Path} = hb_cache:write(Base, Opts),
+ ?event({wrote_message, Path}),
+ ID = hb_message:id(Base, all, Opts),
+ ?event({generated_id, ID}),
+ ID
+ end;
+ <<".">> -> all;
+ ReqTarget -> ReqTarget
+ end,
+ MaxSize = hb_util:int(hb_ao:get(<<"max-size">>, Req, 250, Opts)),
+ ?event({max_size, MaxSize}),
+ ?event({generating_json_for, {target, Target}}),
+ Res = hb_cache_render:get_graph_data(Target, MaxSize, Opts),
+ ?event({graph_data, Res}),
+ Res.
+```
+
+### index
+
+Return a renderer in HTML form for the JSON format.
+Return a JS library that can be used to render the JSON format.
+
+```erlang
+index(Base, _, _Opts) ->
+ ?event({cacheviz_index, {base, Base}}),
+ dev_hyperbuddy:return_file(<<"cacheviz@1.0">>, <<"graph.html">>).
+```
+
+### js
+
+Return a renderer in HTML form for the JSON format.
+Return a JS library that can be used to render the JSON format.
+
+```erlang
+js(_, _, _Opts) ->
+```
+
+---
+
+*Generated from [dev_cacheviz.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_cacheviz.erl)*
diff --git a/docs/book/src/dev_codec_ans104.erl.md b/docs/book/src/dev_codec_ans104.erl.md
new file mode 100644
index 000000000..2eda359ed
--- /dev/null
+++ b/docs/book/src/dev_codec_ans104.erl.md
@@ -0,0 +1,511 @@
+# dev_codec_ans104
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104.erl)
+
+Codec for managing transformations from `ar_bundles`-style Arweave TX
+records to and from TABMs.
+
+---
+
+## Exported Functions
+
+- `commit/3`
+- `content_type/1`
+- `deserialize/3`
+- `from/3`
+- `serialize/3`
+- `to/3`
+- `verify/3`
+
+---
+
+### content_type
+
+Codec for managing transformations from `ar_bundles`-style Arweave TX
+Return the content type for the codec.
+Serialize a message or TX to a binary.
+
+```erlang
+content_type(_) -> {ok, <<"application/ans104">>}.
+```
+
+### serialize
+
+Codec for managing transformations from `ar_bundles`-style Arweave TX
+Return the content type for the codec.
+Serialize a message or TX to a binary.
+
+```erlang
+serialize(Msg, Req, Opts) when is_map(Msg) ->
+ serialize(to(Msg, Req, Opts), Req, Opts);
+```
+
+### serialize
+
+Codec for managing transformations from `ar_bundles`-style Arweave TX
+Return the content type for the codec.
+Serialize a message or TX to a binary.
+
+```erlang
+serialize(TX, _Req, _Opts) when is_record(TX, tx) ->
+ {ok, ar_bundles:serialize(TX)}.
+```
+
+### deserialize
+
+Deserialize a binary ans104 message to a TABM.
+
+```erlang
+deserialize(#{ <<"body">> := Binary }, Req, Opts) ->
+ deserialize(Binary, Req, Opts);
+```
+
+### deserialize
+
+Deserialize a binary ans104 message to a TABM.
+
+```erlang
+deserialize(Binary, Req, Opts) when is_binary(Binary) ->
+ deserialize(ar_bundles:deserialize(Binary), Req, Opts);
+```
+
+### deserialize
+
+Deserialize a binary ans104 message to a TABM.
+
+```erlang
+deserialize(TX, Req, Opts) when is_record(TX, tx) ->
+ from(TX, Req, Opts).
+```
+
+### commit
+
+Sign a message using the `priv_wallet` key in the options. Supports both
+
+```erlang
+commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) ->
+ commit(Msg, Req#{ <<"type">> => <<"unsigned-sha256">> }, Opts);
+```
+
+### commit
+
+Sign a message using the `priv_wallet` key in the options. Supports both
+
+```erlang
+commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) ->
+ commit(Msg, Req#{ <<"type">> => <<"rsa-pss-sha256">> }, Opts);
+```
+
+### commit
+
+Sign a message using the `priv_wallet` key in the options. Supports both
+
+```erlang
+commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) ->
+ % Convert the given message to an ANS-104 TX record, sign it, and convert
+ % it back to a structured message.
+```
+
+### commit
+
+```erlang
+commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) ->
+ % Remove the commitments from the message, convert it to ANS-104, then back.
+```
+
+### verify
+
+Verify an ANS-104 commitment.
+
+```erlang
+verify(Msg, Req, Opts) ->
+ ?event({verify, {base, Msg}, {req, Req}}),
+ OnlyWithCommitment =
+ hb_private:reset(
+ hb_message:with_commitments(
+ Req,
+ Msg,
+ Opts
+ )
+ ),
+ ?event({verify, {only_with_commitment, OnlyWithCommitment}}),
+ {ok, TX} = to(OnlyWithCommitment, Req, Opts),
+ ?event({verify, {encoded, TX}}),
+ Res = ar_bundles:verify_item(TX),
+ {ok, Res}.
+```
+
+### from
+
+Convert a #tx record into a message map recursively.
+
+```erlang
+from(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, Binary};
+```
+
+### from
+
+Convert a #tx record into a message map recursively.
+
+```erlang
+from(TX, Req, Opts) when is_record(TX, tx) ->
+ case lists:keyfind(<<"ao-type">>, 1, TX#tx.tags) of
+ false ->
+ do_from(TX, Req, Opts);
+ {<<"ao-type">>, <<"binary">>} ->
+ {ok, TX#tx.data}
+ end.
+```
+
+### do_from
+
+```erlang
+do_from(RawTX, Req, Opts) ->
+ % Ensure the TX is fully deserialized.
+```
+
+### to
+
+Internal helper to translate a message to its #tx record representation,
+
+```erlang
+to(Binary, _Req, _Opts) when is_binary(Binary) ->
+ % ar_bundles cannot serialize just a simple binary or get an ID for it, so
+ % we turn it into a TX record with a special tag, tx_to_message will
+ % identify this tag and extract just the binary.
+```
+
+### to
+
+```erlang
+to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX};
+```
+
+### to
+
+```erlang
+to(RawTABM, Req, Opts) when is_map(RawTABM) ->
+ % Ensure that the TABM is fully loaded if the `bundle` key is set to true.
+```
+
+### to
+
+```erlang
+to(Other, _Req, _Opts) ->
+ throw({invalid_tx, Other}).
+```
+
+### normal_tags_test
+
+```erlang
+normal_tags_test() ->
+ Msg = #{
+ <<"first-tag">> => <<"first-value">>,
+ <<"second-tag">> => <<"second-value">>
+ },
+ {ok, Encoded} = to(Msg, #{}, #{}),
+ ?event({encoded, Encoded}),
+ {ok, Decoded} = from(Encoded, #{}, #{}),
+ ?event({decoded, Decoded}),
+ ?assert(hb_message:match(Msg, Decoded)).
+```
+
+### from_maintains_tag_name_case_test
+
+```erlang
+from_maintains_tag_name_case_test() ->
+ TX = #tx {
+ tags = [
+ {<<"Test-Tag">>, <<"test-value">>}
+ ]
+ },
+ SignedTX = ar_bundles:sign_item(TX, hb:wallet()),
+ ?event({signed_tx, SignedTX}),
+ ?assert(ar_bundles:verify_item(SignedTX)),
+ TABM = hb_util:ok(from(SignedTX, #{}, #{})),
+ ?event({tabm, TABM}),
+ ConvertedTX = hb_util:ok(to(TABM, #{}, #{})),
+ ?event({converted_tx, ConvertedTX}),
+ ?assert(ar_bundles:verify_item(ConvertedTX)),
+ ?assertEqual(ConvertedTX, ar_bundles:normalize(SignedTX)).
+```
+
+### restore_tag_name_case_from_cache_test
+
+```erlang
+restore_tag_name_case_from_cache_test() ->
+ Opts = #{ store => hb_test_utils:test_store() },
+ TX = #tx {
+ tags = [
+ {<<"Test-Tag">>, <<"test-value">>},
+ {<<"test-tag-2">>, <<"test-value-2">>}
+ ]
+ },
+ SignedTX = ar_bundles:sign_item(TX, ar_wallet:new()),
+ SignedMsg =
+ hb_message:convert(
+ SignedTX,
+ <<"structured@1.0">>,
+ <<"ans104@1.0">>,
+ Opts
+ ),
+ SignedID = hb_message:id(SignedMsg, all),
+ ?event({signed_msg, SignedMsg}),
+ OnlyCommitted = hb_message:with_only_committed(SignedMsg, Opts),
+ ?event({only_committed, OnlyCommitted}),
+ {ok, ID} = hb_cache:write(SignedMsg, Opts),
+ ?event({id, ID}),
+ {ok, ReadMsg} = hb_cache:read(SignedID, Opts),
+ ?event({restored_msg, ReadMsg}),
+ {ok, ReadTX} = to(ReadMsg, #{}, Opts),
+ ?event({restored_tx, ReadTX}),
+ ?assert(hb_message:match(ReadMsg, SignedMsg)),
+ ?assert(ar_bundles:verify_item(ReadTX)).
+```
+
+### unsigned_duplicated_tag_name_test
+
+```erlang
+unsigned_duplicated_tag_name_test() ->
+ TX = ar_bundles:reset_ids(ar_bundles:normalize(#tx {
+ tags = [
+ {<<"Test-Tag">>, <<"test-value">>},
+ {<<"test-tag">>, <<"test-value-2">>}
+ ]
+ })),
+ Msg = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
+ ?event({msg, Msg}),
+ TX2 = hb_message:convert(Msg, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
+ ?event({tx2, TX2}),
+ ?assertEqual(TX, TX2).
+```
+
+### signed_duplicated_tag_name_test
+
+```erlang
+signed_duplicated_tag_name_test() ->
+ TX = ar_bundles:sign_item(#tx {
+ tags = [
+ {<<"Test-Tag">>, <<"test-value">>},
+ {<<"test-tag">>, <<"test-value-2">>}
+ ]
+ }, ar_wallet:new()),
+ Msg = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
+ ?event({msg, Msg}),
+ TX2 = hb_message:convert(Msg, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
+ ?event({tx2, TX2}),
+ ?assertEqual(TX, TX2),
+ ?assert(ar_bundles:verify_item(TX2)).
+```
+
+### simple_to_conversion_test
+
+```erlang
+simple_to_conversion_test() ->
+ Msg = #{
+ <<"first-tag">> => <<"first-value">>,
+ <<"second-tag">> => <<"second-value">>
+ },
+ {ok, Encoded} = to(Msg, #{}, #{}),
+ ?event({encoded, Encoded}),
+ {ok, Decoded} = from(Encoded, #{}, #{}),
+ ?event({decoded, Decoded}),
+ ?assert(hb_message:match(Msg, hb_message:uncommitted(Decoded, #{}))).
+```
+
+### external_item_with_target_field_test
+
+Ensure that items with an explicitly defined target field lead to:
+
+```erlang
+external_item_with_target_field_test() ->
+ TX =
+ ar_bundles:sign_item(
+ #tx {
+ target = crypto:strong_rand_bytes(32),
+ tags = [
+ {<<"test-tag">>, <<"test-value">>},
+ {<<"test-tag-2">>, <<"test-value-2">>}
+ ],
+ data = <<"test-data">>
+ },
+ ar_wallet:new()
+ ),
+ EncodedTarget = hb_util:encode(TX#tx.target),
+ ?event({tx, TX}),
+ Decoded = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
+ ?event({decoded, Decoded}),
+ ?assertEqual(EncodedTarget, hb_maps:get(<<"target">>, Decoded, undefined, #{})),
+ {ok, OnlyCommitted} = hb_message:with_only_committed(Decoded, #{}),
+ ?event({only_committed, OnlyCommitted}),
+ ?assertEqual(EncodedTarget, hb_maps:get(<<"target">>, OnlyCommitted, undefined, #{})),
+ Encoded = hb_message:convert(OnlyCommitted, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
+ ?assertEqual(TX#tx.target, Encoded#tx.target),
+ ?event({result, {initial, TX}, {result, Encoded}}),
+ ?assertEqual(TX, Encoded).
+```
+
+### generate_item_with_target_tag_test
+
+Ensure that items made inside HyperBEAM use the tags to encode `target`
+
+```erlang
+generate_item_with_target_tag_test() ->
+ Msg =
+ #{
+ <<"target">> => Target = <<"NON-ID-TARGET">>,
+ <<"other-key">> => <<"other-value">>
+ },
+ {ok, TX} = to(Msg, #{}, #{}),
+ ?event({encoded_tx, TX}),
+ % The encoded TX should have ignored the `target' field, setting a tag instead.
+```
+
+### generate_item_with_target_field_test
+
+```erlang
+generate_item_with_target_field_test() ->
+ Msg =
+ hb_message:commit(
+ #{
+ <<"target">> => Target = hb_util:encode(crypto:strong_rand_bytes(32)),
+ <<"other-key">> => <<"other-value">>
+ },
+ #{ priv_wallet => hb:wallet() },
+ <<"ans104@1.0">>
+ ),
+ {ok, TX} = to(Msg, #{}, #{}),
+ ?event({encoded_tx, TX}),
+ ?assertEqual(Target, hb_util:encode(TX#tx.target)),
+ Decoded = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
+ ?event({decoded, Decoded}),
+ ?assertEqual(Target, hb_maps:get(<<"target">>, Decoded, undefined, #{})),
+ {ok, OnlyCommitted} = hb_message:with_only_committed(Decoded, #{}),
+ ?event({only_committed, OnlyCommitted}),
+ ?assertEqual(Target, hb_maps:get(<<"target">>, OnlyCommitted, undefined, #{})),
+ Encoded = hb_message:convert(OnlyCommitted, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
+ ?event({result, {initial, TX}, {result, Encoded}}),
+ ?assertEqual(TX, Encoded).
+```
+
+### type_tag_test
+
+```erlang
+type_tag_test() ->
+ TX =
+ ar_bundles:sign_item(
+ #tx {
+ tags = [{<<"type">>, <<"test-value">>}]
+ },
+ ar_wallet:new()
+ ),
+ ?event({tx, TX}),
+ Structured = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
+ ?event({structured, Structured}),
+ TX2 = hb_message:convert(Structured, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
+ ?event({after_conversion, TX2}),
+ ?assertEqual(TX, TX2).
+```
+
+### ao_data_key_test
+
+```erlang
+ao_data_key_test() ->
+ Msg =
+ hb_message:commit(
+ #{
+ <<"other-key">> => <<"Normal value">>,
+ <<"body">> => <<"Body value">>
+ },
+ #{ priv_wallet => hb:wallet() },
+ <<"ans104@1.0">>
+ ),
+ ?event({msg, Msg}),
+ Enc = hb_message:convert(Msg, <<"ans104@1.0">>, #{}),
+ ?event({enc, Enc}),
+ ?assertEqual(<<"Body value">>, Enc#tx.data),
+ Dec = hb_message:convert(Enc, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
+ ?event({dec, Dec}),
+ ?assert(hb_message:verify(Dec, all, #{})).
+```
+
+### simple_signed_to_httpsig_test
+
+```erlang
+simple_signed_to_httpsig_test() ->
+ Structured =
+ hb_message:commit(
+ #{ <<"test-tag">> => <<"test-value">> },
+ #{ priv_wallet => ar_wallet:new() },
+ #{
+ <<"commitment-device">> => <<"ans104@1.0">>
+ }
+ ),
+ ?event(debug_test, {msg, Structured}),
+ HTTPSig =
+ hb_message:convert(
+ Structured,
+ <<"httpsig@1.0">>,
+ <<"structured@1.0">>,
+ #{}
+ ),
+ ?event(debug_test, {httpsig, HTTPSig}),
+ Structured2 =
+ hb_message:convert(
+ HTTPSig,
+ <<"structured@1.0">>,
+ <<"httpsig@1.0">>,
+ #{}
+ ),
+ ?event(debug_test, {decoded, Structured2}),
+ Match = hb_message:match(Structured, Structured2, #{}),
+ ?assert(Match),
+ ?assert(hb_message:verify(Structured2, all, #{})),
+ HTTPSig2 = hb_message:convert(Structured2, <<"httpsig@1.0">>, <<"structured@1.0">>, #{}),
+ ?event(debug_test, {httpsig2, HTTPSig2}),
+ ?assert(hb_message:verify(HTTPSig2, all, #{})),
+ ?assert(hb_message:match(HTTPSig, HTTPSig2)).
+```
+
+### unsorted_tag_map_test
+
+```erlang
+unsorted_tag_map_test() ->
+ TX =
+ ar_bundles:sign_item(
+ #tx{
+ format = ans104,
+ tags = [
+ {<<"z">>, <<"position-1">>},
+ {<<"a">>, <<"position-2">>}
+ ],
+ data = <<"data">>
+ },
+ ar_wallet:new()
+ ),
+ ?assert(ar_bundles:verify_item(TX)),
+ ?event(debug_test, {tx, TX}),
+ {ok, TABM} = dev_codec_ans104:from(TX, #{}, #{}),
+ ?event(debug_test, {tabm, TABM}),
+ {ok, Decoded} = dev_codec_ans104:to(TABM, #{}, #{}),
+ ?event(debug_test, {decoded, Decoded}),
+ ?assert(ar_bundles:verify_item(Decoded)).
+```
+
+### field_and_tag_ordering_test
+
+```erlang
+field_and_tag_ordering_test() ->
+ UnsignedTABM = #{
+ <<"a">> => <<"value1">>,
+ <<"z">> => <<"value2">>,
+ <<"target">> => <<"NON-ID-TARGET">>
+ },
+ Wallet = hb:wallet(),
+ SignedTABM = hb_message:commit(
+ UnsignedTABM, #{priv_wallet => Wallet}, <<"ans104@1.0">>),
+```
+
+---
+
+*Generated from [dev_codec_ans104.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104.erl)*
diff --git a/docs/book/src/dev_codec_ans104_from.erl.md b/docs/book/src/dev_codec_ans104_from.erl.md
new file mode 100644
index 000000000..d9b42e17e
--- /dev/null
+++ b/docs/book/src/dev_codec_ans104_from.erl.md
@@ -0,0 +1,355 @@
+# dev_codec_ans104_from
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104_from.erl)
+
+Library functions for decoding ANS-104-style data items to TABM form.
+
+---
+
+## Exported Functions
+
+- `base/5`
+- `committed/5`
+- `data/4`
+- `fields/2`
+- `tags/2`
+- `with_commitments/5`
+
+---
+
+### fields
+
+Library functions for decoding ANS-104-style data items to TABM form.
+Return a TABM message containing the fields of the given decoded
+
+```erlang
+fields(Item, _Opts) ->
+ case Item#tx.target of
+ ?DEFAULT_TARGET -> #{};
+ Target ->
+ #{
+ <<"target">> => hb_util:encode(Target)
+ }
+ end.
+```
+
+### tags
+
+Return a TABM of the raw tags of the item, including all metadata
+
+```erlang
+tags(Item, Opts) ->
+ Tags = hb_ao:normalize_keys(
+ deduplicating_from_list(Item#tx.tags, Opts),
+ Opts
+ ),
+ ao_types(Tags, Opts).
+```
+
+### ao_types
+
+Ensure the encoded keys in the `ao-types` field are lowercased and
+
+```erlang
+ao_types(#{ <<"ao-types">> := AoTypes } = Tags, Opts) ->
+ AOTypes = dev_codec_structured:decode_ao_types(AoTypes, Opts),
+ % Normalize all keys in the ao-types map and re-encode
+ NormAOTypes =
+ maps:fold(
+ fun(Key, Val, Acc) ->
+ NormKey = hb_util:to_lower(hb_ao:normalize_key(Key)),
+ Acc#{ NormKey => Val }
+ end,
+ #{},
+ AOTypes
+ ),
+ EncodedAOTypes = dev_codec_structured:encode_ao_types(NormAOTypes, Opts),
+ Tags#{ <<"ao-types">> := EncodedAOTypes };
+```
+
+### ao_types
+
+Ensure the encoded keys in the `ao-types` field are lowercased and
+
+```erlang
+ao_types(Tags, _Opts) ->
+ Tags.
+```
+
+### data
+
+Return a TABM of the keys and values found in the data field of the item.
+
+```erlang
+data(Item, Req, Tags, Opts) ->
+ % If the data field is empty, we return an empty map. If it is a map, we
+ % return it as such. Otherwise, we return a map with the data key set to
+ % the raw data value. This handles unbundling nested messages, as well as
+ % applying the `ao-data-key' tag if given.
+```
+
+### committed
+
+Calculate the list of committed keys for an item, based on its
+
+```erlang
+committed(Item, Fields, Tags, Data, Opts) ->
+ hb_util:unique(
+ data_keys(Data, Opts) ++
+ tag_keys(Item, Opts) ++
+ field_keys(Fields, Tags, Data, Opts)
+ ).
+```
+
+### field_keys
+
+Return the list of the keys from the fields TABM.
+
+```erlang
+field_keys(BaseFields, Tags, Data, Opts) ->
+ HasTarget =
+ hb_maps:is_key(<<"target">>, BaseFields, Opts) orelse
+ hb_maps:is_key(<<"target">>, Tags, Opts) orelse
+ hb_maps:is_key(<<"target">>, Data, Opts),
+ case HasTarget of
+ true -> [<<"target">>];
+ false -> []
+ end.
+```
+
+### data_keys
+
+Return the list of the keys from the data TABM.
+
+```erlang
+data_keys(Data, Opts) ->
+ hb_util:to_sorted_keys(Data, Opts).
+```
+
+### tag_keys
+
+Return the list of the keys from the tags TABM. Filter all metadata
+
+```erlang
+tag_keys(Item, _Opts) ->
+ MetaTags = [
+ <<"bundle-format">>,
+ <<"bundle-version">>,
+ <<"bundle-map">>,
+ <<"ao-data-key">>
+ ],
+ lists:filtermap(
+ fun({Tag, _}) ->
+ case lists:member(Tag, MetaTags) of
+ true -> false;
+ false -> {true, hb_util:to_lower(hb_ao:normalize_key(Tag))}
+ end
+ end,
+ Item#tx.tags
+ ).
+```
+
+### base
+
+Return the complete message for an item, less its commitments. The
+
+```erlang
+base(CommittedKeys, Fields, Tags, Data, Opts) ->
+ hb_maps:from_list(
+ lists:map(
+ fun(Key) ->
+ case hb_maps:find(Key, Data, Opts) of
+ error ->
+ case hb_maps:find(Key, Fields, Opts) of
+ error ->
+ case hb_maps:find(Key, Tags, Opts) of
+ error -> throw({missing_key, Key});
+ {ok, Value} -> {Key, Value}
+ end;
+ {ok, Value} -> {Key, Value}
+ end;
+ {ok, Value} -> {Key, Value}
+ end
+ end,
+ CommittedKeys
+ )
+ ).
+```
+
+### with_commitments
+
+Return a message with the appropriate commitments added to it.
+
+```erlang
+with_commitments(Item, Tags, Base, CommittedKeys, Opts) ->
+ case Item#tx.signature of
+ ?DEFAULT_SIG ->
+ case normal_tags(Item#tx.tags) of
+ true -> Base;
+ false ->
+ with_unsigned_commitment(Item, Tags, Base, CommittedKeys, Opts)
+ end;
+ _ -> with_signed_commitment(Item, Tags, Base, CommittedKeys, Opts)
+ end.
+```
+
+### with_unsigned_commitment
+
+Returns a commitments message for an item, containing an unsigned
+
+```erlang
+with_unsigned_commitment(Item, Tags, UncommittedMessage, CommittedKeys, Opts) ->
+ ID = hb_util:human_id(Item#tx.unsigned_id),
+ UncommittedMessage#{
+ <<"commitments">> => #{
+ ID =>
+ filter_unset(
+ #{
+ <<"commitment-device">> => <<"ans104@1.0">>,
+ <<"committed">> => CommittedKeys,
+ <<"type">> => <<"unsigned-sha256">>,
+ <<"bundle">> => bundle_commitment_key(Tags, Opts),
+ <<"original-tags">> => original_tags(Item, Opts),
+ <<"field-target">> =>
+ case Item#tx.target of
+ ?DEFAULT_TARGET -> unset;
+ Target -> hb_util:encode(Target)
+ end,
+ <<"field-anchor">> =>
+ case Item#tx.anchor of
+ ?DEFAULT_LAST_TX -> unset;
+ LastTX -> LastTX
+ end
+ },
+ Opts
+ )
+ }
+ }.
+```
+
+### with_signed_commitment
+
+Returns a commitments message for an item, containing a signed
+
+```erlang
+with_signed_commitment(Item, Tags, UncommittedMessage, CommittedKeys, Opts) ->
+ Address = hb_util:human_id(ar_wallet:to_address(Item#tx.owner)),
+ ID = hb_util:human_id(Item#tx.id),
+ Commitment =
+ filter_unset(
+ #{
+ <<"commitment-device">> => <<"ans104@1.0">>,
+ <<"committer">> => Address,
+ <<"committed">> => CommittedKeys,
+ <<"signature">> => hb_util:encode(Item#tx.signature),
+ <<"keyid">> =>
+ <<"publickey:", (hb_util:encode(Item#tx.owner))/binary>>,
+ <<"type">> => <<"rsa-pss-sha256">>,
+ <<"bundle">> => bundle_commitment_key(Tags, Opts),
+ <<"original-tags">> => original_tags(Item, Opts),
+ <<"field-anchor">> =>
+ case Item#tx.anchor of
+ ?DEFAULT_LAST_TX -> unset;
+ LastTX -> LastTX
+ end,
+ <<"field-target">> =>
+ case Item#tx.target of
+ ?DEFAULT_TARGET -> unset;
+ Target -> hb_util:encode(Target)
+ end
+ },
+ Opts
+ ),
+ UncommittedMessage#{
+ <<"commitments">> => #{
+ ID => Commitment
+ }
+ }.
+```
+
+### bundle_commitment_key
+
+Return the bundle key for an item.
+Check whether a list of key-value pairs contains only normalized keys.
+
+```erlang
+bundle_commitment_key(Tags, Opts) ->
+ hb_util:bin(hb_maps:is_key(<<"bundle-format">>, Tags, Opts)).
+```
+
+### normal_tags
+
+Return the bundle key for an item.
+Check whether a list of key-value pairs contains only normalized keys.
+
+```erlang
+normal_tags(Tags) ->
+ lists:all(
+ fun({Key, _}) ->
+ hb_util:to_lower(hb_ao:normalize_key(Key)) =:= Key
+ end,
+ Tags
+ ).
+```
+
+### original_tags
+
+Return the original tags of an item if it is applicable. Otherwise,
+
+```erlang
+original_tags(Item, _Opts) ->
+ case normal_tags(Item#tx.tags) of
+ true -> unset;
+ false -> encoded_tags_to_map(Item#tx.tags)
+ end.
+```
+
+### encoded_tags_to_map
+
+Convert an ANS-104 encoded tag list into a HyperBEAM-compatible map.
+
+```erlang
+encoded_tags_to_map(Tags) ->
+ hb_util:list_to_numbered_message(
+ lists:map(
+ fun({Key, Value}) ->
+ #{
+ <<"name">> => Key,
+ <<"value">> => Value
+ }
+ end,
+ Tags
+ )
+ ).
+```
+
+### filter_unset
+
+Remove all undefined values from a map.
+
+```erlang
+filter_unset(Map, Opts) ->
+ hb_maps:filter(
+ fun(_, Value) ->
+ case Value of
+ unset -> false;
+ _ -> true
+ end
+ end,
+ Map,
+ Opts
+ ).
+```
+
+### deduplicating_from_list
+
+Deduplicate a list of key-value pairs by key, generating a list of
+
+```erlang
+deduplicating_from_list(Tags, Opts) ->
+ % Aggregate any duplicated tags into an ordered list of values.
+```
+
+---
+
+*Generated from [dev_codec_ans104_from.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104_from.erl)*
diff --git a/docs/book/src/dev_codec_ans104_to.erl.md b/docs/book/src/dev_codec_ans104_to.erl.md
new file mode 100644
index 000000000..6c045f251
--- /dev/null
+++ b/docs/book/src/dev_codec_ans104_to.erl.md
@@ -0,0 +1,240 @@
+# dev_codec_ans104_to
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104_to.erl)
+
+Library functions for encoding messages to the ANS-104 format.
+
+---
+
+## Exported Functions
+
+- `data/3`
+- `maybe_load/3`
+- `siginfo/2`
+- `tags/4`
+
+---
+
+### maybe_load
+
+Library functions for encoding messages to the ANS-104 format.
+Determine if the message should be loaded from the cache and re-converted
+
+```erlang
+maybe_load(RawTABM, Req, Opts) ->
+ case hb_util:atom(hb_ao:get(<<"bundle">>, Req, false, Opts)) of
+ false -> RawTABM;
+ true ->
+ % Convert back to the fully loaded structured@1.0 message, then
+ % convert to TABM with bundling enabled.
+```
+
+### siginfo
+
+Calculate the fields for a message, returning an initial TX record.
+
+```erlang
+siginfo(Message, Opts) ->
+ MaybeCommitment =
+ hb_message:commitment(
+ #{ <<"commitment-device">> => <<"ans104@1.0">> },
+ Message,
+ Opts
+ ),
+ case MaybeCommitment of
+ {ok, _, Commitment} -> commitment_to_tx(Commitment, Opts);
+ not_found ->
+ case hb_maps:find(<<"target">>, Message, Opts) of
+ {ok, EncodedTarget} ->
+ case hb_util:safe_decode(EncodedTarget) of
+ {ok, Target} when ?IS_ID(Target) ->
+ #tx{ target = Target };
+ _ -> #tx{}
+ end;
+ error -> #tx{}
+ end;
+ multiple_matches ->
+ throw({multiple_ans104_commitments_unsupported, Message})
+ end.
+```
+
+### commitment_to_tx
+
+Convert a commitment to a base TX record. Extracts the owner, signature,
+
+```erlang
+commitment_to_tx(Commitment, Opts) ->
+ Signature =
+ hb_util:decode(
+ maps:get(<<"signature">>, Commitment, hb_util:encode(?DEFAULT_SIG))
+ ),
+ Owner =
+ case hb_maps:find(<<"keyid">>, Commitment, Opts) of
+ {ok, KeyID} ->
+ hb_util:decode(
+ dev_codec_httpsig_keyid:remove_scheme_prefix(KeyID)
+ );
+ error -> ?DEFAULT_OWNER
+ end,
+ Tags =
+ case hb_maps:find(<<"original-tags">>, Commitment, Opts) of
+ {ok, OriginalTags} -> original_tags_to_tags(OriginalTags);
+ error -> []
+ end,
+ LastTX =
+ case hb_maps:find(<<"field-anchor">>, Commitment, Opts) of
+ {ok, EncodedLastTX} -> hb_util:decode(EncodedLastTX);
+ error -> ?DEFAULT_LAST_TX
+ end,
+ Target =
+ case hb_maps:find(<<"field-target">>, Commitment, Opts) of
+ {ok, EncodedTarget} -> hb_util:decode(EncodedTarget);
+ error -> ?DEFAULT_TARGET
+ end,
+ ?event({commitment_owner, Owner}),
+ ?event({commitment_signature, Signature}),
+ ?event({commitment_tags, Tags}),
+ ?event({commitment_last_tx, LastTX}),
+ #tx{
+ owner = Owner,
+ signature = Signature,
+ tags = Tags,
+ anchor = LastTX,
+ target = Target
+ }.
+```
+
+### data
+
+Calculate the data field for a message.
+
+```erlang
+data(TABM, Req, Opts) ->
+ DataKey = inline_key(TABM),
+ % Translate the keys into a binary map. If a key has a value that is a map,
+ % we recursively turn its children into messages.
+```
+
+### data_messages
+
+Calculate the data value for a message. The rules are:
+
+```erlang
+data_messages(TABM, Opts) when is_map(TABM) ->
+ UncommittedTABM =
+ hb_maps:without(
+ [<<"commitments">>, <<"data">>, <<"target">>],
+ hb_private:reset(TABM),
+ Opts
+ ),
+ % If there are too many keys in the TABM, throw an error.
+```
+
+### tags
+
+Calculate the tags field for a data item. If the TX already has tags
+
+```erlang
+tags(#tx{ tags = ExistingTags }, _, _, _) when ExistingTags =/= [] ->
+ ExistingTags;
+```
+
+### tags
+
+Calculate the tags field for a data item. If the TX already has tags
+
+```erlang
+tags(TX, TABM, Data, Opts) ->
+ DataKey = inline_key(TABM),
+ MaybeCommitment =
+ hb_message:commitment(
+ #{ <<"commitment-device">> => <<"ans104@1.0">> },
+ TABM,
+ Opts
+ ),
+ CommittedTagKeys =
+ case MaybeCommitment of
+ {ok, _, Commitment} ->
+ % There is already a commitment, so the tags and order are
+ % pre-determined. However, if the message has been bundled,
+ % any `+link`-suffixed keys in the committed list may need to
+ % be resolved to their base keys (e.g., `output+link` -> `output`).
+```
+
+### include_target_tag
+
+Return whether to include the `target` tag in the tags list.
+
+```erlang
+include_target_tag(TX, TABM, Opts) ->
+ case {TX#tx.target, hb_maps:get(<<"target">>, TABM, undefined, Opts)} of
+ {?DEFAULT_TARGET, _} -> true;
+ {FieldTarget, TagTarget} when FieldTarget =/= TagTarget -> false;
+ _ -> true
+ end.
+```
+
+### committed_tag_keys_to_tags
+
+Apply the `ao-data-key` to the committed keys to generate the list of
+
+```erlang
+committed_tag_keys_to_tags(TX, TABM, DataKey, Committed, Opts) ->
+ DataKeysToExclude =
+ case TX#tx.data of
+ Data when is_map(Data)-> maps:keys(Data);
+ _ -> []
+ end,
+ case DataKey of
+ <<"data">> -> [];
+ _ -> [{<<"ao-data-key">>, DataKey}]
+ end ++
+ lists:map(
+ fun(Key) ->
+ case hb_maps:find(Key, TABM, Opts) of
+ error -> throw({missing_committed_key, Key});
+ {ok, Value} -> {Key, Value}
+ end
+ end,
+ hb_util:list_without(
+ [DataKey | DataKeysToExclude],
+ Committed
+ )
+ ).
+```
+
+### inline_key
+
+Determine if an `ao-data-key` should be added to the message.
+
+```erlang
+inline_key(Msg) ->
+ InlineKey = maps:get(<<"ao-data-key">>, Msg, undefined),
+ case {
+ InlineKey,
+ maps:get(<<"data">>, Msg, ?DEFAULT_DATA) == ?DEFAULT_DATA,
+ maps:is_key(<<"body">>, Msg)
+ andalso not ?IS_LINK(maps:get(<<"body">>, Msg, undefined))
+ } of
+ {Explicit, _, _} when Explicit =/= undefined ->
+ % ao-data-key already exists, so we honor it.
+```
+
+### original_tags_to_tags
+
+Convert a HyperBEAM-compatible map into an ANS-104 encoded tag list,
+
+```erlang
+original_tags_to_tags(TagMap) ->
+ OrderedList = hb_util:message_to_ordered_list(hb_private:reset(TagMap)),
+ ?event({ordered_tagmap, {explicit, OrderedList}, {input, {explicit, TagMap}}}),
+ lists:map(
+ fun(#{ <<"name">> := Key, <<"value">> := Value }) ->
+ {Key, Value}
+ end,
+ OrderedList
+```
+
+---
+
+*Generated from [dev_codec_ans104_to.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104_to.erl)*
diff --git a/docs/book/src/dev_codec_cookie.erl.md b/docs/book/src/dev_codec_cookie.erl.md
new file mode 100644
index 000000000..b377ac2c8
--- /dev/null
+++ b/docs/book/src/dev_codec_cookie.erl.md
@@ -0,0 +1,570 @@
+# dev_codec_cookie
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie.erl)
+
+A utility device that manages setting and encoding/decoding the cookies
+found in requests from a caller. This device implements the `~cookie@1.0`
+codec, inline with the `~message@1.0` schema for conversion.
+Additionally, a `commit` to a message using a secret generated and stored
+in the cookies of the caller, and a `verify` key that validates said
+commitments. In addition, a `generate` key is provided to perform only the
+generation side of the commitment process. The `finalize` key may be
+employed to add a `set` operation to the end of a message sequence, which
+is used in hooks that need to ensure a caller always receives cookies
+generated outside of the normal AO-Core execution flow. In totality, these
+keys implement the `generator` interface type, and may be employed in
+various contexts. For example, `~auth-hook@1.0` may be configured to use
+this device to generate and store secrets in the cookies of the caller,
+which are then used with the `~proxy-wallet@1.0` device to sign requests.
+The `commit` and `verify` keys utilize the `~httpsig@1.0`'s HMAC `secret`
+commitment scheme, which uses a secret key to commit to a message, with the
+`committer` being listed as a hash of the secret.
+This device supports the following paths:
+`/commit`: Sets a `secret` key in the cookies of the caller. The name of
+the cookie is calculated as the hash of the secret.
+`/verify`: Verifies the caller's request by checking the committer in the
+request matches the secret in the cookies of the base message.
+`/store`: Sets the keys in the request message in the cookies of the caller.
+`/extract`: Extracts the cookies from a base message.
+`/reset`: Removes all cookie keys from the base message.
+`/to`: Converts a message containing cookie sources (`cookie`, `set-cookie`,
+or `priv/cookie`) into the format specified in the request message (e.g.
+`set-cookie`, `cookie`).
+`/from`: Converts a message containing encoded cookies into a message
+containing the cookies parsed and normalized.
+
+---
+
+## Exported Functions
+
+- `commit/3`
+- `extract/3`
+- `finalize/3`
+- `from/3`
+- `generate/3`
+- `get_cookie/3`
+- `opts/1`
+- `reset/2`
+- `store/3`
+- `to/3`
+- `verify/3`
+
+---
+
+### opts
+
+A utility device that manages setting and encoding/decoding the cookies
+Get the private store options to use for functions in the cookie device.
+
+```erlang
+opts(Opts) -> hb_private:opts(Opts).
+%%% ~message@1.0 Commitments API keys.
+```
+
+### commit
+
+```erlang
+commit(Base, Req, RawOpts) -> dev_codec_cookie_auth:commit(Base, Req, RawOpts).
+```
+
+### verify
+
+Preprocessor keys that utilize cookies and the `~secret@1.0` device to
+
+```erlang
+verify(Base, Req, RawOpts) -> dev_codec_cookie_auth:verify(Base, Req, RawOpts).
+```
+
+### generate
+
+Preprocessor keys that utilize cookies and the `~secret@1.0` device to
+
+```erlang
+generate(Base, Req, Opts) ->
+ dev_codec_cookie_auth:generate(Base, Req, Opts).
+```
+
+### finalize
+
+Finalize an `on-request` hook by adding the `set-cookie` header to the
+
+```erlang
+finalize(Base, Request, Opts) ->
+ dev_codec_cookie_auth:finalize(Base, Request, Opts).
+```
+
+### get_cookie
+
+Get the cookie with the given key from the base message. The format of
+
+```erlang
+get_cookie(Base, Req, RawOpts) ->
+ Opts = opts(RawOpts),
+ {ok, Cookies} = extract(Base, Req, Opts),
+ Key = hb_maps:get(<<"key">>, Req, undefined, Opts),
+ case hb_maps:get(Key, Cookies, undefined, Opts) of
+ undefined -> {error, not_found};
+ Cookie ->
+ Format = hb_maps:get(<<"format">>, Req, <<"default">>, Opts),
+ case Format of
+ <<"default">> -> {ok, Cookie};
+ <<"set-cookie">> -> {ok, normalize_cookie_value(Cookie)};
+ <<"cookie">> -> {ok, value(Cookie)}
+ end
+ end.
+```
+
+### extract
+
+Return the parsed and normalized cookies from a message.
+
+```erlang
+extract(Msg, Req, Opts) ->
+ {ok, MsgWithCookie} = from(Msg, Req, Opts),
+ Cookies = hb_private:get(<<"cookie">>, MsgWithCookie, #{}, Opts),
+ {ok, Cookies}.
+```
+
+### store
+
+Set the keys in the request message in the cookies of the caller. Removes
+
+```erlang
+store(Base, Req, RawOpts) ->
+ Opts = opts(RawOpts),
+ ?event({store, {base, Base}, {req, Req}}),
+ {ok, ExistingCookies} = extract(Base, Req, Opts),
+ ?event({store, {existing_cookies, ExistingCookies}}),
+ {ok, ResetBase} = reset(Base, Opts),
+ ?event({store, {reset_base, ResetBase}}),
+ MsgToSet =
+ hb_maps:without(
+ [
+ <<"path">>,
+ <<"accept-bundle">>,
+ <<"ao-peer">>,
+ <<"host">>,
+ <<"method">>,
+ <<"body">>
+ ],
+ hb_private:reset(Req),
+ Opts
+ ),
+ ?event({store, {msg_to_set, MsgToSet}}),
+ NewCookies = hb_maps:merge(ExistingCookies, MsgToSet, Opts),
+ NewBase = hb_private:set(ResetBase, <<"cookie">>, NewCookies, Opts),
+ {ok, NewBase}.
+```
+
+### reset
+
+Remove all cookie keys from the given message (including `cookie` and
+
+```erlang
+reset(Base, RawOpts) ->
+ Opts = opts(RawOpts),
+ WithoutBaseCookieKeys =
+ hb_maps:without(
+ [<<"cookie">>, <<"set-cookie">>],
+ Base,
+ Opts
+ ),
+ WithoutPrivCookie =
+ hb_private:set(
+ WithoutBaseCookieKeys,
+ <<"cookie">>,
+ unset,
+ Opts
+ ),
+ {ok, WithoutPrivCookie}.
+```
+
+### to
+
+Convert a message containing cookie sources (`cookie`, `set-cookie`,
+
+```erlang
+to(Msg, Req, Opts) ->
+ ?event({to, {msg, Msg}, {req, Req}}),
+ CookieOpts = opts(Opts),
+ LoadedMsg = hb_cache:ensure_all_loaded(Msg, CookieOpts),
+ ?event({to, {loaded_msg, LoadedMsg}}),
+ do_to(LoadedMsg, Req, CookieOpts).
+```
+
+### do_to
+
+```erlang
+do_to(Msg, Req = #{ <<"format">> := <<"set-cookie">> }, Opts) when is_map(Msg) ->
+ ?event({to_set_cookie, {msg, Msg}, {req, Req}}),
+ {ok, ExtractedParsedCookies} = extract(Msg, Req, Opts),
+ {ok, ResetBase} = reset(Msg, Opts),
+ SetCookieLines =
+ maps:values(
+ maps:map(
+ fun to_set_cookie_line/2,
+ ExtractedParsedCookies
+ )
+ ),
+ MsgWithSetCookie =
+ ResetBase#{
+ <<"set-cookie">> => SetCookieLines
+ },
+ {ok, MsgWithSetCookie};
+```
+
+### do_to
+
+```erlang
+do_to(Msg, Req = #{ <<"format">> := <<"cookie">> }, Opts) when is_map(Msg) ->
+ ?event({to_cookie, {msg, Msg}, {req, Req}}),
+ {ok, ExtractedParsedCookies} = extract(Msg, Req, Opts),
+ {ok, ResetBase} = reset(Msg, Opts),
+ CookieLines =
+ hb_maps:values(
+ hb_maps:map(
+ fun to_cookie_line/2,
+ ExtractedParsedCookies,
+ Opts
+ ),
+ Opts
+ ),
+ ?event({to_cookie, {cookie_lines, CookieLines}}),
+ CookieLine = join(CookieLines, <<"; ">>),
+ {ok, ResetBase#{ <<"cookie">> => CookieLine }};
+```
+
+### do_to
+
+```erlang
+do_to(Msg, _Req, _Opts) when is_map(Msg) ->
+ error({cookie_to_error, {no_format_specified, Msg}});
+```
+
+### do_to
+
+```erlang
+do_to(Msg, _Req, _Opts) ->
+ error({cookie_to_error, {unexpected_message_format, Msg}}).
+```
+
+### to_set_cookie_line
+
+Convert a single cookie into a `set-cookie` header line. The cookie
+
+```erlang
+to_set_cookie_line(Key, RawCookie) ->
+ Cookie = normalize_cookie_value(RawCookie),
+ % Encode the cookie key-value pair as a string to use as the base.
+```
+
+### to_cookie_line
+
+Convert a single cookie into a `cookie` header component. These
+
+```erlang
+to_cookie_line(Key, Cookie) ->
+ to_set_cookie_line(Key, value(Cookie)).
+```
+
+### from
+
+Normalize a message containing a `cookie`, `set-cookie`, and potentially
+
+```erlang
+from(Msg, Req, Opts) ->
+ CookieOpts = opts(Opts),
+ LoadedMsg = hb_cache:ensure_all_loaded(Msg, Opts),
+ do_from(LoadedMsg, Req, CookieOpts).
+```
+
+### do_from
+
+```erlang
+do_from(Msg, Req, Opts) when is_map(Msg) ->
+ {ok, ResetBase} = reset(Msg, Opts),
+ % Get the cookies, parsed, from each available source.
+```
+
+### do_from
+
+```erlang
+do_from(CookiesMsg, _Req, _Opts) ->
+ error({cookie_from_error, {unexpected_message_format, CookiesMsg}}).
+```
+
+### from_cookie
+
+Convert the `cookie` key into a parsed cookie message. `cookie` headers
+
+```erlang
+from_cookie(#{ <<"cookie">> := Cookie }, Req, Opts) ->
+ from_cookie(Cookie, Req, Opts);
+```
+
+### from_cookie
+
+Convert the `cookie` key into a parsed cookie message. `cookie` headers
+
+```erlang
+from_cookie(Cookies, Req, Opts) when is_list(Cookies) ->
+ MergedParsed =
+ lists:foldl(
+ fun(Cookie, Acc) ->
+ {ok, Parsed} = from_cookie(Cookie, Req, Opts),
+ hb_maps:merge(Acc, Parsed, Opts)
+ end,
+ #{},
+ Cookies
+ ),
+ {ok, MergedParsed};
+```
+
+### from_cookie
+
+Convert the `cookie` key into a parsed cookie message. `cookie` headers
+
+```erlang
+from_cookie(Cookie, _Req, _Opts) when is_binary(Cookie) ->
+ BinaryCookiePairs = split(semicolon, Cookie),
+ KeyValList =
+ lists:map(
+ fun(BinaryCookiePair) ->
+ {[Key, Value], _Rest} = split(pair, BinaryCookiePair),
+ {Key, hb_escape:decode(Value)}
+ end,
+ BinaryCookiePairs
+ ),
+ NormalizedMessage = maps:from_list(KeyValList),
+ {ok, NormalizedMessage};
+```
+
+### from_cookie
+
+Convert the `cookie` key into a parsed cookie message. `cookie` headers
+
+```erlang
+from_cookie(_MsgWithoutCookie, _Req, _Opts) ->
+ % The cookie key is not present in the message, so we return an empty map.
+```
+
+### from_set_cookie
+
+Convert a `set-cookie` header line into a cookie message. The `set-cookie`
+
+```erlang
+from_set_cookie(#{ <<"set-cookie">> := Cookie }, Req, Opts) ->
+ ?event({from_set_cookie, {cookie, Cookie}}),
+ from_set_cookie(Cookie, Req, Opts);
+```
+
+### from_set_cookie
+
+Convert a `set-cookie` header line into a cookie message. The `set-cookie`
+
+```erlang
+from_set_cookie(MsgWithoutSet, _Req, _Opts) when is_map(MsgWithoutSet) ->
+ % The set-cookie key is not present in the message, so we return an empty map.
+```
+
+### from_set_cookie
+
+```erlang
+from_set_cookie(Lines, Req, Opts) when is_list(Lines) ->
+ MergedParsed =
+ lists:foldl(
+ fun(Line, Acc) ->
+ {ok, Parsed} = from_set_cookie(Line, Req, Opts),
+ hb_maps:merge(Acc, Parsed)
+ end,
+ #{},
+ Lines
+ ),
+ {ok, MergedParsed};
+```
+
+### from_set_cookie
+
+```erlang
+from_set_cookie(Line, _Req, Opts) when is_binary(Line) ->
+ {[Key, Value], Rest} = split(pair, Line),
+ ValueDecoded = hb_escape:decode(Value),
+ % If there is no remaining binary after the pair, we have a simple key-value
+ % pair, returning just the binary as the value. Otherwise, we split the
+ % remaining binary into attributes and flags and return a message with the
+ % value and those parsed elements.
+```
+
+### to_sorted_list
+
+Takes a message or list of binaries and returns a sorted list of key-
+
+```erlang
+to_sorted_list(Msg) when is_map(Msg) ->
+ lists:keysort(
+ 1,
+ [
+ {trim_bin(hb_util:bin(K)), trim_bin(V)}
+ || {K, V} <- maps:to_list(Msg)
+ ]
+ );
+```
+
+### to_sorted_list
+
+Takes a message or list of binaries and returns a sorted list of key-
+
+```erlang
+to_sorted_list(Binaries) when is_list(Binaries) ->
+ lists:sort(
+ lists:map(
+ fun(Bin) -> trim_bin(hb_util:bin(Bin)) end,
+ Binaries
+ )
+ ).
+```
+
+### value
+
+Take a single parse cookie and return only the value (ignoring attributes
+
+```erlang
+value(Msg) when is_map(Msg) ->
+ maps:get(<<"value">>, Msg, Msg);
+```
+
+### value
+
+Take a single parse cookie and return only the value (ignoring attributes
+
+```erlang
+value(Bin) when is_binary(Bin) ->
+ Bin.
+```
+
+### normalize_cookie_value
+
+Normalize a cookie value to a map with the following keys:
+
+```erlang
+normalize_cookie_value(Msg) when is_map(Msg) ->
+ Msg#{
+ <<"value">> => maps:get(<<"value">>, Msg, Msg),
+ <<"attributes">> => maps:get(<<"attributes">>, Msg, #{}),
+ <<"flags">> => maps:get(<<"flags">>, Msg, [])
+ };
+```
+
+### normalize_cookie_value
+
+Normalize a cookie value to a map with the following keys:
+
+```erlang
+normalize_cookie_value(Bin) when is_binary(Bin) ->
+ #{
+ <<"value">> => Bin,
+ <<"attributes">> => #{},
+ <<"flags">> => []
+ }.
+```
+
+### trim_bin
+
+Trim a binary of leading and trailing whitespace.
+
+```erlang
+trim_bin(Bin) when is_binary(Bin) ->
+ list_to_binary(string:trim(binary_to_list(Bin))).
+```
+
+### join
+
+Join a list of binaries into a `separator`-separated string. Abstracts
+
+```erlang
+join(Binaries, Separator) ->
+ hb_util:bin(
+ string:join(
+ lists:map(fun hb_util:list/1, Binaries),
+ hb_util:list(Separator)
+ )
+ ).
+```
+
+### split
+
+Split a binary by a separator type (`pair`, `lines`, or `attributes`).
+
+```erlang
+split(pair, Bin) ->
+ [Key, ValueRest] = binary:split(Bin, <<"=">>),
+ {_, Value, Rest} = hb_util:split_depth_string_aware_single($;, ValueRest),
+ {[Key, unquote(Value)], trim_leading(Rest)};
+```
+
+### split
+
+Split a binary by a separator type (`pair`, `lines`, or `attributes`).
+
+```erlang
+split(lines, Bin) ->
+ lists:map(fun trim_leading/1, hb_util:split_depth_string_aware($,, Bin));
+```
+
+### split
+
+Split a binary by a separator type (`pair`, `lines`, or `attributes`).
+
+```erlang
+split(semicolon, Bin) ->
+ lists:map(fun trim_leading/1, hb_util:split_depth_string_aware($;, Bin)).
+```
+
+### trim_leading
+
+Remove leading whitespace from a binary, if present.
+
+```erlang
+trim_leading(Line) when not is_binary(Line) ->
+ trim_leading(hb_util:bin(Line));
+```
+
+### trim_leading
+
+Remove leading whitespace from a binary, if present.
+
+```erlang
+trim_leading(<<>>) -> <<>>;
+```
+
+### trim_leading
+
+Remove leading whitespace from a binary, if present.
+
+```erlang
+trim_leading(<<" ", Rest/binary>>) -> trim_leading(Rest);
+```
+
+### trim_leading
+
+Remove leading whitespace from a binary, if present.
+Unquote a binary if it is quoted. If it is not quoted, we return the
+
+```erlang
+trim_leading(Line) -> Line.
+```
+
+### unquote
+
+Remove leading whitespace from a binary, if present.
+Unquote a binary if it is quoted. If it is not quoted, we return the
+
+```erlang
+unquote(<< $\", Rest/binary>>) ->
+ {Unquoted, _} = hb_util:split_escaped_single($\", Rest),
+ Unquoted;
+```
+
+---
+
+*Generated from [dev_codec_cookie.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie.erl)*
diff --git a/docs/book/src/dev_codec_cookie_auth.erl.md b/docs/book/src/dev_codec_cookie_auth.erl.md
new file mode 100644
index 000000000..6f50f3641
--- /dev/null
+++ b/docs/book/src/dev_codec_cookie_auth.erl.md
@@ -0,0 +1,329 @@
+# dev_codec_cookie_auth
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie_auth.erl)
+
+Implements the `message@1.0` commitment interface for the `~cookie@1.0`,
+as well as the `generator` interface type for the `~auth-hook@1.0` device.
+See the [cookie codec](dev_codec_cookie.html) documentation for more details.
+
+---
+
+## Exported Functions
+
+- `commit/3`
+- `finalize/3`
+- `generate/3`
+- `verify/3`
+
+---
+
+### generate
+
+Implements the `message@1.0` commitment interface for the `~cookie@1.0`,
+Generate a new secret (if no `committer` specified), and use it as the
+
+```erlang
+generate(Base, Request, Opts) ->
+ {WithCookie, Secrets} =
+ case find_secrets(Request, Opts) of
+ [] ->
+ {ok, GeneratedSecret} = generate_secret(Base, Request, Opts),
+ {ok, Updated} = store_secret(GeneratedSecret, Request, Opts),
+ {Updated, [GeneratedSecret]};
+ FoundSecrets ->
+ {Request, FoundSecrets}
+ end,
+ ?event({normalized_cookies_found, {secrets, Secrets}}),
+ {
+ ok,
+ WithCookie#{
+ <<"secret">> => Secrets
+ }
+ }.
+```
+
+### finalize
+
+Finalize an `on-request` hook by adding the cookie to the chain of
+
+```erlang
+finalize(Base, Request, Opts) ->
+ ?event(debug_auth, {finalize, {base, Base}, {request, Request}}),
+ maybe
+ {ok, SignedMsg} ?= hb_maps:find(<<"request">>, Request, Opts),
+ {ok, MessageSequence} ?= hb_maps:find(<<"body">>, Request, Opts),
+ % Cookie auth adds set-cookie to response
+ {ok, #{ <<"set-cookie">> := SetCookie }} =
+ dev_codec_cookie:to(
+ SignedMsg,
+ #{ <<"format">> => <<"set-cookie">> },
+ Opts
+ ),
+ {
+ ok,
+ MessageSequence ++
+ [#{ <<"path">> => <<"set">>, <<"set-cookie">> => SetCookie }]
+ }
+ else error ->
+ {error, no_request}
+ end.
+```
+
+### commit
+
+Generate a new secret (if no `committer` specified), and use it as the
+
+```erlang
+commit(Base, Request, RawOpts) when ?IS_LINK(Request) ->
+ Opts = dev_codec_cookie:opts(RawOpts),
+ commit(Base, hb_cache:ensure_loaded(Request, Opts), Opts);
+```
+
+### commit
+
+Generate a new secret (if no `committer` specified), and use it as the
+
+```erlang
+commit(Base, Req = #{ <<"secret">> := Secret }, RawOpts) ->
+ Opts = dev_codec_cookie:opts(RawOpts),
+ commit(hb_cache:ensure_loaded(Secret, Opts), Base, Req, Opts);
+```
+
+### commit
+
+Generate a new secret (if no `committer` specified), and use it as the
+
+```erlang
+commit(Base, Request, RawOpts) ->
+ Opts = dev_codec_cookie:opts(RawOpts),
+ % Calculate the key to use for the commitment.
+```
+
+### commit
+
+Given the secret key, commit the message and set the cookie. This
+
+```erlang
+commit(Secret, Base, Request, Opts) ->
+ {ok, CommittedMsg} =
+ dev_codec_httpsig_proxy:commit(
+ <<"cookie@1.0">>,
+ Secret,
+ Base,
+ Request,
+ Opts
+ ),
+ store_secret(Secret, CommittedMsg, Opts).
+```
+
+### store_secret
+
+Update the nonces for a given secret.
+
+```erlang
+store_secret(Secret, Msg, Opts) ->
+ CookieAddr = dev_codec_httpsig_keyid:secret_key_to_committer(Secret),
+ % Create the cookie parameters, using the name as the key and the secret as
+ % the value.
+```
+
+### verify
+
+Verify the HMAC commitment with the key being the secret from the
+
+```erlang
+verify(Base, ReqLink, RawOpts) when ?IS_LINK(ReqLink) ->
+ Opts = dev_codec_cookie:opts(RawOpts),
+ verify(Base, hb_cache:ensure_loaded(ReqLink, Opts), Opts);
+```
+
+### verify
+
+Verify the HMAC commitment with the key being the secret from the
+
+```erlang
+verify(Base, Req = #{ <<"secret">> := Secret }, RawOpts) ->
+ Opts = dev_codec_cookie:opts(RawOpts),
+ ?event({verify_with_explicit_key, {base, Base}, {request, Req}}),
+ dev_codec_httpsig_proxy:verify(
+ hb_util:decode(Secret),
+ Base,
+ Req,
+ Opts
+ );
+```
+
+### verify
+
+Verify the HMAC commitment with the key being the secret from the
+
+```erlang
+verify(Base, Request, RawOpts) ->
+ Opts = dev_codec_cookie:opts(RawOpts),
+ ?event({verify_finding_key, {base, Base}, {request, Request}}),
+ case find_secret(Request, Opts) of
+ {ok, Secret} ->
+ dev_codec_httpsig_proxy:verify(
+ hb_util:decode(Secret),
+ Base,
+ Request,
+ Opts
+ );
+ {error, Err} ->
+ {error, Err}
+ end.
+```
+
+### generate_secret
+
+Generate a new secret key for the given request. The user may specify
+
+```erlang
+generate_secret(_Base, Request, Opts) ->
+ case hb_maps:get(<<"generator">>, Request, undefined, Opts) of
+ undefined ->
+ % If no generator is specified, use the default generator.
+```
+
+### default_generator
+
+Generate a new secret key using the default generator.
+
+```erlang
+default_generator(_Opts) ->
+ {ok, hb_util:encode(crypto:strong_rand_bytes(64))}.
+```
+
+### execute_generator
+
+Execute a generator function. See `generate_secret/3` for more details.
+
+```erlang
+execute_generator(GeneratorPath, Opts) when is_binary(GeneratorPath) ->
+ hb_ao:resolve(GeneratorPath, Opts);
+```
+
+### execute_generator
+
+Execute a generator function. See `generate_secret/3` for more details.
+Find all secrets in the cookie of a message.
+
+```erlang
+execute_generator(Generator, Opts) ->
+ Path = hb_maps:get(<<"path">>, Generator, <<"generate">>, Opts),
+ hb_ao:resolve(Generator#{ <<"path">> => Path }, Opts).
+```
+
+### find_secrets
+
+Execute a generator function. See `generate_secret/3` for more details.
+Find all secrets in the cookie of a message.
+
+```erlang
+find_secrets(Request, Opts) ->
+ maybe
+ {ok, Cookie} ?= dev_codec_cookie:extract(Request, #{}, Opts),
+ [
+ hb_maps:get(SecretRef, Cookie, secret_unavailable, Opts)
+ ||
+ SecretRef = <<"secret-", _/binary>> <- hb_maps:keys(Cookie)
+ ]
+ else error -> []
+ end.
+```
+
+### find_secret
+
+Find the secret key for the given committer, if it exists in the cookie.
+
+```erlang
+find_secret(Request, Opts) ->
+ maybe
+ {ok, Committer} ?= hb_maps:find(<<"committer">>, Request, Opts),
+ find_secret(Committer, Request, Opts)
+ else error -> {error, no_secret}
+ end.
+```
+
+### find_secret
+
+```erlang
+find_secret(Committer, Request, Opts) ->
+ maybe
+ {ok, Cookie} ?= dev_codec_cookie:extract(Request, #{}, Opts),
+ {ok, _Secret} ?= hb_maps:find(<<"secret-", Committer/binary>>, Cookie, Opts)
+ else error -> {error, not_found}
+ end.
+```
+
+### directly_invoke_commit_verify_test
+
+Call the cookie codec's `commit` and `verify` functions directly.
+
+```erlang
+directly_invoke_commit_verify_test() ->
+ Base = #{ <<"test-key">> => <<"test-value">> },
+ CommittedMsg =
+ hb_message:commit(
+ Base,
+ #{},
+ #{
+ <<"commitment-device">> => <<"cookie@1.0">>
+ }
+ ),
+ ?event({committed_msg, CommittedMsg}),
+ ?assertEqual(1, length(hb_message:signers(CommittedMsg, #{}))),
+ VerifyReq =
+ apply_cookie(
+ CommittedMsg#{
+ <<"committers">> => hb_message:signers(CommittedMsg, #{})
+ },
+ CommittedMsg,
+ #{}
+ ),
+ VerifyReqWithoutComms = hb_maps:without([<<"commitments">>], VerifyReq, #{}),
+ ?event({verify_req_without_comms, VerifyReqWithoutComms}),
+ ?assert(hb_message:verify(CommittedMsg, VerifyReqWithoutComms, #{})),
+ ok.
+```
+
+### http_set_get_cookies_test
+
+Set keys in a cookie and verify that they can be parsed into a message.
+
+```erlang
+http_set_get_cookies_test() ->
+ Node = hb_http_server:start_node(#{}),
+ {ok, SetRes} =
+ hb_http:get(
+ Node,
+ <<"/~cookie@1.0/store?k1=v1&k2=v2">>,
+ #{}
+ ),
+ ?event(debug_cookie, {set_cookie_test, {set_res, SetRes}}),
+ ?assertMatch(#{ <<"set-cookie">> := _ }, SetRes),
+ Req = apply_cookie(#{ <<"path">> => <<"/~cookie@1.0/extract">> }, SetRes, #{}),
+ {ok, Res} = hb_http:get(Node, Req, #{}),
+ ?assertMatch(#{ <<"k1">> := <<"v1">>, <<"k2">> := <<"v2">> }, Res),
+ ok.
+```
+
+### apply_cookie
+
+Takes the cookies from the `GenerateResponse` and applies them to the
+
+```erlang
+apply_cookie(NextReq, GenerateResponse, Opts) ->
+ {ok, Cookie} = dev_codec_cookie:extract(GenerateResponse, #{}, Opts),
+ {ok, NextWithParsedCookie} = dev_codec_cookie:store(NextReq, Cookie, Opts),
+ {ok, NextWithCookie} =
+ dev_codec_cookie:to(
+ NextWithParsedCookie,
+ #{ <<"format">> => <<"cookie">> },
+ Opts
+ ),
+```
+
+---
+
+*Generated from [dev_codec_cookie_auth.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie_auth.erl)*
diff --git a/docs/book/src/dev_codec_cookie_test_vectors.erl.md b/docs/book/src/dev_codec_cookie_test_vectors.erl.md
new file mode 100644
index 000000000..54cf6b062
--- /dev/null
+++ b/docs/book/src/dev_codec_cookie_test_vectors.erl.md
@@ -0,0 +1,904 @@
+# dev_codec_cookie_test_vectors
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie_test_vectors.erl)
+
+A battery of cookie parsing and encoding test vectors.
+
+---
+
+### assert_set
+
+A battery of cookie parsing and encoding test vectors.
+Assert that when given the inputs in the test set, the outputs are
+
+```erlang
+assert_set(TestSet, Fun) ->
+ {Inputs, Expected} = maps:get(TestSet, test_data()),
+ ?event(match_cookie, {starting_group_match, {inputs, {explicit, Inputs}}}),
+ lists:foreach(
+ fun(Input) ->
+ Res = Fun(Input),
+ ?event(
+ match_cookie,
+ {matching,
+ {expected, {explicit, Expected}, {output, {explicit, Res}}}
+ }
+ ),
+ ?assertEqual(Expected, Res)
+ end,
+ Inputs
+ ).
+```
+
+### to_string
+
+Convert a cookie message to a string.
+Convert a string to a cookie message.
+
+```erlang
+to_string(CookieMsg) ->
+ {ok, BaseMsg} = dev_codec_cookie:store(#{}, CookieMsg, #{}),
+ {ok, Msg} =
+ dev_codec_cookie:to(
+ BaseMsg,
+ #{ <<"format">> => <<"set-cookie">> },
+ #{}
+ ),
+ hb_maps:get(<<"set-cookie">>, Msg, [], #{}).
+```
+
+### from_string
+
+Convert a cookie message to a string.
+Convert a string to a cookie message.
+
+```erlang
+from_string(String) ->
+ {ok, BaseMsg} =
+ dev_codec_cookie:from(
+ #{ <<"set-cookie">> => String },
+ #{},
+ #{}
+ ),
+ {ok, Cookie} = dev_codec_cookie:extract(BaseMsg, #{}, #{}),
+ Cookie.
+```
+
+### test_data
+
+returns a map of tuples of the form `testset_name => {[before], after}`.
+
+```erlang
+test_data() ->
+ #{
+ from_string_raw_value =>
+ {
+ [<<"k1=v1">>, <<"k1=\"v1\"">>],
+ #{ <<"k1">> => <<"v1">> }
+ },
+ from_string_attributes =>
+ {
+ [<<"k1=v1; k2=v2">>, <<"k1=\"v1\"; k2=\"v2\"">>],
+ #{
+ <<"k1">> =>
+ #{
+ <<"value">> => <<"v1">>,
+ <<"attributes">> => #{ <<"k2">> => <<"v2">> }
+ }
+ }
+ },
+ from_string_flags =>
+ {
+ [<<"k1=v1; k2=v2; f1; f2">>, <<"k1=\"v1\"; k2=\"v2\"; f1; f2">>],
+ #{
+ <<"k1">> =>
+ #{
+ <<"value">> => <<"v1">>,
+ <<"attributes">> => #{ <<"k2">> => <<"v2">> },
+ <<"flags">> => [<<"f1">>, <<"f2">>]
+ }
+ }
+ },
+ to_string_raw_value =>
+ {
+ [
+ #{ <<"k1">> => <<"v1">> },
+ #{ <<"k1">> => #{ <<"value">> => <<"v1">> } },
+ #{
+ <<"k1">> =>
+ #{
+ <<"value">> => <<"v1">>,
+ <<"attributes">> => #{},
+ <<"flags">> => []
+ }
+ }
+ ],
+ [<<"k1=\"v1\"">>]
+ },
+ to_string_attributes =>
+ {
+ [
+ #{
+ <<"k1">> =>
+ #{
+ <<"value">> => <<"v1">>,
+ <<"attributes">> => #{ <<"k2">> => <<"v2">> }
+ }
+ },
+ #{
+ <<"k1">> =>
+ #{
+ <<"value">> => <<"v1">>,
+ <<"attributes">> => #{ <<"k2">> => <<"v2">> },
+ <<"flags">> => []
+ }
+ }
+ ],
+ [<<"k1=\"v1\"; k2=v2">>]
+ },
+ to_string_flags =>
+ {
+ [
+ #{
+ <<"k1">> =>
+ #{
+ <<"value">> => <<"v1">>,
+ <<"flags">> => [<<"f1">>, <<"f2">>]
+ }
+ },
+ #{
+ <<"k1">> =>
+ #{
+ <<"value">> => <<"v1">>,
+ <<"attributes">> => #{},
+ <<"flags">> => [<<"f1">>, <<"f2">>]
+ }
+ }
+ ],
+ [<<"k1=\"v1\"; f1; f2">>]
+ },
+ parse_realworld_1 =>
+ {
+ [
+ [
+ <<"cart=110045_77895_53420; SameSite=Strict">>,
+ <<"affiliate=e4rt45dw; SameSite=Lax">>
+ ]
+ ],
+ #{
+ <<"cart">> =>
+ #{
+ <<"value">> => <<"110045_77895_53420">>,
+ <<"attributes">> => #{ <<"SameSite">> => <<"Strict">> }
+ },
+ <<"affiliate">> =>
+ #{
+ <<"value">> => <<"e4rt45dw">>,
+ <<"attributes">> => #{ <<"SameSite">> => <<"Lax">> }
+ }
+ }
+ },
+ parse_user_settings_and_permissions =>
+ {
+ [
+ [
+ <<"user_settings=notifications=true,privacy=strict,layout=grid; Path=/; HttpOnly; Secure">>,
+ <<"user_permissions=\"read;write;delete\"; Path=/; SameSite=None; Secure">>
+ ]
+ ],
+ #{
+ <<"user_settings">> =>
+ #{
+ <<"value">> => <<"notifications=true,privacy=strict,layout=grid">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"HttpOnly">>, <<"Secure">>]
+ },
+ <<"user_permissions">> =>
+ #{
+ <<"value">> => <<"read;write;delete">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">>, <<"SameSite">> => <<"None">> },
+ <<"flags">> => [<<"Secure">>]
+ }
+ }
+ },
+ parse_session_and_temp_data =>
+ {
+ [
+ [
+ <<"SESSION_ID=abc123xyz ; path= /dashboard ; samesite=Strict ; Secure">>,
+ <<"temp_data=cleanup_me; Max-Age=-1; Path=/">>
+ ]
+ ],
+ #{
+ <<"SESSION_ID">> =>
+ #{
+ <<"value">> => <<"abc123xyz ">>,
+ <<"attributes">> => #{ <<"path">> => <<"/dashboard">>, <<"samesite">> => <<"Strict">> },
+ <<"flags">> => [<<"Secure">>]
+ },
+ <<"temp_data">> =>
+ #{
+ <<"value">> => <<"cleanup_me">>,
+ <<"attributes">> => #{ <<"Max-Age">> => <<"-1">>, <<"Path">> => <<"/">> }
+ }
+ }
+ },
+ parse_empty_and_anonymous =>
+ {
+ [
+ [
+ <<"user_preference=; Path=/; HttpOnly">>,
+ <<"=anonymous_session_123; Path=/guest">>
+ ]
+ ],
+ #{
+ <<"user_preference">> =>
+ #{
+ <<"value">> => <<"">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"HttpOnly">>]
+ },
+ <<>> =>
+ #{
+ <<"value">> => <<"anonymous_session_123">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/guest">> }
+ }
+ }
+ },
+ parse_app_config_and_analytics =>
+ {
+ [
+ [
+ <<"$app_config$=theme@dark!%20mode; Path=/">>,
+ <<"analytics_session_data_with_very_long_name_for_tracking_purposes=comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more; Path=/">>
+ ]
+ ],
+ #{
+ <<"$app_config$">> =>
+ #{
+ <<"value">> => <<"theme@dark! mode">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ },
+ <<"analytics_session_data_with_very_long_name_for_tracking_purposes">> =>
+ #{
+ <<"value">> => <<"comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ }
+ }
+ },
+ parse_debug_and_tracking =>
+ {
+ [
+ [
+ <<"debug_info=\\tIndented\\t\\nMultiline\\n; Path=/">>,
+ <<"tracking_id=user_12345; CustomAttr=CustomValue; Analytics=Enabled; Path=/; HttpOnly">>
+ ]
+ ],
+ #{
+ <<"debug_info">> =>
+ #{
+ <<"value">> => <<"\\tIndented\\t\\nMultiline\\n">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ },
+ <<"tracking_id">> =>
+ #{
+ <<"value">> => <<"user_12345">>,
+ <<"attributes">> => #{
+ <<"CustomAttr">> => <<"CustomValue">>,
+ <<"Analytics">> => <<"Enabled">>,
+ <<"Path">> => <<"/">>
+ },
+ <<"flags">> => [<<"HttpOnly">>]
+ }
+ }
+ },
+ parse_cache_and_form_token =>
+ {
+ [
+ [
+ <<"cache_bust=v1.2.3; Expires=Mon, 99 Feb 2099 25:99:99 GMT; Path=/">>,
+ <<"form_token=form_abc123; SameSite=Strick; Secure">>
+ ]
+ ],
+ #{
+ <<"cache_bust">> =>
+ #{
+ <<"value">> => <<"v1.2.3">>,
+ <<"attributes">> => #{
+ <<"Expires">> => <<"Mon, 99 Feb 2099 25:99:99 GMT">>,
+ <<"Path">> => <<"/">>
+ }
+ },
+ <<"form_token">> =>
+ #{
+ <<"value">> => <<"form_abc123">>,
+ <<"attributes">> => #{ <<"SameSite">> => <<"Strick">> },
+ <<"flags">> => [<<"Secure">>]
+ }
+ }
+ },
+ parse_token_and_reactions =>
+ {
+ [
+ [
+ <<"access_token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c; Path=/; HttpOnly; Secure">>,
+ <<"reaction_prefs=👍👎; Path=/; Secure">>
+ ]
+ ],
+ #{
+ <<"access_token">> =>
+ #{
+ <<"value">> => <<"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"HttpOnly">>, <<"Secure">>]
+ },
+ <<"reaction_prefs">> =>
+ #{
+ <<"value">> => <<"👍👎">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"Secure">>]
+ }
+ }
+ },
+ parse_error_log_and_auth_token =>
+ {
+ [
+ [
+ <<"error_log=\"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed\"; Path=/">>,
+ <<"auth_token=bearer_xyz789; Secure; Path=/api; Secure; HttpOnly">>
+ ]
+ ],
+ #{
+ <<"error_log">> =>
+ #{
+ <<"value">> => <<"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ },
+ <<"auth_token">> =>
+ #{
+ <<"value">> => <<"bearer_xyz789">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/api">> },
+ <<"flags">> => [<<"HttpOnly">>,<<"Secure">>, <<"Secure">>]
+ }
+ }
+ },
+ parse_csrf_and_quick_setting =>
+ {
+ [
+ [
+ <<"csrf_token=abc123; \"HttpOnly\"; Path=/">>,
+ <<"quick_setting=\"enabled\"">>
+ ]
+ ],
+ #{
+ <<"csrf_token">> =>
+ #{
+ <<"value">> => <<"abc123">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"HttpOnly">>]
+ },
+ <<"quick_setting">> => <<"enabled">>
+ }
+ },
+ parse_admin_and_upload =>
+ {
+ [
+ [
+ <<"secret_key=confidential; Path=%2Fadmin">>,
+ <<"admin_flag=true; Path=/">>
+ ]
+ ],
+ #{
+ <<"secret_key">> =>
+ #{
+ <<"value">> => <<"confidential">>,
+ <<"attributes">> => #{ <<"Path">> => <<"%2Fadmin">> }
+ },
+ <<"admin_flag">> =>
+ #{
+ <<"value">> => <<"true">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ }
+ }
+ },
+ parse_search_and_tags =>
+ {
+ [
+ [
+ <<"search_history=\"query,results\"; Path=/">>,
+ <<"user_tags=\"work,personal\"; Path=/">>
+ ]
+ ],
+ #{
+ <<"search_history">> =>
+ #{
+ <<"value">> => <<"query,results">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ },
+ <<"user_tags">> =>
+ #{
+ <<"value">> => <<"work,personal">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ }
+ }
+ },
+ to_string_realworld_1 =>
+ {
+ [
+ #{
+ <<"cart">> =>
+ #{
+ <<"value">> => <<"110045_77895_53420">>,
+ <<"attributes">> => #{ <<"SameSite">> => <<"Strict">> }
+ },
+ <<"affiliate">> =>
+ #{
+ <<"value">> => <<"e4rt45dw">>,
+ <<"attributes">> => #{ <<"SameSite">> => <<"Lax">> }
+ }
+ }
+ ],
+ [
+ <<"affiliate=\"e4rt45dw\"; SameSite=Lax">>,
+ <<"cart=\"110045_77895_53420\"; SameSite=Strict">>
+ ]
+ },
+ to_string_user_settings_and_permissions =>
+ {
+ [
+ #{
+ <<"user_settings">> =>
+ #{
+ <<"value">> => <<"notifications=true,privacy=strict,layout=grid">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"HttpOnly">>, <<"Secure">>]
+ },
+ <<"user_permissions">> =>
+ #{
+ <<"value">> => <<"read;write;delete">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">>, <<"SameSite">> => <<"None">> },
+ <<"flags">> => [<<"Secure">>]
+ }
+ }
+ ],
+ [
+ <<"user_permissions=\"read;write;delete\"; Path=/; SameSite=None; Secure">>,
+ <<"user_settings=\"notifications=true,privacy=strict,layout=grid\"; Path=/; HttpOnly; Secure">>
+ ]
+ },
+ to_string_session_and_temp_data =>
+ {
+ [
+ #{
+ <<"SESSION_ID">> =>
+ #{
+ <<"value">> => <<"abc123xyz ">>,
+ <<"attributes">> => #{ <<"path">> => <<"/dashboard">>, <<"samesite">> => <<"Strict">> },
+ <<"flags">> => [<<"Secure">>]
+ },
+ <<"temp_data">> =>
+ #{
+ <<"value">> => <<"cleanup_me">>,
+ <<"attributes">> => #{ <<"Max-Age">> => <<"-1">>, <<"Path">> => <<"/">> }
+ }
+ }
+ ],
+ [
+ <<"SESSION_ID=\"abc123xyz \"; path=/dashboard; samesite=Strict; Secure">>,
+ <<"temp_data=\"cleanup_me\"; Max-Age=-1; Path=/">>
+ ]
+ },
+ to_string_empty_and_anonymous =>
+ {
+ [
+ #{
+ <<"user_preference">> =>
+ #{
+ <<"value">> => <<"">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"HttpOnly">>]
+ },
+ <<>> =>
+ #{
+ <<"value">> => <<"anonymous_session_123">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/guest">> }
+ }
+ }
+ ],
+ [
+ <<"=\"anonymous_session_123\"; Path=/guest">>,
+ <<"user_preference=\"\"; Path=/; HttpOnly">>
+ ]
+ },
+ to_string_app_config_and_analytics =>
+ {
+ [
+ #{
+ <<"$app_config$">> =>
+ #{
+ <<"value">> => <<"theme@dark!%20mode">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ },
+ <<"analytics_session_data_with_very_long_name_for_tracking_purposes">> =>
+ #{
+ <<"value">> => <<"comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ }
+ }
+ ],
+ [
+ <<"$app_config$=\"theme@dark!%20mode\"; Path=/">>,
+ <<"analytics_session_data_with_very_long_name_for_tracking_purposes=\"comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more\"; Path=/">>
+ ]
+ },
+ to_string_debug_and_tracking =>
+ {
+ [
+ #{
+ <<"debug_info">> =>
+ #{
+ <<"value">> => <<"\\tIndented\\t\\nMultiline\\n">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ },
+ <<"tracking_id">> =>
+ #{
+ <<"value">> => <<"user_12345">>,
+ <<"attributes">> => #{
+ <<"CustomAttr">> => <<"CustomValue">>,
+ <<"Analytics">> => <<"Enabled">>,
+ <<"Path">> => <<"/">>
+ },
+ <<"flags">> => [<<"HttpOnly">>]
+ }
+ }
+ ],
+ [
+ <<"debug_info=\"\\tIndented\\t\\nMultiline\\n\"; Path=/">>,
+ <<"tracking_id=\"user_12345\"; Analytics=Enabled; CustomAttr=CustomValue; Path=/; HttpOnly">>
+ ]
+ },
+ to_string_cache_and_form_token =>
+ {
+ [
+ #{
+ <<"cache_bust">> =>
+ #{
+ <<"value">> => <<"v1.2.3">>,
+ <<"attributes">> => #{
+ <<"Expires">> => <<"Mon, 99 Feb 2099 25:99:99 GMT">>,
+ <<"Path">> => <<"/">>
+ }
+ },
+ <<"form_token">> =>
+ #{
+ <<"value">> => <<"form_abc123">>,
+ <<"attributes">> => #{ <<"SameSite">> => <<"Strick">> },
+ <<"flags">> => [<<"Secure">>]
+ }
+ }
+ ],
+ [
+ <<"cache_bust=\"v1.2.3\"; Expires=Mon, 99 Feb 2099 25:99:99 GMT; Path=/">>,
+ <<"form_token=\"form_abc123\"; SameSite=Strick; Secure">>
+ ]
+ },
+ to_string_token_and_reactions =>
+ {
+ [
+ #{
+ <<"access_token">> =>
+ #{
+ <<"value">> => <<"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"HttpOnly">>, <<"Secure">>]
+ },
+ <<"reaction_prefs">> =>
+ #{
+ <<"value">> => <<"👍👎">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"Secure">>]
+ }
+ }
+ ],
+ [
+ <<"access_token=\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c\"; Path=/; HttpOnly; Secure">>,
+ <<"reaction_prefs=\"👍👎\"; Path=/; Secure">>
+ ]
+ },
+ to_string_error_log_and_auth_token =>
+ {
+ [
+ #{
+ <<"error_log">> =>
+ #{
+ <<"value">> => <<"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ },
+ <<"auth_token">> =>
+ #{
+ <<"value">> => <<"bearer_xyz789">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/api">> },
+ <<"flags">> => [<<"HttpOnly">>, <<"Secure">>, <<"Secure">>]
+ }
+ }
+ ],
+ [
+ <<"auth_token=\"bearer_xyz789\"; Path=/api; HttpOnly; Secure; Secure">>,
+ <<"error_log=\"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed\"; Path=/">>
+ ]
+ },
+ to_string_csrf_and_quick_setting =>
+ {
+ [
+ #{
+ <<"csrf_token">> =>
+ #{
+ <<"value">> => <<"abc123">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> },
+ <<"flags">> => [<<"HttpOnly">>]
+ },
+ <<"quick_setting">> => <<"enabled">>
+ }
+ ],
+ [
+ <<"csrf_token=\"abc123\"; Path=/; HttpOnly">>,
+ <<"quick_setting=\"enabled\"">>
+ ]
+ },
+ to_string_admin_and_upload =>
+ {
+ [
+ #{
+ <<"secret_key">> =>
+ #{
+ <<"value">> => <<"confidential">>,
+ <<"attributes">> => #{ <<"Path">> => <<"%2Fadmin">> }
+ },
+ <<"admin_flag">> =>
+ #{
+ <<"value">> => <<"true">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ }
+ }
+ ],
+ [
+ <<"admin_flag=\"true\"; Path=/">>,
+ <<"secret_key=\"confidential\"; Path=%2Fadmin">>
+ ]
+ },
+ to_string_search_and_tags =>
+ {
+ [
+ #{
+ <<"search_history">> =>
+ #{
+ <<"value">> => <<"query,results">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ },
+ <<"user_tags">> =>
+ #{
+ <<"value">> => <<"work,personal">>,
+ <<"attributes">> => #{ <<"Path">> => <<"/">> }
+ }
+ }
+ ],
+ [
+ <<"search_history=\"query,results\"; Path=/">>,
+ <<"user_tags=\"work,personal\"; Path=/">>
+ ]
+ }
+ }.
+```
+
+### from_string_basic_test
+
+```erlang
+from_string_basic_test() ->
+ assert_set(from_string_raw_value, fun from_string/1).
+```
+
+### from_string_attributes_test
+
+```erlang
+from_string_attributes_test() ->
+ assert_set(from_string_attributes, fun from_string/1).
+```
+
+### from_string_flags_test
+
+```erlang
+from_string_flags_test() ->
+ assert_set(from_string_flags, fun from_string/1).
+```
+
+### to_string_basic_test
+
+```erlang
+to_string_basic_test() ->
+ assert_set(to_string_raw_value, fun to_string/1).
+```
+
+### to_string_attributes_test
+
+```erlang
+to_string_attributes_test() ->
+ assert_set(to_string_attributes, fun to_string/1).
+```
+
+### to_string_flags_test
+
+```erlang
+to_string_flags_test() ->
+ assert_set(to_string_flags, fun to_string/1).
+```
+
+### parse_realworld_test
+
+```erlang
+parse_realworld_test() ->
+ assert_set(parse_realworld_1, fun from_string/1).
+```
+
+### parse_user_settings_and_permissions_test
+
+```erlang
+parse_user_settings_and_permissions_test() ->
+ assert_set(parse_user_settings_and_permissions, fun from_string/1).
+```
+
+### parse_session_and_temp_data_test
+
+```erlang
+parse_session_and_temp_data_test() ->
+ assert_set(parse_session_and_temp_data, fun from_string/1).
+```
+
+### parse_empty_and_anonymous_test
+
+```erlang
+parse_empty_and_anonymous_test() ->
+ assert_set(parse_empty_and_anonymous, fun from_string/1).
+```
+
+### parse_app_config_and_analytics_test
+
+```erlang
+parse_app_config_and_analytics_test() ->
+ assert_set(parse_app_config_and_analytics, fun from_string/1).
+```
+
+### parse_debug_and_tracking_test
+
+```erlang
+parse_debug_and_tracking_test() ->
+ assert_set(parse_debug_and_tracking, fun from_string/1).
+```
+
+### parse_cache_and_form_token_test
+
+```erlang
+parse_cache_and_form_token_test() ->
+ assert_set(parse_cache_and_form_token, fun from_string/1).
+```
+
+### parse_token_and_reactions_test
+
+```erlang
+parse_token_and_reactions_test() ->
+ assert_set(parse_token_and_reactions, fun from_string/1).
+```
+
+### parse_error_log_and_auth_token_test
+
+```erlang
+parse_error_log_and_auth_token_test() ->
+ assert_set(parse_error_log_and_auth_token, fun from_string/1).
+```
+
+### parse_csrf_and_quick_setting_test
+
+```erlang
+parse_csrf_and_quick_setting_test() ->
+ assert_set(parse_csrf_and_quick_setting, fun from_string/1).
+```
+
+### parse_admin_and_upload_test
+
+```erlang
+parse_admin_and_upload_test() ->
+ assert_set(parse_admin_and_upload, fun from_string/1).
+```
+
+### parse_search_and_tags_test
+
+```erlang
+parse_search_and_tags_test() ->
+ assert_set(parse_search_and_tags, fun from_string/1).
+```
+
+### to_string_realworld_1_test
+
+```erlang
+to_string_realworld_1_test() ->
+ assert_set(to_string_realworld_1, fun to_string/1).
+```
+
+### to_string_user_settings_and_permissions_test
+
+```erlang
+to_string_user_settings_and_permissions_test() ->
+ assert_set(to_string_user_settings_and_permissions, fun to_string/1).
+```
+
+### to_string_session_and_temp_data_test
+
+```erlang
+to_string_session_and_temp_data_test() ->
+ assert_set(to_string_session_and_temp_data, fun to_string/1).
+```
+
+### to_string_empty_and_anonymous_test
+
+```erlang
+to_string_empty_and_anonymous_test() ->
+ assert_set(to_string_empty_and_anonymous, fun to_string/1).
+```
+
+### to_string_app_config_and_analytics_test
+
+```erlang
+to_string_app_config_and_analytics_test() ->
+ assert_set(to_string_app_config_and_analytics, fun to_string/1).
+```
+
+### to_string_debug_and_tracking_test
+
+```erlang
+to_string_debug_and_tracking_test() ->
+ assert_set(to_string_debug_and_tracking, fun to_string/1).
+```
+
+### to_string_cache_and_form_token_test
+
+```erlang
+to_string_cache_and_form_token_test() ->
+ assert_set(to_string_cache_and_form_token, fun to_string/1).
+```
+
+### to_string_token_and_reactions_test
+
+```erlang
+to_string_token_and_reactions_test() ->
+ assert_set(to_string_token_and_reactions, fun to_string/1).
+```
+
+### to_string_error_log_and_auth_token_test
+
+```erlang
+to_string_error_log_and_auth_token_test() ->
+ assert_set(to_string_error_log_and_auth_token, fun to_string/1).
+```
+
+### to_string_csrf_and_quick_setting_test
+
+```erlang
+to_string_csrf_and_quick_setting_test() ->
+ assert_set(to_string_csrf_and_quick_setting, fun to_string/1).
+```
+
+### to_string_admin_and_upload_test
+
+```erlang
+to_string_admin_and_upload_test() ->
+ assert_set(to_string_admin_and_upload, fun to_string/1).
+```
+
+### to_string_search_and_tags_test
+
+```erlang
+to_string_search_and_tags_test() ->
+```
+
+---
+
+*Generated from [dev_codec_cookie_test_vectors.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie_test_vectors.erl)*
diff --git a/docs/book/src/dev_codec_flat.erl.md b/docs/book/src/dev_codec_flat.erl.md
new file mode 100644
index 000000000..4bb41a9a6
--- /dev/null
+++ b/docs/book/src/dev_codec_flat.erl.md
@@ -0,0 +1,266 @@
+# dev_codec_flat
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_flat.erl)
+
+A codec for turning TABMs into/from flat Erlang maps that have
+(potentially multi-layer) paths as their keys, and a normal TABM binary as
+their value.
+
+---
+
+## Exported Functions
+
+- `commit/3`
+- `deserialize/1`
+- `from/3`
+- `serialize/1`
+- `serialize/2`
+- `to/3`
+- `verify/3`
+
+---
+
+### commit
+
+A codec for turning TABMs into/from flat Erlang maps that have
+
+```erlang
+commit(Msg, Req, Opts) -> dev_codec_httpsig:commit(Msg, Req, Opts).
+```
+
+### verify
+
+A codec for turning TABMs into/from flat Erlang maps that have
+Convert a flat map to a TABM.
+
+```erlang
+verify(Msg, Req, Opts) -> dev_codec_httpsig:verify(Msg, Req, Opts).
+```
+
+### from
+
+A codec for turning TABMs into/from flat Erlang maps that have
+Convert a flat map to a TABM.
+
+```erlang
+from(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin};
+```
+
+### from
+
+A codec for turning TABMs into/from flat Erlang maps that have
+Convert a flat map to a TABM.
+
+```erlang
+from(Map, Req, Opts) when is_map(Map) ->
+ {ok,
+ maps:fold(
+ fun(Path, Value, Acc) ->
+ case Value of
+ [] ->
+ ?event(error,
+ {empty_list_value,
+ {path, Path},
+ {value, Value},
+ {map, Map}
+ }
+ );
+ _ ->
+ ok
+ end,
+ hb_util:deep_set(
+ hb_path:term_to_path_parts(Path, Opts),
+ hb_util:ok(from(Value, Req, Opts)),
+ Acc,
+ Opts
+ )
+ end,
+ #{},
+ Map
+ )
+ }.
+```
+
+### to
+
+Convert a TABM to a flat map.
+
+```erlang
+to(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin};
+```
+
+### to
+
+Convert a TABM to a flat map.
+
+```erlang
+to(Map, Req, Opts) when is_map(Map) ->
+ Res =
+ maps:fold(
+ fun(Key, Value, Acc) ->
+ case to(Value, Req, Opts) of
+ {ok, SubMap} when is_map(SubMap) ->
+ maps:fold(
+ fun(SubKey, SubValue, InnerAcc) ->
+ maps:put(
+ hb_path:to_binary([Key, SubKey]),
+ SubValue,
+ InnerAcc
+ )
+ end,
+ Acc,
+ SubMap
+ );
+ {ok, SimpleValue} ->
+ maps:put(hb_path:to_binary([Key]), SimpleValue, Acc)
+ end
+ end,
+ #{},
+ Map
+ ),
+ {ok, Res}.
+```
+
+### serialize
+
+```erlang
+serialize(Map) when is_map(Map) ->
+ serialize(Map, #{}).
+```
+
+### serialize
+
+```erlang
+serialize(Map, Opts) when is_map(Map) ->
+ Flattened = hb_message:convert(Map, <<"flat@1.0">>, #{}),
+ {ok,
+ iolist_to_binary(lists:foldl(
+ fun(Key, Acc) ->
+ [
+ Acc,
+ hb_path:to_binary(Key),
+ <<": ">>,
+ hb_maps:get(Key, Flattened, Opts), <<"\n">>
+ ]
+ end,
+ <<>>,
+ hb_util:to_sorted_keys(Flattened, Opts)
+ )
+ )
+ }.
+```
+
+### deserialize
+
+```erlang
+deserialize(Bin) when is_binary(Bin) ->
+ Flat = lists:foldl(
+ fun(Line, Acc) ->
+ case binary:split(Line, <<": ">>, [global]) of
+ [Key, Value] ->
+ Acc#{ Key => Value };
+ _ ->
+ Acc
+ end
+ end,
+ #{},
+ binary:split(Bin, <<"\n">>, [global])
+ ),
+ {ok, hb_message:convert(Flat, <<"structured@1.0">>, <<"flat@1.0">>, #{})}.
+%%% Tests
+```
+
+### simple_conversion_test
+
+```erlang
+simple_conversion_test() ->
+ Flat = #{[<<"a">>] => <<"value">>},
+ Nested = #{<<"a">> => <<"value">>},
+ ?assert(hb_message:match(Nested, hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})))),
+ ?assert(hb_message:match(Flat, hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})))).
+```
+
+### nested_conversion_test
+
+```erlang
+nested_conversion_test() ->
+ Flat = #{<<"a/b">> => <<"value">>},
+ Nested = #{<<"a">> => #{<<"b">> => <<"value">>}},
+ Unflattened = hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})),
+ Flattened = hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})),
+ ?assert(hb_message:match(Nested, Unflattened)),
+ ?assert(hb_message:match(Flat, Flattened)).
+```
+
+### multiple_paths_test
+
+```erlang
+multiple_paths_test() ->
+ Flat = #{
+ <<"x/y">> => <<"1">>,
+ <<"x/z">> => <<"2">>,
+ <<"a">> => <<"3">>
+ },
+ Nested = #{
+ <<"x">> => #{
+ <<"y">> => <<"1">>,
+ <<"z">> => <<"2">>
+ },
+ <<"a">> => <<"3">>
+ },
+ ?assert(hb_message:match(Nested, hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})))),
+ ?assert(hb_message:match(Flat, hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})))).
+```
+
+### path_list_test
+
+```erlang
+path_list_test() ->
+ Nested = #{
+ <<"x">> => #{
+ [<<"y">>, <<"z">>] => #{
+ <<"a">> => <<"2">>
+ },
+ <<"a">> => <<"2">>
+ }
+ },
+ Flat = hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})),
+ lists:foreach(
+ fun(Key) ->
+ ?assert(not lists:member($\n, binary_to_list(Key)))
+ end,
+ hb_maps:keys(Flat, #{})
+ ).
+```
+
+### binary_passthrough_test
+
+```erlang
+binary_passthrough_test() ->
+ Bin = <<"raw binary">>,
+ ?assertEqual(Bin, hb_util:ok(dev_codec_flat:from(Bin, #{}, #{}))),
+ ?assertEqual(Bin, hb_util:ok(dev_codec_flat:to(Bin, #{}, #{}))).
+```
+
+### deep_nesting_test
+
+```erlang
+deep_nesting_test() ->
+ Flat = #{<<"a/b/c/d">> => <<"deep">>},
+ Nested = #{<<"a">> => #{<<"b">> => #{<<"c">> => #{<<"d">> => <<"deep">>}}}},
+ Unflattened = hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})),
+ Flattened = hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})),
+ ?assert(hb_message:match(Nested, Unflattened)),
+ ?assert(hb_message:match(Flat, Flattened)).
+```
+
+### empty_map_test
+
+```erlang
+empty_map_test() ->
+ ?assertEqual(#{}, hb_util:ok(dev_codec_flat:from(#{}, #{}, #{}))),
+```
+
+---
+
+*Generated from [dev_codec_flat.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_flat.erl)*
diff --git a/docs/book/src/dev_codec_http_auth.erl.md b/docs/book/src/dev_codec_http_auth.erl.md
new file mode 100644
index 000000000..50b677225
--- /dev/null
+++ b/docs/book/src/dev_codec_http_auth.erl.md
@@ -0,0 +1,210 @@
+# dev_codec_http_auth
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_http_auth.erl)
+
+Implements a two-step authentication process for HTTP requests, using
+the `Basic` authentication scheme. This device is a viable implementation
+of the `generator` interface type employed by `~auth-hook@1.0`, as well as
+the `~message@1.0` commitment scheme interface.
+`http-auth@1.0``s `commit` and `verify` keys proxy to the `~httpsig@1.0`
+secret key HMAC commitment scheme, utilizing a secret key derived from the
+user's authentication information. Callers may also utilize the `generate`
+key directly to derive entropy from HTTP Authorization headers provided by
+the user. If no Authorization header is provided, the `generate` key will
+return a `401 Unauthorized` response, which triggers a recipient`s browser
+to prompt the user for authentication details and resend the request.
+The `generate` key derives secrets for it's users by calling PBKDF2 with
+the user's authentication information. The parameters for the PBKDF2
+algorithm are configurable, and can be specified in the request message:
+
+ salt: The salt to use for the PBKDF2 algorithm. Defaults to
+ `sha256("constant:ao")`.
+ iterations: The number of iterations to use for the PBKDF2 algorithm.
+ Defaults to `1,200,000`.
+ alg: The hashing algorithm to use with PBKDF2. Defaults to
+ `sha256`.
+ key-length: The length of the key to derive from PBKDF2. Defaults to
+ `64`.
+
+The default iteration count was chosen at two times the recommendation of
+OWASP in 2023 (600,000), and executes at a run rate of ~5-10 key derivations
+per second on modern CPU hardware. Additionally, the default salt was chosen
+such that it is a public constant (needed in order for reproducibility
+between nodes), and hashed in order to provide additional entropy, in
+alignment with RFC 8018, Section 4.1.
+
+---
+
+## Exported Functions
+
+- `commit/3`
+- `generate/3`
+- `verify/3`
+
+---
+
+### commit
+
+Implements a two-step authentication process for HTTP requests, using
+The default salt to use for the PBKDF2 algorithm. This value must be
+Generate or extract a new secret and commit to the message with the
+
+```erlang
+commit(Base, Req, Opts) ->
+ case generate(Base, Req, Opts) of
+ {ok, Key} ->
+ {ok, CommitRes} =
+ dev_codec_httpsig_proxy:commit(
+ <<"http-auth@1.0">>,
+ Key,
+ Base,
+ Req,
+ Opts
+ ),
+ ?event({commit_result, CommitRes}),
+ {ok, CommitRes};
+ {error, Err} ->
+ {error, Err}
+ end.
+```
+
+### verify
+
+Verify a given `Base` message with a derived `Key` using the
+
+```erlang
+verify(Base, RawReq, Opts) ->
+ ?event({verify_invoked, {base, Base}, {req, RawReq}}),
+ {ok, Key} = generate(Base, RawReq, Opts),
+ ?event({verify_found_key, {key, Key}, {base, Base}, {req, RawReq}}),
+ {ok, VerifyRes} =
+ dev_codec_httpsig_proxy:verify(
+ Key,
+ Base,
+ RawReq,
+ Opts
+ ),
+ ?event({verify_result, VerifyRes}),
+ {ok, VerifyRes}.
+```
+
+### generate
+
+Collect authentication information from the client. If the `raw` flag
+
+```erlang
+generate(_Msg, ReqLink, Opts) when ?IS_LINK(ReqLink) ->
+ generate(_Msg, hb_cache:ensure_loaded(ReqLink, Opts), Opts);
+```
+
+### generate
+
+Collect authentication information from the client. If the `raw` flag
+
+```erlang
+generate(_Msg, #{ <<"secret">> := Secret }, _Opts) ->
+ {ok, Secret};
+```
+
+### generate
+
+Collect authentication information from the client. If the `raw` flag
+
+```erlang
+generate(_Msg, Req, Opts) ->
+ case hb_maps:get(<<"authorization">>, Req, undefined, Opts) of
+ <<"Basic ", Auth/binary>> ->
+ Decoded = base64:decode(Auth),
+ ?event(key_gen, {generated_key, {auth, Auth}, {decoded, Decoded}}),
+ case hb_maps:get(<<"raw">>, Req, false, Opts) of
+ true -> {ok, Decoded};
+ false -> derive_key(Decoded, Req, Opts)
+ end;
+ undefined ->
+ {error,
+ #{
+ <<"status">> => 401,
+ <<"www-authenticate">> => <<"Basic">>,
+ <<"details">> => <<"No authorization header provided.">>
+ }
+ };
+ Unrecognized ->
+ {error,
+ #{
+ <<"status">> => 400,
+ <<"details">> =>
+ <<"Unrecognized authorization header: ", Unrecognized/binary>>
+ }
+ }
+ end.
+```
+
+### derive_key
+
+Derive a key from the authentication information using the PBKDF2
+
+```erlang
+derive_key(Decoded, Req, Opts) ->
+ Alg = hb_util:atom(hb_maps:get(<<"alg">>, Req, <<"sha256">>, Opts)),
+ Salt =
+ hb_maps:get(
+ <<"salt">>,
+ Req,
+ hb_crypto:sha256(?DEFAULT_SALT),
+ Opts
+ ),
+ Iterations = hb_maps:get(<<"iterations">>, Req, 2 * 600_000, Opts),
+ KeyLength = hb_maps:get(<<"key-length">>, Req, 64, Opts),
+ ?event(key_gen,
+ {derive_key,
+ {alg, Alg},
+ {salt, Salt},
+ {iterations, Iterations},
+ {key_length, KeyLength}
+ }
+ ),
+ case hb_crypto:pbkdf2(Alg, Decoded, Salt, Iterations, KeyLength) of
+ {ok, Key} ->
+ EncodedKey = hb_util:encode(Key),
+ {ok, EncodedKey};
+ {error, Err} ->
+ ?event(key_gen,
+ {pbkdf2_error,
+ {alg, Alg},
+ {salt, Salt},
+ {iterations, Iterations},
+ {key_length, KeyLength},
+ {error, Err}
+ }
+ ),
+ {error,
+ #{
+ <<"status">> => 500,
+ <<"details">> => <<"Failed to derive key.">>
+ }
+ }
+ end.
+```
+
+### benchmark_pbkdf2_test
+
+```erlang
+benchmark_pbkdf2_test() ->
+ Key = crypto:strong_rand_bytes(32),
+ Iterations = 2 * 600_000,
+ KeyLength = 32,
+ Derivations =
+ hb_test_utils:benchmark(
+ fun() ->
+ hb_crypto:pbkdf2(sha256, Key, <<"salt">>, Iterations, KeyLength)
+ end
+ ),
+ hb_test_utils:benchmark_print(
+ <<"Derived">>,
+ <<"keys (1.2m iterations each)">>,
+ Derivations
+```
+
+---
+
+*Generated from [dev_codec_http_auth.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_http_auth.erl)*
diff --git a/docs/book/src/dev_codec_httpsig.erl.md b/docs/book/src/dev_codec_httpsig.erl.md
new file mode 100644
index 000000000..ae5c48ab9
--- /dev/null
+++ b/docs/book/src/dev_codec_httpsig.erl.md
@@ -0,0 +1,447 @@
+# dev_codec_httpsig
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_httpsig.erl)
+
+This module implements HTTP Message Signatures as described in RFC-9421
+(https://datatracker.ietf.org/doc/html/rfc9421), as an AO-Core device.
+It implements the codec standard (from/1, to/1), as well as the optional
+commitment functions (id/3, sign/3, verify/3). The commitment functions
+are found in this module, while the codec functions are relayed to the
+`dev_codec_httpsig_conv` module.
+
+---
+
+## Exported Functions
+
+- `add_content_digest/2`
+- `commit/3`
+- `from/3`
+- `normalize_for_encoding/3`
+- `serialize/2`
+- `serialize/3`
+- `to/3`
+- `verify/3`
+
+---
+
+### to
+
+This module implements HTTP Message Signatures as described in RFC-9421
+
+```erlang
+to(Msg, Req, Opts) -> dev_codec_httpsig_conv:to(Msg, Req, Opts).
+```
+
+### from
+
+This module implements HTTP Message Signatures as described in RFC-9421
+Generate the `Opts` to use during AO-Core operations in the codec.
+
+```erlang
+from(Msg, Req, Opts) -> dev_codec_httpsig_conv:from(Msg, Req, Opts).
+```
+
+### opts
+
+This module implements HTTP Message Signatures as described in RFC-9421
+Generate the `Opts` to use during AO-Core operations in the codec.
+
+```erlang
+opts(RawOpts) ->
+ RawOpts#{
+ hashpath => ignore,
+ cache_control => [<<"no-cache">>, <<"no-store">>],
+ force_message => false
+ }.
+```
+
+### serialize
+
+A helper utility for creating a direct encoding of a HTTPSig message.
+
+```erlang
+serialize(Msg, Opts) -> serialize(Msg, #{}, Opts).
+```
+
+### serialize
+
+A helper utility for creating a direct encoding of a HTTPSig message.
+
+```erlang
+serialize(Msg, #{ <<"format">> := <<"components">> }, Opts) ->
+ % Convert to HTTPSig via TABM through calling `hb_message:convert` rather
+ % than executing `to/3` directly. This ensures that our responses are
+ % normalized.
+```
+
+### serialize
+
+```erlang
+serialize(Msg, _Req, Opts) ->
+ % We assume the default format of `binary` if none of the prior clauses
+ % match.
+```
+
+### verify
+
+```erlang
+verify(Base, Req, RawOpts) ->
+ % A rsa-pss-sha512 commitment is verified by regenerating the signature
+ % base and validating against the signature.
+```
+
+### commit
+
+Commit to a message using the HTTP-Signature format. We use the `type`
+
+```erlang
+commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) ->
+ commit(Msg, Req#{ <<"type">> => <<"hmac-sha256">> }, Opts);
+```
+
+### commit
+
+Commit to a message using the HTTP-Signature format. We use the `type`
+
+```erlang
+commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) ->
+ commit(Msg, Req#{ <<"type">> => <<"rsa-pss-sha512">> }, Opts);
+```
+
+### commit
+
+Commit to a message using the HTTP-Signature format. We use the `type`
+
+```erlang
+commit(MsgToSign, Req = #{ <<"type">> := <<"rsa-pss-sha512">> }, RawOpts) ->
+ ?event(
+ {generating_rsa_pss_sha512_commitment, {msg, MsgToSign}, {req, Req}}
+ ),
+ Opts = opts(RawOpts),
+ Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts),
+ if Wallet =:= no_viable_wallet ->
+ throw({cannot_commit, no_viable_wallet, MsgToSign});
+ true ->
+ ok
+ end,
+ % Utilize the hashpath, if present, as the tag for the commitment.
+```
+
+### commit
+
+```erlang
+commit(BaseMsg, Req = #{ <<"type">> := <<"hmac-sha256">> }, RawOpts) ->
+ % Extract the key material from the request.
+```
+
+### maybe_bundle_tag_commitment
+
+Annotate the commitment with the `bundle` key if the request contains
+
+```erlang
+maybe_bundle_tag_commitment(Commitment, Req, _Opts) ->
+ case hb_util:atom(maps:get(<<"bundle">>, Req, false)) of
+ true -> Commitment#{ <<"bundle">> => <<"true">> };
+ false -> Commitment
+ end.
+```
+
+### keys_to_commit
+
+Derive the set of keys to commit to from a `commit` request and a
+
+```erlang
+keys_to_commit(_Base, #{ <<"committed">> := Explicit}, _Opts) ->
+ % Case 1: Explicitly provided keys to commit.
+```
+
+### keys_to_commit
+
+```erlang
+keys_to_commit(Base, _Req, Opts) ->
+ % Extract the set of committed keys from the message.
+```
+
+### add_content_digest
+
+If the `body` key is present and a binary, replace it with a
+
+```erlang
+add_content_digest(Msg, _Opts) ->
+ case maps:get(<<"body">>, Msg, not_found) of
+ Body when is_binary(Body) ->
+ % Remove the body from the message and add the content-digest,
+ % encoded as a structured field.
+```
+
+### normalize_for_encoding
+
+Given a base message and a commitment, derive the message and commitment
+
+```erlang
+normalize_for_encoding(Msg, Commitment, Opts) ->
+ % Extract the requested keys to include in the signature base.
+```
+
+### key_present
+
+Calculate if a key or its `+link` TABM variant is present in a message.
+create the signature base that will be signed in order to create the
+
+```erlang
+key_present(Key, Msg) ->
+ NormalizedKey = hb_ao:normalize_key(Key),
+ maps:is_key(NormalizedKey, Msg)
+ orelse maps:is_key(<+ M1/Computed when /Pass == 1 -> + Assumes: + M1/priv/wasm/instance + M1/Process + M2/Message + M2/Assignment/Block-Height + Generates: + /wasm/handler + /wasm/params + Side-effects: + Writes the process and message as JSON representations into the + WASM environment. + M1/Computed when M2/Pass == 2 -> + Assumes: + M1/priv/wasm/instance + M2/Results + M2/Process + Generates: + /Results/Outbox + /Results/Data+ +--- + +## Exported Functions + +- `compute/3` +- `generate_aos_msg/2` +- `generate_stack/1` +- `generate_stack/2` +- `generate_stack/3` +- `init/3` +- `json_to_message/2` +- `message_to_json_struct/2` + +--- + +### init + +A device that provides a way for WASM execution to interact with +Initialize the device. +On first pass prepare the call, on second pass get the results. + +```erlang +init(M1, _M2, Opts) -> + {ok, hb_ao:set(M1, #{<<"function">> => <<"handle">>}, Opts)}. +``` + +### compute + +A device that provides a way for WASM execution to interact with +Initialize the device. +On first pass prepare the call, on second pass get the results. + +```erlang +compute(M1, M2, Opts) -> + case hb_ao:get(<<"pass">>, M1, Opts) of + 1 -> prep_call(M1, M2, Opts); + 2 -> results(M1, M2, Opts); + _ -> {ok, M1} + end. +``` + +### prep_call + +Prepare the WASM environment for execution by writing the process string and + +```erlang +prep_call(RawM1, RawM2, Opts) -> + M1 = hb_cache:ensure_all_loaded(RawM1, Opts), + M2 = hb_cache:ensure_all_loaded(RawM2, Opts), + ?event({prep_call, M1, M2, Opts}), + Process = hb_ao:get(<<"process">>, M1, Opts#{ hashpath => ignore }), + Message = hb_ao:get(<<"body">>, M2, Opts#{ hashpath => ignore }), + Image = hb_ao:get(<<"process/image">>, M1, Opts), + BlockHeight = hb_ao:get(<<"block-height">>, M2, Opts), + Props = message_to_json_struct(denormalize_message(Message, Opts), Opts), + MsgProps = + Props#{ + <<"Module">> => Image, + <<"Block-Height">> => BlockHeight + }, + MsgJson = hb_json:encode(MsgProps), + ProcessProps = + #{ + <<"Process">> => message_to_json_struct(Process, Opts) + }, + ProcessJson = hb_json:encode(ProcessProps), + env_write(ProcessJson, MsgJson, M1, M2, Opts). +``` + +### denormalize_message + +Normalize a message for AOS-compatibility. + +```erlang +denormalize_message(Message, Opts) -> + NormOwnerMsg = + case hb_message:signers(Message, Opts) of + [] -> Message; + [PrimarySigner|_] -> + {ok, _, Commitment} = hb_message:commitment(PrimarySigner, Message, Opts), + Message#{ + <<"owner">> => hb_util:human_id(PrimarySigner), + <<"signature">> => + hb_ao:get(<<"signature">>, Commitment, <<>>, Opts) + } + end, + NormOwnerMsg#{ + <<"id">> => hb_message:id(Message, all, Opts) + }. +``` + +### message_to_json_struct + +```erlang +message_to_json_struct(RawMsg, Opts) -> + message_to_json_struct(RawMsg, [owner_as_address], Opts). +``` + +### message_to_json_struct + +```erlang +message_to_json_struct(RawMsg, Features, Opts) -> + TABM = + hb_message:convert( + hb_private:reset(RawMsg), + tabm, + Opts + ), + MsgWithoutCommitments = hb_maps:without([<<"commitments">>], TABM, Opts), + ID = hb_message:id(RawMsg, all), + ?event({encoding, {id, ID}, {msg, RawMsg}}), + {Owner, Signature} = + case hb_message:signers(RawMsg, Opts) of + [] -> {<<>>, <<>>}; + [Signer|_] -> + {ok, _, Commitment} = + hb_message:commitment(Signer, RawMsg, Opts), + CommitmentSignature = + hb_ao:get(<<"signature">>, Commitment, <<>>, Opts), + case lists:member(owner_as_address, Features) of + true -> + { + hb_util:native_id(Signer), + CommitmentSignature + }; + false -> + CommitmentOwner = + hb_ao:get_first( + [ + {Commitment, <<"key">>}, + {Commitment, <<"owner">>} + ], + no_signing_public_key_found_in_commitment, + Opts + ), + {CommitmentOwner, CommitmentSignature} + end + end, + Last = + hb_ao:get( + <<"anchor">>, + {as, <<"message@1.0">>, MsgWithoutCommitments}, + <<>>, + Opts + ), + Data = + hb_ao:get( + <<"data">>, + {as, <<"message@1.0">>, MsgWithoutCommitments}, + <<>>, + Opts + ), + Target = + hb_ao:get( + <<"target">>, + {as, <<"message@1.0">>, MsgWithoutCommitments}, + <<>>, + Opts + ), + % Set "From" if From-Process is Tag or set with "Owner" address + From = + hb_ao:get( + <<"from-process">>, + {as, <<"message@1.0">>, MsgWithoutCommitments}, + hb_util:encode(Owner), + Opts + ), + #{ + <<"Id">> => safe_to_id(ID), + % NOTE: In Arweave TXs, these are called "last_tx" + <<"Anchor">> => Last, + % NOTE: When sent to ao "Owner" is the wallet address + <<"Owner">> => hb_util:encode(Owner), + <<"From">> => case ?IS_ID(From) of true -> safe_to_id(From); false -> From end, + <<"Tags">> => prepare_tags(TABM, Opts), + <<"Target">> => safe_to_id(Target), + <<"Data">> => Data, + <<"Signature">> => + case byte_size(Signature) of + 0 -> <<>>; + 512 -> hb_util:encode(Signature); + _ -> Signature + end + }. +``` + +### prepare_tags + +Prepare the tags of a message as a key-value list, for use in the + +```erlang +prepare_tags(Msg, Opts) -> + % Prepare an ANS-104 message for JSON-Struct construction. +``` + +### prepare_header_case_tags + +Convert a message without an `original-tags` field into a list of + +```erlang +prepare_header_case_tags(TABM, Opts) -> + % Prepare a non-ANS-104 message for JSON-Struct construction. +``` + +### json_to_message + +Translates a compute result -- either from a WASM execution using the + +```erlang +json_to_message(JSON, Opts) when is_binary(JSON) -> + json_to_message(hb_json:decode(JSON), Opts); +``` + +### json_to_message + +Translates a compute result -- either from a WASM execution using the + +```erlang +json_to_message(Resp, Opts) when is_map(Resp) -> + {ok, Data, Messages, Patches} = normalize_results(Resp), + Output = + #{ + <<"outbox">> => + hb_maps:from_list( + [ + {MessageNum, preprocess_results(Msg, Opts)} + || + {MessageNum, Msg} <- + lists:zip( + lists:seq(1, length(Messages)), + Messages + ) + ] + ), + <<"patches">> => lists:map(fun(Patch) -> tags_to_map(Patch, Opts) end, Patches), + <<"data">> => Data + }, + {ok, Output}; +``` + +### json_to_message + +Translates a compute result -- either from a WASM execution using the + +```erlang +json_to_message(#{ <<"ok">> := false, <<"error">> := Error }, _Opts) -> + {error, Error}; +``` + +### json_to_message + +Translates a compute result -- either from a WASM execution using the + +```erlang +json_to_message(Other, _Opts) -> + {error, + #{ + <<"error">> => <<"Invalid JSON message input.">>, + <<"received">> => Other + } + }. +``` + +### safe_to_id + +```erlang +safe_to_id(<<>>) -> <<>>; +``` + +### safe_to_id + +```erlang +safe_to_id(ID) -> hb_util:human_id(ID). +``` + +### maybe_list_to_binary + +```erlang +maybe_list_to_binary(List) when is_list(List) -> + list_to_binary(List); +``` + +### maybe_list_to_binary + +```erlang +maybe_list_to_binary(Bin) -> + Bin. +``` + +### header_case_string + +```erlang +header_case_string(Key) -> + NormKey = hb_ao:normalize_key(Key), + Words = string:lexemes(NormKey, "-"), + TitleCaseWords = + lists:map( + fun binary_to_list/1, + lists:map( + fun string:titlecase/1, + Words + ) + ), + TitleCaseKey = list_to_binary(string:join(TitleCaseWords, "-")), + TitleCaseKey. +``` + +### results + +Read the computed results out of the WASM environment, assuming that + +```erlang +results(M1, M2, Opts) -> + Prefix = dev_stack:prefix(M1, M2, Opts), + Type = hb_ao:get(<<"results/", Prefix/binary, "/type">>, M1, Opts), + Proc = hb_ao:get(<<"process">>, M1, Opts), + case hb_ao:normalize_key(Type) of + <<"error">> -> + {error, + hb_ao:set( + M1, + #{ + <<"outbox">> => undefined, + <<"results">> => + #{ + <<"body">> => <<"WASM execution error.">> + } + }, + Opts + ) + }; + <<"ok">> -> + {ok, Str} = env_read(M1, M2, Opts), + try hb_json:decode(Str) of + #{<<"ok">> := true, <<"response">> := Resp} -> + {ok, ProcessedResults} = json_to_message(Resp, Opts), + PostProcessed = postprocess_outbox(ProcessedResults, Proc, Opts), + Out = hb_ao:set( + M1, + <<"results">>, + PostProcessed, + Opts + ), + ?event(debug_iface, {results, {processed, ProcessedResults}, {out, Out}}), + {ok, Out} + catch + _:_ -> + ?event(error, {json_error, Str}), + {error, + hb_ao:set( + M1, + #{ + <<"results/outbox">> => undefined, + <<"results/body">> => + <<"JSON error parsing result output.">> + }, + Opts + ) + } + end + end. +``` + +### env_read + +Read the results out of the execution environment. + +```erlang +env_read(M1, M2, Opts) -> + Prefix = dev_stack:prefix(M1, M2, Opts), + Output = hb_ao:get(<<"results/", Prefix/binary, "/output">>, M1, Opts), + case hb_private:get(<
+ `GET /estimate?type=pre|post&body=[...]&request=RequestMessage` + `GET /price?type=pre|post&body=[...]&request=RequestMessage` ++The `body` key is used to pass either the request or response messages to the +device. The `type` key is used to specify whether the inquiry is for a request +(pre) or a response (post) object. Requests carry lists of messages that will +be executed, while responses carry the results of the execution. The `price` +key may return `infinity` if the node will not serve a user under any +circumstances. Else, the value returned by the `price` key will be passed to +the ledger device as the `amount` key. +A ledger device should implement the following keys: +
+ `POST /credit?message=PaymentMessage&request=RequestMessage` + `POST /charge?amount=PriceMessage&request=RequestMessage` + `GET /balance?request=RequestMessage` ++The `type` key is optional and defaults to `pre`. If `type` is set to `post`, +the charge must be applied to the ledger, whereas the `pre` type is used to +check whether the charge would succeed before execution. + +--- + +## Exported Functions + +- `balance/3` +- `request/3` +- `response/3` + +--- + +### request + +The HyperBEAM core payment ledger. This module allows the operator to +Estimate the cost of a transaction and decide whether to proceed with + +```erlang +request(State, Raw, NodeMsg) -> + PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), + LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), + Messages = hb_ao:get(<<"body">>, Raw, NodeMsg#{ hashpath => ignore }), + Request = hb_ao:get(<<"request">>, Raw, NodeMsg), + IsChargable = is_chargable_req(Request, NodeMsg), + ?event(payment, + {preprocess_with_devices, + PricingDevice, + LedgerDevice, + {chargable, IsChargable} + } + ), + case {IsChargable, (PricingDevice =/= false) and (LedgerDevice =/= false)} of + {false, _} -> + ?event(payment, non_chargable_route), + {ok, #{ <<"body">> => Messages }}; + {true, false} -> + ?event(payment, {p4_pre_pricing_response, {error, <<"infinity">>}}), + {ok, #{ <<"body">> => Messages }}; + {true, true} -> + PricingMsg = State#{ <<"device">> => PricingDevice }, + LedgerMsg = State#{ <<"device">> => LedgerDevice }, + PricingReq = #{ + <<"path">> => <<"estimate">>, + <<"request">> => Request, + <<"body">> => Messages + }, + ?event({p4_pricing_request, {devmsg, PricingMsg}, {req, PricingReq}}), + case hb_ao:resolve(PricingMsg, PricingReq, NodeMsg) of + {ok, <<"infinity">>} -> + % The device states that under no circumstances should we + % proceed with the request. +``` + +### response + +Postprocess the request after it has been fulfilled. + +```erlang +response(State, RawResponse, NodeMsg) -> + PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), + LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), + Response = + hb_ao:get( + <<"body">>, + RawResponse, + NodeMsg#{ hashpath => ignore } + ), + Request = hb_ao:get(<<"request">>, RawResponse, NodeMsg), + ?event(payment, {post_processing_with_devices, PricingDevice, LedgerDevice}), + ?event({response_hook, {request, Request}, {response, Response}}), + case ((PricingDevice =/= false) and (LedgerDevice =/= false)) andalso + is_chargable_req(Request, NodeMsg) of + false -> + {ok, #{ <<"body">> => Response }}; + true -> + PricingMsg = State#{ <<"device">> => PricingDevice }, + LedgerMsg = State#{ <<"device">> => LedgerDevice }, + PricingReq = #{ + <<"path">> => <<"price">>, + <<"request">> => Request, + <<"body">> => Response + }, + ?event({post_pricing_request, PricingReq}), + PricingRes = + case hb_ao:resolve(PricingMsg, PricingReq, NodeMsg) of + {error, _Error} -> + % The pricing device is unable to give us a cost for + % the request, so we try to estimate it instead. +``` + +### balance + +Get the balance of a user in the ledger. + +```erlang +balance(_, Req, NodeMsg) -> + case dev_hook:find(<<"request">>, NodeMsg) of + [] -> + {error, <<"No request hook found.">>}; + [Handler] -> + LedgerDevice = + hb_ao:get(<<"ledger-device">>, Handler, false, NodeMsg), + LedgerMsg = Handler#{ <<"device">> => LedgerDevice }, + LedgerReq = #{ + <<"path">> => <<"balance">>, + <<"request">> => Req + }, + ?event({ledger_message, {ledger_msg, LedgerMsg}}), + case hb_ao:resolve(LedgerMsg, LedgerReq, NodeMsg) of + {ok, Balance} -> + {ok, Balance}; + {error, Error} -> + {error, Error} + end + end. +``` + +### is_chargable_req + +The node operator may elect to make certain routes non-chargable, using + +```erlang +is_chargable_req(Req, NodeMsg) -> + NonChargableRoutes = + hb_opts:get( + p4_non_chargable_routes, + ?DEFAULT_NON_CHARGABLE_ROUTES, + NodeMsg + ), + Matches = + dev_router:match( + #{ <<"routes">> => NonChargableRoutes }, + Req, + NodeMsg + ), + ?event( + { + is_chargable, + {non_chargable_routes, NonChargableRoutes}, + {req, Req}, + {matches, Matches} + } + ), + case Matches of + {error, no_matching_route} -> true; + _ -> false + end. +``` + +### test_opts + +```erlang +test_opts(Opts) -> + test_opts(Opts, <<"faff@1.0">>). +``` + +### test_opts + +```erlang +test_opts(Opts, PricingDev) -> + test_opts(Opts, PricingDev, <<"faff@1.0">>). +``` + +### test_opts + +```erlang +test_opts(Opts, PricingDev, LedgerDev) -> + ProcessorMsg = + #{ + <<"device">> => <<"p4@1.0">>, + <<"pricing-device">> => PricingDev, + <<"ledger-device">> => LedgerDev + }, + Opts#{ + on => #{ + <<"request">> => ProcessorMsg, + <<"response">> => ProcessorMsg + } + }. +``` + +### faff_test + +Simple test of p4's capabilities with the `faff@1.0` device. + +```erlang +faff_test() -> + GoodWallet = ar_wallet:new(), + BadWallet = ar_wallet:new(), + Node = hb_http_server:start_node( + test_opts( + #{ + faff_allow_list => + [hb_util:human_id(ar_wallet:to_address(GoodWallet))] + } + ) + ), + Req = #{ + <<"path">> => <<"/greeting">>, + <<"greeting">> => <<"Hello, world!">> + }, + GoodSignedReq = hb_message:commit(Req, GoodWallet), + ?event({req, GoodSignedReq}), + BadSignedReq = hb_message:commit(Req, BadWallet), + ?event({req, BadSignedReq}), + {ok, Res} = hb_http:get(Node, GoodSignedReq, #{}), + ?event(payment, {res, Res}), + ?assertEqual(<<"Hello, world!">>, Res), + ?assertMatch({error, _}, hb_http:get(Node, BadSignedReq, #{})). +``` + +### non_chargable_route_test + +Test that a non-chargable route is not charged for. + +```erlang +non_chargable_route_test() -> + Wallet = ar_wallet:new(), + Processor = + #{ + <<"device">> => <<"p4@1.0">>, + <<"ledger-device">> => <<"simple-pay@1.0">>, + <<"pricing-device">> => <<"simple-pay@1.0">> + }, + Node = hb_http_server:start_node( + #{ + p4_non_chargable_routes => + [ + #{ <<"template">> => <<"/~p4@1.0/balance">> }, + #{ <<"template">> => <<"/~meta@1.0/*/*">> } + ], + on => #{ + <<"request">> => Processor, + <<"response">> => Processor + }, + operator => hb:address() + } + ), + Req = #{ + <<"path">> => <<"/~p4@1.0/balance">> + }, + GoodSignedReq = hb_message:commit(Req, Wallet), + Res = hb_http:get(Node, GoodSignedReq, #{}), + ?event({res1, Res}), + ?assertMatch({ok, 0}, Res), + Req2 = #{ <<"path">> => <<"/~meta@1.0/info/operator">> }, + GoodSignedReq2 = hb_message:commit(Req2, Wallet), + Res2 = hb_http:get(Node, GoodSignedReq2, #{}), + ?event({res2, Res2}), + OperatorAddress = hb_util:human_id(hb:address()), + ?assertEqual({ok, OperatorAddress}, Res2), + Req3 = #{ <<"path">> => <<"/~scheduler@1.0">> }, + BadSignedReq3 = hb_message:commit(Req3, Wallet), + Res3 = hb_http:get(Node, BadSignedReq3, #{}), + ?event({res3, Res3}), + ?assertMatch({error, _}, Res3). +``` + +### hyper_token_ledger_test_ + +Ensure that Lua scripts can be used as pricing and ledger devices. Our + +```erlang +hyper_token_ledger_test_() -> + {timeout, 60, fun hyper_token_ledger/0}. +``` + +### hyper_token_ledger + +```erlang +hyper_token_ledger() -> + % Create the wallets necessary and read the files containing the scripts. +``` + +--- + +*Generated from [dev_p4.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_p4.erl)* diff --git a/docs/book/src/dev_patch.erl.md b/docs/book/src/dev_patch.erl.md new file mode 100644 index 000000000..7fc36cc2e --- /dev/null +++ b/docs/book/src/dev_patch.erl.md @@ -0,0 +1,288 @@ +# dev_patch + +[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_patch.erl) + +A device that can be used to reorganize a message: Moving data from +one path inside it to another. This device's function runs in two modes: +1. When using `all` to move all data at the path given in `from` to the + path given in `to`. +2. When using `patches` to move all submessages in the source to the target, + _if_ they have a `method` key of `PATCH` or a `device` key of `patch@1.0`. +Source and destination paths may be prepended by `base:` or `req:` keys to +indicate that they are relative to either of the message's that the +computation is being performed on. +The search order for finding the source and destination keys is as follows, +where `X` is either `from` or `to`: +1. The `patch-X` key of the execution message. +2. The `X` key of the execution message. +3. The `patch-X` key of the request message. +4. The `X` key of the request message. +Additionally, this device implements the standard computation device keys, +allowing it to be used as an element of an execution stack pipeline, etc. + +--- + +## Exported Functions + +- `all/3` +- `compute/3` +- `init/3` +- `normalize/3` +- `patches/3` +- `snapshot/3` + +--- + +### init + +A device that can be used to reorganize a message: Moving data from +Necessary hooks for compliance with the `execution-device` standard. + +```erlang +init(Msg1, _Msg2, _Opts) -> {ok, Msg1}. +``` + +### normalize + +A device that can be used to reorganize a message: Moving data from +Necessary hooks for compliance with the `execution-device` standard. + +```erlang +normalize(Msg1, _Msg2, _Opts) -> {ok, Msg1}. +``` + +### snapshot + +A device that can be used to reorganize a message: Moving data from +Necessary hooks for compliance with the `execution-device` standard. + +```erlang +snapshot(Msg1, _Msg2, _Opts) -> {ok, Msg1}. +``` + +### compute + +A device that can be used to reorganize a message: Moving data from +Necessary hooks for compliance with the `execution-device` standard. +Get the value found at the `patch-from` key of the message, or the + +```erlang +compute(Msg1, Msg2, Opts) -> patches(Msg1, Msg2, Opts). +``` + +### all + +A device that can be used to reorganize a message: Moving data from +Necessary hooks for compliance with the `execution-device` standard. +Get the value found at the `patch-from` key of the message, or the + +```erlang +all(Msg1, Msg2, Opts) -> + move(all, Msg1, Msg2, Opts). +``` + +### patches + +Find relevant `PATCH` messages in the given source key of the execution + +```erlang +patches(Msg1, Msg2, Opts) -> + move(patches, Msg1, Msg2, Opts). +``` + +### move + +Unified executor for the `all` and `patches` modes. + +```erlang +move(Mode, Msg1, Msg2, Opts) -> + maybe + % Find the input paths. +``` + +### uninitialized_patch_test + +```erlang +uninitialized_patch_test() -> + InitState = #{ + <<"device">> => <<"patch@1.0">>, + <<"results">> => #{ + <<"outbox">> => #{ + <<"1">> => #{ + <<"method">> => <<"PATCH">>, + <<"prices">> => #{ + <<"apple">> => 100, + <<"banana">> => 200 + } + }, + <<"2">> => #{ + <<"method">> => <<"GET">>, + <<"prices">> => #{ + <<"apple">> => 1000 + } + } + } + }, + <<"other-message">> => <<"other-value">>, + <<"patch-to">> => <<"/">>, + <<"patch-from">> => <<"/results/outbox">> + }, + {ok, ResolvedState} = + hb_ao:resolve( + InitState, + <<"compute">>, + #{} + ), + ?event({resolved_state, ResolvedState}), + ?assertEqual( + 100, + hb_ao:get(<<"prices/apple">>, ResolvedState, #{}) + ), + ?assertMatch( + not_found, + hb_ao:get(<<"results/outbox/1">>, ResolvedState, #{}) + ). +``` + +### patch_to_submessage_test + +```erlang +patch_to_submessage_test() -> + InitState = #{ + <<"device">> => <<"patch@1.0">>, + <<"results">> => #{ + <<"outbox">> => #{ + <<"1">> => + hb_message:commit(#{ + <<"method">> => <<"PATCH">>, + <<"prices">> => #{ + <<"apple">> => 100, + <<"banana">> => 200 + } + }, + hb:wallet() + ) + } + }, + <<"state">> => #{ + <<"prices">> => #{ + <<"apple">> => 1000 + } + }, + <<"other-message">> => <<"other-value">>, + <<"patch-to">> => <<"/state">>, + <<"patch-from">> => <<"/results/outbox">> + }, + {ok, ResolvedState} = + hb_ao:resolve( + InitState, + <<"compute">>, + #{} + ), + ?event({resolved_state, ResolvedState}), + ?assertEqual( + 100, + hb_ao:get(<<"state/prices/apple">>, ResolvedState, #{}) + ). +``` + +### all_mode_test + +```erlang +all_mode_test() -> + InitState = #{ + <<"device">> => <<"patch@1.0">>, + <<"input">> => #{ + <<"zones">> => #{ + <<"1">> => #{ + <<"method">> => <<"PATCH">>, + <<"prices">> => #{ + <<"apple">> => 100, + <<"banana">> => 200 + } + }, + <<"2">> => #{ + <<"method">> => <<"GET">>, + <<"prices">> => #{ + <<"orange">> => 300 + } + } + } + }, + <<"state">> => #{ + <<"prices">> => #{ + <<"apple">> => 1000 + } + } + }, + {ok, ResolvedState} = + hb_ao:resolve( + InitState, + #{ + <<"path">> => <<"all">>, + <<"patch-to">> => <<"/state">>, + <<"patch-from">> => <<"/input/zones">> + }, + #{} + ), + ?event({resolved_state, ResolvedState}), + ?assertEqual( + 100, + hb_ao:get(<<"state/1/prices/apple">>, ResolvedState, #{}) + ), + ?assertEqual( + 300, + hb_ao:get(<<"state/2/prices/orange">>, ResolvedState, #{}) + ), + ?assertEqual( + not_found, + hb_ao:get(<<"input/zones">>, ResolvedState, #{}) + ). +``` + +### req_prefix_test + +```erlang +req_prefix_test() -> + BaseMsg = #{ + <<"device">> => <<"patch@1.0">>, + <<"state">> => #{ + <<"prices">> => #{ + <<"apple">> => 1000 + } + } + }, + ReqMsg = #{ + <<"path">> => <<"all">>, + <<"patch-from">> => <<"req:/results/outbox/1">>, + <<"patch-to">> => <<"/state">>, + <<"results">> => #{ + <<"outbox">> => #{ + <<"1">> => #{ + <<"method">> => <<"PATCH">>, + <<"prices">> => #{ + <<"apple">> => 100, + <<"banana">> => 200 + } + } + } + } + }, + {ok, ResolvedState} = hb_ao:resolve(BaseMsg, ReqMsg, #{}), + ?event({resolved_state, ResolvedState}), + ?assertEqual( + 100, + hb_ao:get(<<"state/prices/apple">>, ResolvedState, #{}) + ), + ?assertEqual( + 200, + hb_ao:get(<<"state/prices/banana">>, ResolvedState, #{}) + ), + ?assertEqual( + not_found, + hb_ao:get(<<"results/outbox/1">>, ResolvedState, #{}) +``` + +--- + +*Generated from [dev_patch.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_patch.erl)* diff --git a/docs/book/src/dev_poda.erl.md b/docs/book/src/dev_poda.erl.md new file mode 100644 index 000000000..27df7947a --- /dev/null +++ b/docs/book/src/dev_poda.erl.md @@ -0,0 +1,298 @@ +# dev_poda + +[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_poda.erl) + +A simple exemplar decentralized proof of authority consensus algorithm +A simple exemplar decentralized proof of authority consensus algorithm +for AO processes. This device is split into two flows, spanning three +actions. +Execution flow: +1. Initialization. +2. Validation of incoming messages before execution. +Commitment flow: +1. Adding commitments to results, either on a CU or MU. + +--- + +## Exported Functions + +- `execute/3` +- `init/2` +- `is_user_signed/1` +- `push/3` + +--- + +### init + +A simple exemplar decentralized proof of authority consensus algorithm + +```erlang +init(S, Params) -> + {ok, S, extract_opts(Params)}. +``` + +### extract_opts + +```erlang +extract_opts(Params) -> + Authorities = + lists:filtermap( + fun({<<"authority">>, Addr}) -> {true, Addr}; + (_) -> false end, + Params + ), + {_, RawQuorum} = lists:keyfind(<<"quorum">>, 1, Params), + Quorum = binary_to_integer(RawQuorum), + ?event({poda_authorities, Authorities}), + #{ + authorities => Authorities, + quorum => Quorum + }. +``` + +### execute + +```erlang +execute(Outer = #tx { data = #{ <<"body">> := Msg } }, S = #{ <<"pass">> := 1 }, Opts) -> + case is_user_signed(Msg) of + true -> + {ok, S}; + false -> + case validate(Msg, Opts) of + true -> + ?event({poda_validated, ok}), + % Add the validations to the VFS. +``` + +### execute + +```erlang +execute(_M, S = #{ <<"pass">> := 3, <<"results">> := _Results }, _Opts) -> + {ok, S}; +``` + +### execute + +```erlang +execute(_M, S, _Opts) -> + {ok, S}. +``` + +### validate + +```erlang +validate(Msg, Opts) -> + validate_stage(1, Msg, Opts). +``` + +### validate_stage + +```erlang +validate_stage(1, Msg, Opts) when is_record(Msg, tx) -> + validate_stage(1, Msg#tx.data, Opts); +``` + +### validate_stage + +```erlang +validate_stage(1, #{ <<"commitments">> := Commitments, <<"body">> := Content }, Opts) -> + validate_stage(2, Commitments, Content, Opts); +``` + +### validate_stage + +```erlang +validate_stage(1, _M, _Opts) -> {false, <<"Required PoDA messages missing">>}. +``` + +### validate_stage + +```erlang +validate_stage(2, #tx { data = Commitments }, Content, Opts) -> + validate_stage(2, Commitments, Content, Opts); +``` + +### validate_stage + +```erlang +validate_stage(2, Commitments, Content, Opts) -> + % Ensure that all commitments are valid and signed by a + % trusted authority. +``` + +### validate_stage + +```erlang +validate_stage(3, Content, Commitments, Opts = #{ <<"quorum">> := Quorum }) -> + Validations = + lists:filter( + fun({_, Comm}) -> validate_commitment(Content, Comm, Opts) end, + hb_maps:to_list(Commitments, Opts) + ), + ?event({poda_validations, length(Validations)}), + case length(Validations) >= Quorum of + true -> + ?event({poda_quorum_reached, length(Validations)}), + true; + false -> {false, <<"Not enough validations">>} + end. +``` + +### validate_commitment + +```erlang +validate_commitment(Msg, Comm, Opts) -> + MsgID = hb_util:encode(ar_bundles:id(Msg, unsigned)), + AttSigner = hb_util:encode(ar_bundles:signer(Comm)), + ?event({poda_commitment, {signer, AttSigner, hb_maps:get(authorities, Opts, undefined, Opts)}, {msg_id, MsgID}}), + ValidSigner = lists:member(AttSigner, hb_maps:get(authorities, Opts, undefined, Opts)), + ValidSignature = ar_bundles:verify_item(Comm), + RelevantMsg = ar_bundles:id(Comm, unsigned) == MsgID orelse + (lists:keyfind(<<"commitment-for">>, 1, Comm#tx.tags) + == {<<"commitment-for">>, MsgID}) orelse + ar_bundles:member(ar_bundles:id(Msg, unsigned), Comm), + case ValidSigner and ValidSignature and RelevantMsg of + false -> + ?event({poda_commitment_invalid, + {commitment, ar_bundles:id(Comm, signed)}, + {signer, AttSigner}, + {valid_signer, ValidSigner}, + {valid_signature, ValidSignature}, + {relevant_msg, RelevantMsg}} + ), + false; + true -> true + end. +``` + +### return_error + +```erlang +return_error(S = #{ <<"wallet">> := Wallet }, Reason) -> + ?event({poda_return_error, Reason}), + ?debug_wait(10000), + {skip, S#{ + results => #{ + <<"/outbox">> => + ar_bundles:sign_item( + #tx{ + data = Reason, + tags = [{<<"error">>, <<"PoDA">>}] + }, + Wallet + ) + } + }}. +``` + +### is_user_signed + +Determines if a user committed + +```erlang +is_user_signed(#tx { data = #{ <<"body">> := Msg } }) -> + ?no_prod(use_real_commitment_detection), + lists:keyfind(<<"from-process">>, 1, Msg#tx.tags) == false; +``` + +### is_user_signed + +Determines if a user committed + +```erlang +is_user_signed(_) -> true. +%%% Commitment flow: Adding commitments to results. +``` + +### push + +Hook used by the MU pathway (currently) to add commitments to an + +```erlang +push(_Item, S = #{ <<"results">> := ResultsMsg }, Opts) -> + NewRes = commit_to_results(ResultsMsg, S, Opts), + {ok, S#{ <<"results">> => NewRes }}. +``` + +### commit_to_results + +Hook used by the MU pathway (currently) to add commitments to an + +```erlang +commit_to_results(Msg, S, Opts) -> + case is_map(Msg#tx.data) of + true -> + % Add commitments to the outbox and spawn items. +``` + +### add_commitments + +```erlang +add_commitments(NewMsg, S = #{ <<"assignment">> := Assignment, <<"store">> := _Store, <<"logger">> := _Logger, <<"wallet">> := Wallet }, Opts) -> + Process = find_process(NewMsg, S), + case is_record(Process, tx) andalso lists:member({<<"device">>, <<"PODA">>}, Process#tx.tags) of + true -> + #{ <<"authorities">> := InitAuthorities, <<"quorum">> := Quorum } = + extract_opts(Process#tx.tags), + ?event({poda_push, InitAuthorities, Quorum}), + % Aggregate validations from other nodes. +``` + +### pfiltermap + +Helper function for parallel execution of commitment + +```erlang +pfiltermap(Pred, List) -> + Parent = self(), + Pids = lists:map(fun(X) -> + spawn_monitor(fun() -> + Result = {X, Pred(X)}, + ?event({pfiltermap, sending_result, self()}), + Parent ! {self(), Result} + end) + end, List), + ?event({pfiltermap, waiting_for_results, Pids}), + [ + Res + || + {true, Res} <- + lists:map(fun({Pid, Ref}) -> + receive + {Pid, {_Item, Result}} -> + ?event({pfiltermap, received_result, Pid}), + Result; + % Handle crashes as filterable events + {'DOWN', Ref, process, Pid, _Reason} -> + ?event({pfiltermap, crashed, Pid}), + false; + Other -> + ?event({pfiltermap, unexpected_message, Other}), + false + end + end, Pids) + ]. +``` + +### find_process + +Find the process that this message is targeting, in order to + +```erlang +find_process(Item, #{ <<"logger">> := _Logger, <<"store">> := Store }) -> + case Item#tx.target of + X when X =/= <<>> -> + ?event({poda_find_process, hb_util:id(Item#tx.target)}), + {ok, Proc} = hb_cache:read(Store, hb_util:id(Item#tx.target)), + Proc; + _ -> + case lists:keyfind(<<"type">>, 1, Item#tx.tags) of + {<<"type">>, <<"process">>} -> Item; + _ -> process_not_specified + end +``` + +--- + +*Generated from [dev_poda.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_poda.erl)* diff --git a/docs/book/src/dev_process.erl.md b/docs/book/src/dev_process.erl.md new file mode 100644 index 000000000..76b2f0283 --- /dev/null +++ b/docs/book/src/dev_process.erl.md @@ -0,0 +1,1221 @@ +# dev_process + +[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_process.erl) + +This module contains the device implementation of AO processes +in AO-Core. The core functionality of the module is in 'routing' requests +for different functionality (scheduling, computing, and pushing messages) +to the appropriate device. This is achieved by swapping out the device +of the process message with the necessary component in order to run the +execution, then swapping it back before returning. Computation is supported +as a stack of devices, customizable by the user, while the scheduling +device is (by default) a single device. +This allows the devices to share state as needed. Additionally, after each +computation step the device caches the result at a path relative to the +process definition itself, such that the process message's ID can act as an +immutable reference to the process's growing list of interactions. See +`dev_process_cache` for details. +The external API of the device is as follows: +
+GET /ID/Schedule: Returns the messages in the schedule +POST /ID/Schedule: Adds a message to the schedule +GET /ID/Compute/[IDorSlotNum]: Returns the state of the process after + applying a message +GET /ID/Now: Returns the `/Results` key of the latest + computed message ++An example process definition will look like this: +
+ Device: Process/1.0 + Scheduler-Device: Scheduler/1.0 + Execution-Device: Stack/1.0 + Execution-Stack: "Scheduler/1.0", "Cron/1.0", "WASM/1.0", "PoDA/1.0" + Cron-Frequency: 10-Minutes + WASM-Image: WASMImageID + PoDA: + Device: PoDA/1.0 + Authority: A + Authority: B + Authority: C + Quorum: 2 ++Runtime options: + Cache-Frequency: The number of assignments that will be computed + before the full (restorable) state should be cached. + Cache-Keys: A list of the keys that should be cached for all + assignments, in addition to `/Results`. + +--- + +## Exported Functions + +- `as_process/2` +- `as/3` +- `compute/3` +- `dev_test_process/0` +- `do_test_restore/0` +- `ensure_process_key/2` +- `info/1` +- `init/0` +- `now/3` +- `process_id/3` +- `push/3` +- `schedule_aos_call/2` +- `schedule_aos_call/3` +- `schedule/3` +- `slot/3` +- `snapshot/3` +- `test_aos_process/0` +- `test_aos_process/1` +- `test_wasm_process/1` + +--- + +### info + +This module contains the device implementation of AO processes +When the info key is called, we should return the process exports. + +```erlang +info(_Msg1) -> + #{ + worker => fun dev_process_worker:server/3, + grouper => fun dev_process_worker:group/3, + await => fun dev_process_worker:await/5, + excludes => [ + <<"test">>, + <<"init">>, + <<"ping_ping_script">>, + <<"schedule_aos_call">>, + <<"test_aos_process">>, + <<"dev_test_process">>, + <<"test_wasm_process">> + ] + }. +``` + +### as + +Return the process state with the device swapped out for the device + +```erlang +as(RawMsg1, Msg2, Opts) -> + {ok, Msg1} = ensure_loaded(RawMsg1, Msg2, Opts), + Key = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, Msg2}, <<"as">>}, + {{as, <<"message@1.0">>, Msg2}, <<"as-device">>} + ], + <<"execution">>, + Opts + ), + {ok, + hb_util:deep_merge( + ensure_process_key(Msg1, Opts), + #{ + <<"device">> => + hb_maps:get( + << Key/binary, "-device">>, + Msg1, + default_device(Msg1, Key, Opts), + Opts + ), + % Configure input prefix for proper message routing within the + % device + <<"input-prefix">> => + case hb_maps:get(<<"input-prefix">>, Msg1, not_found, Opts) of + not_found -> <<"process">>; + Prefix -> Prefix + end, + % Configure output prefixes for result organization + <<"output-prefixes">> => + hb_maps:get( + <
+ curl /~relay@.1.0/call?method=GET?0.path=https://www.arweave.net/ ++ +--- + +## Exported Functions + +- `call/3` +- `cast/3` +- `request/3` + +--- + +### call + +This module implements the relay device, which is responsible for +Execute a `call` request using a node's routes. + +```erlang +call(M1, RawM2, Opts) -> + ?event({relay_call, {m1, M1}, {raw_m2, RawM2}}), + {ok, BaseTarget} = hb_message:find_target(M1, RawM2, Opts), + ?event({relay_call, {message_to_relay, BaseTarget}}), + RelayPath = + hb_ao:get_first( + [ + {M1, <<"path">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"path">>}, + {RawM2, <<"relay-path">>}, + {M1, <<"relay-path">>} + ], + Opts + ), + RelayDevice = + hb_ao:get_first( + [ + {M1, <<"relay-device">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"relay-device">>}, + {RawM2, <<"relay-device">>} + ], + Opts + ), + RelayPeer = + hb_ao:get_first( + [ + {M1, <<"peer">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"peer">>}, + {RawM2, <<"peer">>} + ], + Opts + ), + RelayMethod = + hb_ao:get_first( + [ + {M1, <<"method">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"method">>}, + {RawM2, <<"relay-method">>}, + {M1, <<"relay-method">>}, + {RawM2, <<"method">>} + ], + Opts + ), + RelayBody = + hb_ao:get_first( + [ + {M1, <<"body">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"body">>}, + {RawM2, <<"relay-body">>}, + {M1, <<"relay-body">>}, + {RawM2, <<"body">>} + ], + Opts + ), + Commit = + hb_ao:get_first( + [ + {{as, <<"message@1.0">>, BaseTarget}, <<"commit-request">>}, + {RawM2, <<"relay-commit-request">>}, + {M1, <<"relay-commit-request">>}, + {RawM2, <<"commit-request">>}, + {M1, <<"commit-request">>} + ], + false, + Opts + ), + TargetMod1 = + if RelayBody == not_found -> BaseTarget; + true -> BaseTarget#{<<"body">> => RelayBody} + end, + TargetMod2 = + TargetMod1#{ + <<"method">> => RelayMethod, + <<"path">> => RelayPath + }, + TargetMod3 = + case RelayDevice of + not_found -> hb_maps:without([<<"device">>], TargetMod2); + _ -> TargetMod2#{<<"device">> => RelayDevice} + end, + TargetMod4 = + case Commit of + true -> + case hb_opts:get(relay_allow_commit_request, false, Opts) of + true -> + ?event(debug_relay, {recommitting, TargetMod3}, Opts), + Committed = hb_message:commit(TargetMod3, Opts), + ?event(debug_relay, {relay_call, {committed, Committed}}, Opts), + true = hb_message:verify(Committed, all), + Committed; + false -> + throw(relay_commit_request_not_allowed) + end; + false -> TargetMod3 + end, + ?event(debug_relay, {relay_call, {without_http_params, TargetMod3}}), + ?event(debug_relay, {relay_call, {with_http_params, TargetMod4}}), + true = hb_message:verify(TargetMod4), + ?event(debug_relay, {relay_call, {verified, true}}), + Client = + case hb_maps:get(<<"http-client">>, BaseTarget, not_found, Opts) of + not_found -> hb_opts:get(relay_http_client, Opts); + RequestedClient -> RequestedClient + end, + % Let `hb_http:request/2' handle finding the peer and dispatching the + % request, unless the peer is explicitly given. +``` + +### cast + +Execute a request in the same way as `call/3`, but asynchronously. Always +Preprocess a request to check if it should be relayed to a different node. + +```erlang +cast(M1, M2, Opts) -> + spawn(fun() -> call(M1, M2, Opts) end), + {ok, <<"OK">>}. +``` + +### request + +Execute a request in the same way as `call/3`, but asynchronously. Always +Preprocess a request to check if it should be relayed to a different node. + +```erlang +request(_Msg1, Msg2, Opts) -> + {ok, + #{ + <<"body">> => + [ + #{ <<"device">> => <<"relay@1.0">> }, + #{ + <<"path">> => <<"call">>, + <<"target">> => <<"body">>, + <<"body">> => + hb_ao:get(<<"request">>, Msg2, Opts#{ hashpath => ignore }) + } + ] + } + }. +``` + +### call_get_test + +```erlang +call_get_test() -> + application:ensure_all_started([hb]), + {ok, #{<<"body">> := Body}} = + hb_ao:resolve( + #{ + <<"device">> => <<"relay@1.0">>, + <<"method">> => <<"GET">>, + <<"path">> => <<"https://www.google.com/">> + }, + <<"call">>, + #{ protocol => http2 } + ), + ?assertEqual(true, byte_size(Body) > 10_000). +``` + +### relay_nearest_test + +```erlang +relay_nearest_test() -> + Peer1 = hb_http_server:start_node(#{ priv_wallet => W1 = ar_wallet:new() }), + Peer2 = hb_http_server:start_node(#{ priv_wallet => W2 = ar_wallet:new() }), + Address1 = hb_util:human_id(ar_wallet:to_address(W1)), + Address2 = hb_util:human_id(ar_wallet:to_address(W2)), + Peers = [Address1, Address2], + Node = + hb_http_server:start_node(Opts = #{ + store => hb_opts:get(store), + priv_wallet => ar_wallet:new(), + routes => [ + #{ + <<"template">> => <<"/.*">>, + <<"strategy">> => <<"Nearest">>, + <<"nodes">> => [ + #{ + <<"prefix">> => Peer1, + <<"wallet">> => Address1 + }, + #{ + <<"prefix">> => Peer2, + <<"wallet">> => Address2 + } + ] + } + ] + }), + {ok, RelayRes} = + hb_http:get( + Node, + <<"/~relay@1.0/call?relay-path=/~meta@1.0/info">>, + Opts#{ http_only_result => false } + ), + ?event( + {relay_res, + {response, RelayRes}, + {signer, hb_message:signers(RelayRes, Opts)}, + {peers, Peers} + } + ), + HasValidSigner = + lists:any( + fun(Peer) -> + lists:member(Peer, hb_message:signers(RelayRes, Opts)) + end, + Peers + ), + ?assert(HasValidSigner). +``` + +### commit_request_test + +Test that a `relay@1.0/call` correctly commits requests as specified. + +```erlang +commit_request_test() -> + Port = 10000 + rand:uniform(10000), + Wallet = ar_wallet:new(), + Executor = + hb_http_server:start_node( + #{ + port => Port, + force_signed_requests => true + } + ), + Node = + hb_http_server:start_node(#{ + priv_wallet => Wallet, + relay_allow_commit_request => true, + routes => + [ + #{ + <<"template">> => <<"/test-key">>, + <<"strategy">> => <<"Nearest">>, + <<"nodes">> => [ + #{ + <<"wallet">> => hb_util:human_id(Wallet), + <<"prefix">> => Executor + } + ] + } + ], + on => #{ + <<"request">> => + #{ + <<"device">> => <<"router@1.0">>, + <<"path">> => <<"preprocess">>, + <<"commit-request">> => true + } + } + }), + {ok, Res} = + hb_http:get( + Node, + #{ + <<"path">> => <<"test-key">>, + <<"test-key">> => <<"value">> + }, + #{} + ), + ?event({res, Res}), +``` + +--- + +*Generated from [dev_relay.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_relay.erl)* diff --git a/docs/book/src/dev_router.erl.md b/docs/book/src/dev_router.erl.md new file mode 100644 index 000000000..5249f8434 --- /dev/null +++ b/docs/book/src/dev_router.erl.md @@ -0,0 +1,1450 @@ +# dev_router + +[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_router.erl) + +A device that routes outbound messages from the node to their +appropriate network recipients via HTTP. All messages are initially +routed to a single process per node, which then load-balances them +between downstream workers that perform the actual requests. +The routes for the router are defined in the `routes` key of the `Opts`, +as a precidence-ordered list of maps. The first map that matches the +message will be used to determine the route. +Multiple nodes can be specified as viable for a single route, with the +`Choose` key determining how many nodes to choose from the list (defaulting +to 1). The `Strategy` key determines the load distribution strategy, +which can be one of `Random`, `By-Base`, or `Nearest`. The route may also +define additional parallel execution parameters, which are used by the +`hb_http` module to manage control of requests. +The structure of the routes should be as follows: +
+ Node?: The node to route the message to. + Nodes?: A list of nodes to route the message to. + Strategy?: The load distribution strategy to use. + Choose?: The number of nodes to choose from the list. + Template?: A message template to match the message against, either as a + map or a path regex. ++ +--- + +## Exported Functions + +- `info/1` +- `info/3` +- `match/3` +- `preprocess/3` +- `register/3` +- `route/2` +- `route/3` +- `routes/3` + +--- + +### info + +A device that routes outbound messages from the node to their +Exported function for getting device info, controls which functions are + +```erlang +info(_) -> + #{ exports => [info, routes, route, match, register, preprocess] }. +``` + +### info + +HTTP info response providing information about this device + +```erlang +info(_Msg1, _Msg2, _Opts) -> + InfoBody = #{ + <<"description">> => <<"Router device for handling outbound message routing">>, + <<"version">> => <<"1.0">>, + <<"api">> => #{ + <<"info">> => #{ + <<"description">> => <<"Get device info">> + }, + <<"routes">> => #{ + <<"description">> => <<"Get or add routes">>, + <<"method">> => <<"GET or POST">> + }, + <<"route">> => #{ + <<"description">> => <<"Find a route for a message">>, + <<"required_params">> => #{ + <<"route-path">> => <<"Path to route">> + } + }, + <<"match">> => #{ + <<"description">> => <<"Match a message against available routes">> + }, + <<"register">> => #{ + <<"description">> => <<"Register a route with a remote router node">>, + <<"node-message">> => #{ + <<"routes">> => + [ + #{ + <<"registration-peer">> => <<"Location of the router peer">>, + <<"prefix">> => <<"Prefix for the route">>, + <<"price">> => <<"Price for the route">>, + <<"template">> => <<"Template to match the route">> + } + ] + } + }, + <<"preprocess">> => #{ + <<"description">> => <<"Preprocess a request to check if it should be relayed">> + } + } + }, + {ok, InfoBody}. +``` + +### register + +Register function that allows telling the current node to register +Device function that returns all known routes. + +```erlang +register(_M1, M2, Opts) -> + %% Extract all required parameters from options + %% These values will be used to construct the registration message + RouterOpts = hb_opts:get(router_opts, #{}, Opts), + RouterRegMsgs = + case hb_maps:get(<<"offered">>, RouterOpts, #{}, Opts) of + RegList when is_list(RegList) -> RegList; + RegMsg when is_map(RegMsg) -> [RegMsg] + end, + lists:foreach( + fun(RegMsg) -> + RouterNode = + hb_ao:get( + <<"registration-peer">>, + RegMsg, + not_found, + Opts + ), + {ok, SigOpts} = + case hb_ao:get(<<"as">>, M2, not_found, Opts) of + not_found -> {ok, Opts}; + AsID -> hb_opts:as(AsID, Opts) + end, + % Post registration request to the router node + % The message includes our route details and attestation + % for verification + {ok, Res} = + hb_http:post( + RouterNode, + <<"/~router@1.0/routes">>, + hb_message:commit( + #{ + <<"subject">> => <<"self">>, + <<"action">> => <<"register">>, + <<"route">> => RegMsg + }, + SigOpts + ), + Opts + ), + ?event({registered, {msg, M2}, {res, Res}}), + {ok, <<"Route registered.">>} + end, + RouterRegMsgs + ), + {ok, <<"Routes registered.">>}. +``` + +### routes + +Register function that allows telling the current node to register +Device function that returns all known routes. + +```erlang +routes(M1, M2, Opts) -> + ?event({routes_msg, M1, M2}), + Routes = load_routes(Opts), + ?event({routes, Routes}), + case hb_ao:get(<<"method">>, M2, Opts) of + <<"POST">> -> + RouterOpts = hb_opts:get(router_opts, #{}, Opts), + ?event(debug_route_reg, {router_opts, RouterOpts}), + case hb_maps:get(<<"registrar">>, RouterOpts, not_found, Opts) of + not_found -> + % There is no registrar; register if and only if the message + % is signed by an authorized operator. +``` + +### route + +Find the appropriate route for the given message. If we are able to + +```erlang +route(Msg, Opts) -> route(undefined, Msg, Opts). +``` + +### route + +Find the appropriate route for the given message. If we are able to + +```erlang +route(_, Msg, Opts) -> + Routes = load_routes(Opts), + R = match_routes(Msg, Routes, Opts), + ?event({find_route, {msg, Msg}, {routes, Routes}, {res, R}}), + case (R =/= no_matches) andalso hb_ao:get(<<"node">>, R, Opts) of + false -> {error, no_matches}; + Node when is_binary(Node) -> {ok, Node}; + Node when is_map(Node) -> apply_route(Msg, Node, Opts); + not_found -> + ModR = apply_routes(Msg, R, Opts), + case hb_ao:get(<<"strategy">>, R, Opts) of + not_found -> {ok, ModR}; + <<"All">> -> {ok, ModR}; + Strategy -> + ChooseN = hb_ao:get(<<"choose">>, R, 1, Opts), + % Get the first element of the path -- the `base' message + % of the request. +``` + +### load_routes + +Load the current routes for the node. Allows either explicit routes from + +```erlang +load_routes(Opts) -> + RouterOpts = hb_opts:get(router_opts, #{}, Opts), + case hb_maps:get(<<"provider">>, RouterOpts, not_found, Opts) of + not_found -> hb_opts:get(routes, [], Opts); + RoutesProvider -> + ProviderMsgs = hb_singleton:from(RoutesProvider, Opts), + ?event({<<"provider">>, ProviderMsgs}), + case hb_ao:resolve_many(ProviderMsgs, Opts) of + {ok, Routes} -> hb_cache:ensure_all_loaded(Routes, Opts); + {error, Error} -> throw({routes, routes_provider_failed, Error}) + end + end. +``` + +### extract_base + +Extract the base message ID from a request message. Produces a single + +```erlang +extract_base(#{ <<"path">> := Path }, Opts) -> + extract_base(Path, Opts); +``` + +### extract_base + +Extract the base message ID from a request message. Produces a single + +```erlang +extract_base(RawPath, Opts) when is_binary(RawPath) -> + BasePath = hb_path:hd(#{ <<"path">> => RawPath }, Opts), + case ?IS_ID(BasePath) of + true -> BasePath; + false -> + case binary:split(BasePath, [<<"\~">>, <<"?">>, <<"&">>], [global]) of + [BaseMsgID|_] when ?IS_ID(BaseMsgID) -> BaseMsgID; + _ -> hb_crypto:sha256(BasePath) + end + end. +``` + +### apply_routes + +Generate a `uri` key for each node in a route. +Apply a node map's rules for transforming the path of the message. + +```erlang +apply_routes(Msg, R, Opts) -> + Nodes = hb_ao:get(<<"nodes">>, R, Opts), + NodesWithRouteApplied = + lists:map( + fun(N) -> + ?event({apply_route, {msg, Msg}, {node, N}}), + case apply_route(Msg, N, Opts) of + {ok, URI} when is_binary(URI) -> N#{ <<"uri">> => URI }; + {ok, RMsg} -> hb_maps:merge(N, RMsg); + {error, _} -> N + end + end, + hb_util:message_to_ordered_list(Nodes, Opts) + ), + ?event({nodes_after_apply, NodesWithRouteApplied}), + R#{ <<"nodes">> => NodesWithRouteApplied }. +``` + +### apply_route + +Generate a `uri` key for each node in a route. +Apply a node map's rules for transforming the path of the message. + +```erlang +apply_route(Msg, Route, Opts) -> + % LoadedRoute = hb_cache:ensure_all_loaded(Route, Opts), + RouteOpts = hb_maps:get(<<"opts">>, Route, #{}), + {ok, #{ + <<"opts">> => RouteOpts, + <<"uri">> => + hb_util:ok( + do_apply_route( + Msg, + hb_maps:without([<<"opts">>], Route, Opts), + Opts + ) + ) + }}. +``` + +### do_apply_route + +```erlang +do_apply_route(#{ <<"route-path">> := Path }, R, Opts) -> + do_apply_route(#{ <<"path">> => Path }, R, Opts); +``` + +### do_apply_route + +```erlang +do_apply_route(#{ <<"path">> := RawPath }, #{ <<"prefix">> := RawPrefix }, Opts) -> + Path = hb_cache:ensure_loaded(RawPath, Opts), + Prefix = hb_cache:ensure_loaded(RawPrefix, Opts), + {ok, <
+It exposes the following keys for scheduling:
+ `#{ method: GET, path: <<"/info">> }` ->
+ Returns information about the scheduler.
+ `#{ method: GET, path: <<"/slot">> }` -> `slot(Msg1, Msg2, Opts)`
+ Returns the current slot for a process.
+ `#{ method: GET, path: <<"/schedule">> }` -> `get_schedule(Msg1, Msg2, Opts)`
+ Returns the schedule for a process in a cursor-traversable format.
+ ` #{ method: POST, path: <<"/schedule">> }` -> `post_schedule(Msg1, Msg2, Opts)`
+ Schedules a new message for a process, or starts a new scheduler
+ for the given message.
+
+
+---
+
+## Exported Functions
+
+- `checkpoint/1`
+- `info/0`
+- `location/3`
+- `next/3`
+- `parse_schedulers/1`
+- `router/4`
+- `schedule/3`
+- `slot/3`
+- `start/0`
+- `status/3`
+- `test_process/0`
+
+---
+
+### start
+
+A simple scheduler scheme for AO.
+Helper to ensure that the environment is started.
+
+```erlang
+start() ->
+ % We need the rocksdb backend to run for hb_cache module to work
+ application:ensure_all_started(hb),
+ <+Device -> Stack +Device-Stack/1/Name -> Add-One-Device +Device-Stack/2/Name -> Add-Two-Device ++When called with the message: +
+#{ Path = "FuncName", binary => `<<"0">>` }
+
+Will produce the output:
+
+#{ Path = "FuncName", binary => `<<"3">>` }
+{ok, #{ bin => `<<"3">>` }}
+
+In map mode, the stack will run over all the devices in the stack, and
+combine their results into a single message. Each of the devices'
+output values have a key that is the device's name in the `Device-Stack`
+(its number if the stack is a list).
+You can switch between fold and map modes by setting the `Mode` key in the
+`Msg2` to either `Fold` or `Map`, or set it globally for the stack by
+setting the `Mode` key in the `Msg1` message. The key in `Msg2` takes
+precedence over the key in `Msg1`.
+The key that is called upon the device stack is the same key that is used
+upon the devices that are contained within it. For example, in the above
+scenario we resolve FuncName on the stack, leading FuncName to be called on
+Add-One-Device and Add-Two-Device.
+A device stack responds to special statuses upon responses as follows:
+ `skip`: Skips the rest of the device stack for the current pass.
+ `pass`: Causes the stack to increment its pass number and re-execute
+ the stack from the first device, maintaining the state
+ accumulated so far. Only available in fold mode.
+In all cases, the device stack will return the accumulated state to the
+caller as the result of the call to the stack.
+The dev_stack adds additional metadata to the message in order to track
+the state of its execution as it progresses through devices. These keys
+are as follows:
+ `Stack-Pass`: The number of times the stack has reset and re-executed
+ from the first device for the current message.
+ `Input-Prefix`: The prefix that the device should use for its outputs
+ and inputs.
+ `Output-Prefix`: The device that was previously executed.
+All counters used by the stack are initialized to 1.
+Additionally, as implemented in HyperBEAM, the device stack will honor a
+number of options that are passed to it as keys in the message. Each of
+these options is also passed through to the devices contained within the
+stack during execution. These options include:
+ `Error-Strategy`: Determines how the stack handles errors from devices.
+ See `maybe_error/5` for more information.
+ `Allow-Multipass`: Determines whether the stack is allowed to automatically
+ re-execute from the first device when the `pass` tag is returned. See
+ `maybe_pass/3` for more information.
+Under-the-hood, dev_stack uses a `default` handler to resolve all calls to
+devices, aside `set/2` which it calls itself to mutate the message's `device`
+key in order to change which device is currently being executed. This method
+allows dev_stack to ensure that the message's HashPath is always correct,
+even as it delegates calls to other devices. An example flow for a `dev_stack`
+execution is as follows:
+
+ /Msg1/AlicesExcitingKey ->
+ dev_stack:execute ->
+ /Msg1/Set?device=/Device-Stack/1 ->
+ /Msg2/AlicesExcitingKey ->
+ /Msg3/Set?device=/Device-Stack/2 ->
+ /Msg4/AlicesExcitingKey
+ ... ->
+ /MsgN/Set?device=[This-Device] ->
+ returns {ok, /MsgN+1} ->
+ /MsgN+1
+
+In this example, the `device` key is mutated a number of times, but the
+resulting HashPath remains correct and verifiable.
+
+---
+
+## Exported Functions
+
+- `generate_append_device/1`
+- `info/2`
+- `input_prefix/3`
+- `output_prefix/3`
+- `prefix/3`
+- `router/4`
+
+---
+
+### info
+
+A device that contains a stack of other devices, and manages their
+
+```erlang
+info(Msg, Opts) ->
+ hb_maps:merge(
+ #{
+ handler => fun router/4,
+ excludes => [<<"set">>, <<"keys">>]
+ },
+ case hb_maps:get(<<"stack-keys">>, Msg, not_found, Opts) of
+ not_found -> #{};
+ StackKeys -> #{ exports => StackKeys }
+ end
+ ).
+```
+
+### prefix
+
+Return the default prefix for the stack.
+Return the input prefix for the stack.
+
+```erlang
+prefix(Msg1, _Msg2, Opts) ->
+ hb_ao:get(<<"output-prefix">>, {as, dev_message, Msg1}, <<"">>, Opts).
+```
+
+### input_prefix
+
+Return the default prefix for the stack.
+Return the input prefix for the stack.
+Return the output prefix for the stack.
+
+```erlang
+input_prefix(Msg1, _Msg2, Opts) ->
+ hb_ao:get(<<"input-prefix">>, {as, dev_message, Msg1}, <<"">>, Opts).
+```
+
+### output_prefix
+
+Return the default prefix for the stack.
+Return the input prefix for the stack.
+Return the output prefix for the stack.
+The device stack key router. Sends the request to `resolve_stack`,
+
+```erlang
+output_prefix(Msg1, _Msg2, Opts) ->
+ hb_ao:get(<<"output-prefix">>, {as, dev_message, Msg1}, <<"">>, Opts).
+```
+
+### router
+
+Return the default prefix for the stack.
+Return the input prefix for the stack.
+Return the output prefix for the stack.
+The device stack key router. Sends the request to `resolve_stack`,
+
+```erlang
+router(<<"keys">>, Message1, Message2, Opts) ->
+ ?event({keys_called, {msg1, Message1}, {msg2, Message2}}),
+ dev_message:keys(Message1, Opts);
+```
+
+### router
+
+Return the default prefix for the stack.
+Return the input prefix for the stack.
+Return the output prefix for the stack.
+The device stack key router. Sends the request to `resolve_stack`,
+
+```erlang
+router(Key, Message1, Message2, Opts) ->
+ case hb_path:matches(Key, <<"transform">>) of
+ true -> transformer_message(Message1, Opts);
+ false -> router(Message1, Message2, Opts)
+ end.
+```
+
+### router
+
+```erlang
+router(Message1, Message2, Opts) ->
+ ?event({router_called, {msg1, Message1}, {msg2, Message2}}),
+ Mode =
+ case hb_ao:get(<<"mode">>, Message2, not_found, Opts) of
+ not_found ->
+ hb_ao:get(
+ <<"mode">>,
+ {as, dev_message, Message1},
+ <<"Fold">>,
+ Opts
+ );
+ Msg2Mode -> Msg2Mode
+ end,
+ case Mode of
+ <<"Fold">> -> resolve_fold(Message1, Message2, Opts);
+ <<"Map">> -> resolve_map(Message1, Message2, Opts)
+ end.
+```
+
+### transformer_message
+
+Return a message which, when given a key, will transform the message
+
+```erlang
+transformer_message(Msg1, Opts) ->
+ ?event({creating_transformer, {for, Msg1}}),
+ BaseInfo = info(Msg1, Opts),
+ {ok,
+ Msg1#{
+ <<"device">> => #{
+ info =>
+ fun() ->
+ hb_maps:merge(
+ BaseInfo,
+ #{
+ handler =>
+ fun(Key, MsgX1) ->
+ transform(MsgX1, Key, Opts)
+ end
+ },
+ Opts
+ )
+ end,
+ <<"type">> => <<"stack-transformer">>
+ }
+ }
+ }.
+```
+
+### transform
+
+Return Message1, transformed such that the device named `Key` from the
+
+```erlang
+transform(Msg1, Key, Opts) ->
+ % Get the device stack message from Msg1.
+```
+
+### resolve_fold
+
+The main device stack execution engine. See the moduledoc for more
+
+```erlang
+resolve_fold(Message1, Message2, Opts) ->
+ {ok, InitDevMsg} = dev_message:get(<<"device">>, Message1, Opts),
+ StartingPassValue =
+ hb_ao:get(<<"pass">>, {as, dev_message, Message1}, unset, Opts),
+ PreparedMessage = hb_ao:set(Message1, <<"pass">>, 1, Opts),
+ case resolve_fold(PreparedMessage, Message2, 1, Opts) of
+ {ok, Raw} when not is_map(Raw) ->
+ {ok, Raw};
+ {ok, Result} ->
+ dev_message:set(
+ Result,
+ #{
+ <<"device">> => InitDevMsg,
+ <<"input-prefix">> =>
+ hb_ao:get(
+ <<"previous-input-prefix">>,
+ {as, dev_message, Result},
+ undefined,
+ Opts
+ ),
+ <<"output-prefix">> =>
+ hb_ao:get(
+ <<"previous-output-prefix">>,
+ {as, dev_message, Result},
+ undefined,
+ Opts
+ ),
+ <<"device-key">> => unset,
+ <<"device-stack-previous">> => unset,
+ <<"pass">> => StartingPassValue
+ },
+ Opts
+ );
+ Else ->
+ Else
+ end.
+```
+
+### resolve_fold
+
+```erlang
+resolve_fold(Message1, Message2, DevNum, Opts) ->
+ case transform(Message1, DevNum, Opts) of
+ {ok, Message3} ->
+ ?event({stack_execute, DevNum, {msg1, Message3}, {msg2, Message2}}),
+ case hb_ao:resolve(Message3, Message2, Opts) of
+ {ok, Message4} when is_map(Message4) ->
+ ?event({result, ok, DevNum, Message4}),
+ resolve_fold(Message4, Message2, DevNum + 1, Opts);
+ {error, not_found} ->
+ ?event({skipping_device, not_found, DevNum, Message3}),
+ resolve_fold(Message3, Message2, DevNum + 1, Opts);
+ {ok, RawResult} ->
+ ?event({returning_raw_result, RawResult}),
+ {ok, RawResult};
+ {skip, Message4} when is_map(Message4) ->
+ ?event({result, skip, DevNum, Message4}),
+ {ok, Message4};
+ {pass, Message4} when is_map(Message4) ->
+ ?event({result, pass, {dev, DevNum}, Message4}),
+ resolve_fold(
+ increment_pass(Message4, Opts),
+ Message2,
+ 1,
+ Opts
+ );
+ {error, Info} ->
+ ?event({result, error, {dev, DevNum}, Info}),
+ maybe_error(Message1, Message2, DevNum, Info, Opts);
+ Unexpected ->
+ ?event({result, unexpected, {dev, DevNum}, Unexpected}),
+ maybe_error(
+ Message1,
+ Message2,
+ DevNum,
+ {unexpected_result, Unexpected},
+ Opts
+ )
+ end;
+ not_found ->
+ ?event({execution_complete, DevNum, Message1}),
+ {ok, Message1}
+ end.
+```
+
+### resolve_map
+
+Map over the devices in the stack, accumulating the output in a single
+
+```erlang
+resolve_map(Message1, Message2, Opts) ->
+ ?event({resolving_map, {msg1, Message1}, {msg2, Message2}}),
+ DevKeys =
+ hb_ao:get(
+ <<"device-stack">>,
+ {as, dev_message, Message1},
+ Opts
+ ),
+ Res = {ok,
+ hb_maps:filtermap(
+ fun(Key, _Dev) ->
+ {ok, OrigWithDev} = transform(Message1, Key, Opts),
+ case hb_ao:resolve(OrigWithDev, Message2, Opts) of
+ {ok, Value} -> {true, Value};
+ _ -> false
+ end
+ end,
+ hb_maps:without(?AO_CORE_KEYS, hb_ao:normalize_keys(DevKeys, Opts), Opts),
+ Opts
+ )
+ },
+ Res.
+```
+
+### increment_pass
+
+Helper to increment the pass number.
+
+```erlang
+increment_pass(Message, Opts) ->
+ hb_ao:set(
+ Message,
+ #{ <<"pass">> => hb_ao:get(<<"pass">>, {as, dev_message, Message}, 1, Opts) + 1 },
+ Opts
+ ).
+```
+
+### maybe_error
+
+```erlang
+maybe_error(Message1, Message2, DevNum, Info, Opts) ->
+ case hb_opts:get(error_strategy, throw, Opts) of
+ stop ->
+ {error, {stack_call_failed, Message1, Message2, DevNum, Info}};
+ throw ->
+ erlang:raise(
+ error,
+ {device_failed,
+ {dev_num, DevNum},
+ {msg1, Message1},
+ {msg2, Message2},
+ {info, Info}
+ },
+ []
+ )
+ end.
+```
+
+### generate_append_device
+
+```erlang
+generate_append_device(Separator) ->
+ generate_append_device(Separator, ok).
+```
+
+### generate_append_device
+
+```erlang
+generate_append_device(Separator, Status) ->
+ #{
+ append =>
+ fun(M1 = #{ <<"pass">> := 3 }, _) ->
+ % Stop after 3 passes.
+```
+
+### transform_internal_call_device_test
+
+Test that the transform function can be called correctly internally
+
+```erlang
+transform_internal_call_device_test() ->
+ AppendDev = generate_append_device(<<"_">>),
+ Msg1 =
+ #{
+ <<"device">> => <<"stack@1.0">>,
+ <<"device-stack">> =>
+ #{
+ <<"1">> => AppendDev,
+ <<"2">> => <<"message@1.0">>
+ }
+ },
+ ?assertMatch(
+ <<"message@1.0">>,
+ hb_ao:get(
+ <<"device">>,
+ element(2, transform(Msg1, <<"2">>, #{}))
+ )
+ ).
+```
+
+### transform_external_call_device_test
+
+Ensure we can generate a transformer message that can be called to
+
+```erlang
+transform_external_call_device_test() ->
+ Msg1 = #{
+ <<"device">> => <<"stack@1.0">>,
+ <<"device-stack">> =>
+ #{
+ <<"make-cool">> =>
+ #{
+ info =>
+ fun() ->
+ #{
+ handler =>
+ fun(<<"keys">>, MsgX1) ->
+ ?event({test_dev_keys_called, MsgX1}),
+ {ok, hb_maps:keys(MsgX1, #{})};
+ (Key, MsgX1) ->
+ {ok, Value} =
+ dev_message:get(Key, MsgX1, #{}),
+ dev_message:set(
+ MsgX1,
+ #{ Key =>
+ << Value/binary, "-Cool">>
+ },
+ #{}
+ )
+ end
+ }
+ end,
+ <<"suffix">> => <<"-Cool">>
+ }
+ },
+ <<"value">> => <<"Super">>
+ },
+ ?assertMatch(
+ {ok, #{ <<"value">> := <<"Super-Cool">> }},
+ hb_ao:resolve(Msg1, #{
+ <<"path">> => <<"/transform/make-cool/value">>
+ }, #{})
+ ).
+```
+
+### example_device_for_stack_test
+
+```erlang
+example_device_for_stack_test() ->
+ % Test the example device that we use for later stack tests, such that
+ % we know that an error later is actually from the stack, and not from
+ % the example device.
+```
+
+### simple_stack_execute_test
+
+```erlang
+simple_stack_execute_test() ->
+ Msg = #{
+ <<"device">> => <<"stack@1.0">>,
+ <<"device-stack">> =>
+ #{
+ <<"1">> => generate_append_device(<<"!D1!">>),
+ <<"2">> => generate_append_device(<<"_D2_">>)
+ },
+ <<"result">> => <<"INIT">>
+ },
+ ?event({stack_executing, test, {explicit, Msg}}),
+ ?assertMatch(
+ {ok, #{ <<"result">> := <<"INIT!D1!2_D2_2">> }},
+ hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"2">> }, #{})
+ ).
+```
+
+### many_devices_test
+
+```erlang
+many_devices_test() ->
+ Msg = #{
+ <<"device">> => <<"stack@1.0">>,
+ <<"device-stack">> =>
+ #{
+ <<"1">> => generate_append_device(<<"+D1">>),
+ <<"2">> => generate_append_device(<<"+D2">>),
+ <<"3">> => generate_append_device(<<"+D3">>),
+ <<"4">> => generate_append_device(<<"+D4">>),
+ <<"5">> => generate_append_device(<<"+D5">>),
+ <<"6">> => generate_append_device(<<"+D6">>),
+ <<"7">> => generate_append_device(<<"+D7">>),
+ <<"8">> => generate_append_device(<<"+D8">>)
+ },
+ <<"result">> => <<"INIT">>
+ },
+ ?assertMatch(
+ {ok,
+ #{
+ <<"result">> :=
+ <<"INIT+D12+D22+D32+D42+D52+D62+D72+D82">>
+ }
+ },
+ hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"2">> }, #{})
+ ).
+```
+
+### benchmark_test
+
+```erlang
+benchmark_test() ->
+ BenchTime = 0.3,
+ Msg = #{
+ <<"device">> => <<"stack@1.0">>,
+ <<"device-stack">> =>
+ #{
+ <<"1">> => generate_append_device(<<"+D1">>),
+ <<"2">> => generate_append_device(<<"+D2">>),
+ <<"3">> => generate_append_device(<<"+D3">>),
+ <<"4">> => generate_append_device(<<"+D4">>),
+ <<"5">> => generate_append_device(<<"+D5">>)
+ },
+ <<"result">> => <<"INIT">>
+ },
+ Iterations =
+ hb_test_utils:benchmark(
+ fun() ->
+ hb_ao:resolve(Msg,
+ #{
+ <<"path">> => <<"append">>,
+ <<"bin">> => <<"2">>
+ },
+ #{}
+ ),
+ {count, 5}
+ end,
+ BenchTime
+ ),
+ hb_test_utils:benchmark_print(
+ <<"Stack:">>,
+ <<"resolutions">>,
+ Iterations,
+ BenchTime
+ ),
+ ?assert(Iterations >= 10).
+```
+
+### test_prefix_msg
+
+```erlang
+test_prefix_msg() ->
+ Dev = #{
+ prefix_set =>
+ fun(M1, M2, Opts) ->
+ In = input_prefix(M1, M2, Opts),
+ Out = output_prefix(M1, M2, Opts),
+ Key = hb_ao:get(<<"key">>, M2, Opts),
+ Value = hb_ao:get(<+ M1/Init -> + Assumes: + M1/process + M1/[Prefix]/image + Generates: + /priv/[Prefix]/instance + /priv/[Prefix]/import-resolver + Side-effects: + Creates a WASM executor loaded in memory of the HyperBEAM node. + M1/Compute -> + Assumes: + M1/priv/[Prefix]/instance + M1/priv/[Prefix]/import-resolver + M1/process + M2/message + M2/message/function OR M1/function + M2/message/parameters OR M1/parameters + Generates: + /results/[Prefix]/type + /results/[Prefix]/output + Side-effects: + Calls the WASM executor with the message and process. + M1/[Prefix]/state -> + Assumes: + M1/priv/[Prefix]/instance + Generates: + Raw binary WASM state ++ +--- + +## Exported Functions + +- `cache_wasm_image/1` +- `cache_wasm_image/2` +- `compute/3` +- `import/3` +- `info/2` +- `init/3` +- `instance/3` +- `normalize/3` +- `snapshot/3` +- `terminate/3` + +--- + +### info + +A device that executes a WASM image on messages using the Memory-64 +Export all functions aside the `instance/3` function. + +```erlang +info(_Msg1, _Opts) -> + #{ + excludes => [instance] + }. +``` + +### init + +Boot a WASM image on the image stated in the `process/image` field of + +```erlang +init(M1, M2, Opts) -> + ?event(running_init), + % Where we should read initial parameters from. +``` + +### default_import_resolver + +Take a BEAMR import call and resolve it using `hb_ao`. + +```erlang +default_import_resolver(Msg1, Msg2, Opts) -> + #{ + instance := WASM, + module := Module, + func := Func, + args := Args, + func_sig := Signature + } = Msg2, + Prefix = dev_stack:prefix(Msg1, Msg2, Opts), + {ok, Msg3} = + hb_ao:resolve( + hb_private:set( + Msg1, + #{ <
+ DevMod:ExportedFunc : Key resolution functions. All are assumed to be + device keys (thus, present in every message that + uses it) unless specified by `DevMod:info()`. + Each function takes a set of parameters + of the form `DevMod:KeyHandler(Msg1, Msg2, Opts)`. + Each of these arguments can be ommitted if not + needed. Non-exported functions are not assumed + to be device keys. + DevMod:info : Optional. Returns a map of options for the device. All + options are optional and assumed to be the defaults if + not specified. This function can accept a `Message1` as + an argument, allowing it to specify its functionality + based on a specific message if appropriate. + info/exports : Overrides the export list of the Erlang module, such that + only the functions in this list are assumed to be device + keys. Defaults to all of the functions that DevMod + exports in the Erlang environment. + info/excludes : A list of keys that should not be resolved by the device, + despite being present in the Erlang module exports list. + info/handler : A function that should be used to handle _all_ keys for + messages using the device. + info/default : A function that should be used to handle all keys that + are not explicitly implemented by the device. Defaults to + the `dev_message` device, which contains general keys for + interacting with messages. + info/default_mod : A different device module that should be used to + handle all keys that are not explicitly implemented + by the device. Defaults to the `dev_message` device. + info/grouper : A function that returns the concurrency 'group' name for + an execution. Executions with the same group name will + be executed by sending a message to the associated process + and waiting for a response. This allows you to control + concurrency of execution and to allow executions to share + in-memory state as applicable. Default: A derivation of + Msg1+Msg2. This means that concurrent calls for the same + output will lead to only a single execution. + info/worker : A function that should be run as the 'server' loop of + the executor for interactions using the device. +The HyperBEAM resolver also takes a number of runtime options that change +the way that the environment operates: +`update_hashpath`: Whether to add the `Msg2` to `HashPath` for the `Msg3`. + Default: true. +`add_key`: Whether to add the key to the start of the arguments. + Default: `+ +--- + +## Exported Functions + +- `deep_set/4` +- `find_exported_function/5` +- `force_message/2` +- `get_first/2` +- `get_first/3` +- `get/2` +- `get/3` +- `get/4` +- `info/2` +- `is_exported/4` +- `keys/1` +- `keys/2` +- `keys/3` +- `load_device/2` +- `message_to_device/2` +- `message_to_fun/3` +- `normalize_key/1` +- `normalize_key/2` +- `normalize_keys/1` +- `normalize_keys/2` +- `remove/2` +- `remove/3` +- `resolve_many/2` +- `resolve/2` +- `resolve/3` +- `set/3` +- `set/4` +- `truncate_args/2` + +--- + +### resolve + +This module is the root of the device call logic of the +Get the value of a message's key by running its associated device + +```erlang +resolve(Path, Opts) when is_binary(Path) -> + resolve(#{ <<"path">> => Path }, Opts); +``` + +### resolve + +This module is the root of the device call logic of the +Get the value of a message's key by running its associated device + +```erlang +resolve(SingletonMsg, _Opts) + when is_map(SingletonMsg), not is_map_key(<<"path">>, SingletonMsg) -> + {error, <<"Attempted to resolve a message without a path.">>}; +``` + +### resolve + +This module is the root of the device call logic of the +Get the value of a message's key by running its associated device + +```erlang +resolve(SingletonMsg, Opts) -> + resolve_many(hb_singleton:from(SingletonMsg, Opts), Opts). +``` + +### resolve + +```erlang +resolve(Msg1, Path, Opts) when not is_map(Path) -> + resolve(Msg1, #{ <<"path">> => Path }, Opts); +``` + +### resolve + +```erlang +resolve(Msg1, Msg2, Opts) -> + PathParts = hb_path:from_message(request, Msg2, Opts), + ?event(ao_core, {stage, 1, prepare_multimessage_resolution, {path_parts, PathParts}}), + MessagesToExec = [ Msg2#{ <<"path">> => Path } || Path <- PathParts ], + ?event(ao_core, {stage, 1, prepare_multimessage_resolution, {messages_to_exec, MessagesToExec}}), + resolve_many([Msg1 | MessagesToExec], Opts). +``` + +### resolve_many + +Resolve a list of messages in sequence. Take the output of the first + +```erlang +resolve_many([ID], Opts) when ?IS_ID(ID) -> + % Note: This case is necessary to place specifically here for two reasons: + % 1. It is not in `do_resolve_many' because we need to handle the case + % where a result from a prior invocation is an ID itself. We should not + % attempt to resolve such IDs further. +``` + +### resolve_many + +```erlang +resolve_many(ListMsg, Opts) when is_map(ListMsg) -> + % We have been given a message rather than a list of messages, so we should + % convert it to a list, assuming that the message is monotonically numbered. +``` + +### resolve_many + +```erlang +resolve_many({as, DevID, Msg}, Opts) -> + subresolve(#{}, DevID, Msg, Opts); +``` + +### resolve_many + +```erlang +resolve_many([{resolve, Subres}], Opts) -> + resolve_many(Subres, Opts); +``` + +### resolve_many + +```erlang +resolve_many(MsgList, Opts) -> + ?event(ao_core, {resolve_many, MsgList}, Opts), + Res = do_resolve_many(MsgList, Opts), + ?event(ao_core, {resolve_many_complete, {res, Res}, {req, MsgList}}, Opts), + Res. +``` + +### do_resolve_many + +```erlang +do_resolve_many([], _Opts) -> + {failure, <<"Attempted to resolve an empty message sequence.">>}; +``` + +### do_resolve_many + +```erlang +do_resolve_many([Msg3], Opts) -> + ?event(ao_core, {stage, 11, resolve_complete, Msg3}), + {ok, hb_cache:ensure_loaded(Msg3, Opts)}; +``` + +### do_resolve_many + +```erlang +do_resolve_many([Msg1, Msg2 | MsgList], Opts) -> + ?event(ao_core, {stage, 0, resolve_many, {msg1, Msg1}, {msg2, Msg2}}), + case resolve_stage(1, Msg1, Msg2, Opts) of + {ok, Msg3} -> + ?event(ao_core, + { + stage, + 13, + resolved_step, + {msg3, Msg3}, + {opts, Opts} + }, + Opts + ), + do_resolve_many([Msg3 | MsgList], Opts); + Res -> + % The result is not a resolvable message. Return it. +``` + +### resolve_stage + +```erlang +resolve_stage(1, Link, Msg2, Opts) when ?IS_LINK(Link) -> + % If the first message is a link, we should load the message and + % continue with the resolution. +``` + +### resolve_stage + +```erlang +resolve_stage(1, Msg1, Link, Opts) when ?IS_LINK(Link) -> + % If the second message is a link, we should load the message and + % continue with the resolution. +``` + +### resolve_stage + +```erlang +resolve_stage(1, {as, DevID, Ref}, Msg2, Opts) when ?IS_ID(Ref) orelse ?IS_LINK(Ref) -> + % Normalize `as' requests with a raw ID or link as the path. Links will be + % loaded in following stages. +``` + +### resolve_stage + +```erlang +resolve_stage(1, {as, DevID, Link}, Msg2, Opts) when ?IS_LINK(Link) -> + % If the first message is an `as' with a link, we should load the message and + % continue with the resolution. +``` + +### resolve_stage + +```erlang +resolve_stage(1, {as, DevID, Raw = #{ <<"path">> := ID }}, Msg2, Opts) when ?IS_ID(ID) -> + % If the first message is an `as' with an ID, we should load the message and + % apply the non-path elements of the sub-request to it. +``` + +### resolve_stage + +```erlang +resolve_stage(1, Raw = {as, DevID, SubReq}, Msg2, Opts) -> + % Set the device of the message to the specified one and resolve the sub-path. +``` + +### resolve_stage + +```erlang +resolve_stage(1, RawMsg1, Msg2Outer = #{ <<"path">> := {as, DevID, Msg2Inner} }, Opts) -> + % Set the device to the specified `DevID' and resolve the message. Merging + % the `Msg2Inner' into the `Msg2Outer' message first. We return the result + % of the sub-resolution directly. +``` + +### resolve_stage + +```erlang +resolve_stage(1, {resolve, Subres}, Msg2, Opts) -> + % If the first message is a `{resolve, Subres}' tuple, we should execute it + % directly, then apply the request to the result. +``` + +### resolve_stage + +```erlang +resolve_stage(1, Msg1, {resolve, Subres}, Opts) -> + % If the second message is a `{resolve, Subresolution}' tuple, we should + % execute the subresolution directly to gain the underlying `Msg2' for + % our execution. We assume that the subresolution is already in a normalized, + % executable form, so we pass it to `resolve_many' for execution. +``` + +### resolve_stage + +```erlang +resolve_stage(1, Msg1, Msg2, Opts) when is_list(Msg1) -> + % Normalize lists to numbered maps (base=1) if necessary. +``` + +### resolve_stage + +```erlang +resolve_stage(1, Msg1, NonMapMsg2, Opts) when not is_map(NonMapMsg2) -> + ?event(ao_core, {stage, 1, path_normalize}), + resolve_stage(1, Msg1, #{ <<"path">> => NonMapMsg2 }, Opts); +``` + +### resolve_stage + +```erlang +resolve_stage(1, RawMsg1, RawMsg2, Opts) -> + % Normalize the path to a private key containing the list of remaining + % keys to resolve. +``` + +### resolve_stage + +```erlang +resolve_stage(2, Msg1, Msg2, Opts) -> + ?event(ao_core, {stage, 2, cache_lookup}, Opts), + % Lookup request in the cache. If we find a result, return it. +``` + +### resolve_stage + +```erlang +resolve_stage(3, Msg1, Msg2, Opts) when not is_map(Msg1) or not is_map(Msg2) -> + % Validation check: If the messages are not maps, we cannot find a key + % in them, so return not_found. +``` + +### resolve_stage + +```erlang +resolve_stage(3, Msg1, Msg2, Opts) -> + ?event(ao_core, {stage, 3, validation_check}, Opts), + % Validation check: Check if the message is valid. +``` + +### resolve_stage + +```erlang +resolve_stage(4, Msg1, Msg2, Opts) -> + ?event(ao_core, {stage, 4, persistent_resolver_lookup}, Opts), + % Persistent-resolver lookup: Search for local (or Distributed + % Erlang cluster) processes that are already performing the execution. +``` + +### resolve_stage + +```erlang +resolve_stage(5, Msg1, Msg2, ExecName, Opts) -> + ?event(ao_core, {stage, 5, device_lookup}, Opts), + % Device lookup: Find the Erlang function that should be utilized to + % execute Msg2 on Msg1. +``` + +### resolve_stage + +```erlang +resolve_stage(6, Func, Msg1, Msg2, ExecName, Opts) -> + ?event(ao_core, {stage, 6, ExecName, execution}, Opts), + % Execution. +``` + +### resolve_stage + +```erlang +resolve_stage(7, Msg1, Msg2, {St, Res}, ExecName, Opts = #{ on := On = #{ <<"step">> := _ }}) -> + ?event(ao_core, {stage, 7, ExecName, executing_step_hook, {on, On}}, Opts), + % If the `step' hook is defined, we execute it. Note: This function clause + % matches directly on the `on' key of the `Opts' map. This is in order to + % remove the expensive lookup check that would otherwise be performed on every + % execution. +``` + +### resolve_stage + +```erlang +resolve_stage(7, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 7, ExecName, no_step_hook}, Opts), + resolve_stage(8, Msg1, Msg2, Res, ExecName, Opts); +``` + +### resolve_stage + +```erlang +resolve_stage(8, Msg1, Msg2, {ok, {resolve, Sublist}}, ExecName, Opts) -> + ?event(ao_core, {stage, 8, ExecName, subresolve_result}, Opts), + % If the result is a `{resolve, Sublist}' tuple, we need to execute it + % as a sub-resolution. +``` + +### resolve_stage + +```erlang +resolve_stage(8, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), + resolve_stage(9, Msg1, Msg2, Res, ExecName, Opts); +``` + +### resolve_stage + +```erlang +resolve_stage(9, Msg1, Msg2, {ok, Msg3}, ExecName, Opts) when is_map(Msg3) -> + ?event(ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), + % Cryptographic linking. Now that we have generated the result, we + % need to cryptographically link the output to its input via a hashpath. +``` + +### resolve_stage + +```erlang +resolve_stage(9, Msg1, Msg2, {Status, Msg3}, ExecName, Opts) when is_map(Msg3) -> + ?event(ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), + ?event(hashpath, {resetting_hashpath_msg3, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), + % Skip cryptographic linking and reset the hashpath if the result is abnormal. +``` + +### resolve_stage + +```erlang +resolve_stage(9, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), + % Skip cryptographic linking and continue if we don't have a map that can have + % a hashpath at all. +``` + +### resolve_stage + +```erlang +resolve_stage(10, Msg1, Msg2, {ok, Msg3}, ExecName, Opts) -> + ?event(ao_core, {stage, 10, ExecName, result_caching}, Opts), + % Result caching: Optionally, cache the result of the computation locally. +``` + +### resolve_stage + +```erlang +resolve_stage(10, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), + % Skip result caching if the result is abnormal. +``` + +### resolve_stage + +```erlang +resolve_stage(11, Msg1, Msg2, Res, ExecName, Opts) -> + ?event(ao_core, {stage, 11, ExecName}, Opts), + % Notify processes that requested the resolution while we were executing and + % unregister ourselves from the group. +``` + +### resolve_stage + +```erlang +resolve_stage(12, _Msg1, _Msg2, {ok, Msg3} = Res, ExecName, Opts) -> + ?event(ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), + % Check if we should spawn a worker for the current execution + case {is_map(Msg3), hb_opts:get(spawn_worker, false, Opts#{ prefer => local })} of + {A, B} when (A == false) or (B == false) -> + Res; + {_, _} -> + % Spawn a worker for the current execution + WorkerPID = hb_persistent:start_worker(ExecName, Msg3, Opts), + hb_persistent:forward_work(WorkerPID, Opts), + Res + end; +``` + +### resolve_stage + +```erlang +resolve_stage(12, _Msg1, _Msg2, OtherRes, ExecName, Opts) -> + ?event(ao_core, {stage, 12, ExecName, abnormal_status_skip_spawning}, Opts), + OtherRes. +``` + +### subresolve + +Execute a sub-resolution. + +```erlang +subresolve(RawMsg1, DevID, ReqPath, Opts) when is_binary(ReqPath) -> + % If the request is a binary, we assume that it is a path. +``` + +### subresolve + +```erlang +subresolve(RawMsg1, DevID, Req, Opts) -> + % First, ensure that the message is loaded from the cache. +``` + +### maybe_profiled_apply + +If the `AO_PROFILING` macro is defined (set by building/launching with + +```erlang +maybe_profiled_apply(Func, Args, _Msg1, _Msg2, _Opts) -> + apply(Func, Args). +``` + +### maybe_profiled_apply + +```erlang +maybe_profiled_apply(Func, Args, Msg1, Msg2, Opts) -> + CallStack = erlang:get(ao_stack), + ?event(ao_trace, + {profiling_apply, + {func, Func}, + {args, Args}, + {call_stack, CallStack} + } + ), + Key = + case hb_maps:get(<<"device">>, Msg1, undefined, Opts) of + undefined -> + hb_util:bin(erlang:fun_to_list(Func)); + Device -> + case hb_maps:get(<<"path">>, Msg2, undefined, Opts) of + undefined -> + hb_util:bin(erlang:fun_to_list(Func)); + Path -> + MethodStr = + case hb_maps:get(<<"method">>, Msg2, undefined, Opts) of + undefined -> <<"">>; + <<"GET">> -> <<"">>; + Method -> <<"<", Method/binary, ">">> + end, + << + (hb_util:bin(Device))/binary, + "/", + MethodStr/binary, + (hb_util:bin(Path))/binary + >> + end + end, + put( + ao_stack, + case CallStack of + undefined -> [Key]; + Stack -> [Key | Stack] + end + ), + {ExecMicroSecs, Res} = timer:tc(fun() -> apply(Func, Args) end), + put(ao_stack, CallStack), + hb_event:increment(<<"ao-call-counts">>, Key, Opts), + hb_event:increment(<<"ao-total-durations">>, Key, Opts, ExecMicroSecs), + case CallStack of + undefined -> ok; + [Caller|_] -> + hb_event:increment( + <<"ao-callers:", Key/binary>>, + hb_util:bin( + [ + <<"duration:">>, + Caller + ] + ), + Opts, + ExecMicroSecs + ), + hb_event:increment( + <<"ao-callers:", Key/binary>>, + hb_util:bin( + [ + <<"calls:">>, + Caller + ]), + Opts + ) + end, + Res. +``` + +### ensure_message_loaded + +Ensure that a message is loaded from the cache if it is an ID, or + +```erlang +ensure_message_loaded(MsgID, Opts) when ?IS_ID(MsgID) -> + case hb_cache:read(MsgID, Opts) of + {ok, LoadedMsg} -> + LoadedMsg; + not_found -> + throw({necessary_message_not_found, <<"/">>, MsgID}) + end; +``` + +### ensure_message_loaded + +Ensure that a message is loaded from the cache if it is an ID, or + +```erlang +ensure_message_loaded(MsgLink, Opts) when ?IS_LINK(MsgLink) -> + hb_cache:ensure_loaded(MsgLink, Opts); +``` + +### ensure_message_loaded + +Ensure that a message is loaded from the cache if it is an ID, or + +```erlang +ensure_message_loaded(Msg, _Opts) -> + Msg. +``` + +### error_invalid_message + +Catch all return if the message is invalid. + +```erlang +error_invalid_message(Msg1, Msg2, Opts) -> + ?event( + ao_core, + {error, {type, invalid_message}, + {msg1, Msg1}, + {msg2, Msg2}, + {opts, Opts} + }, + Opts + ), + { + error, + #{ + <<"status">> => 400, + <<"body">> => <<"Request contains non-verifiable message.">> + } + }. +``` + +### error_infinite + +Catch all return if we are in an infinite loop. + +```erlang +error_infinite(Msg1, Msg2, Opts) -> + ?event( + ao_core, + {error, {type, infinite_recursion}, + {msg1, Msg1}, + {msg2, Msg2}, + {opts, Opts} + }, + Opts + ), + ?trace(), + { + error, + #{ + <<"status">> => 508, + <<"body">> => <<"Request creates infinite recursion.">> + } + }. +``` + +### error_invalid_intermediate_status + +```erlang +error_invalid_intermediate_status(Msg1, Msg2, Msg3, RemainingPath, Opts) -> + ?event( + ao_core, + {error, {type, invalid_intermediate_status}, + {msg2, Msg2}, + {msg3, Msg3}, + {remaining_path, RemainingPath}, + {opts, Opts} + }, + Opts + ), + ?event(ao_result, + {intermediate_failure, {msg1, Msg1}, + {msg2, Msg2}, {msg3, Msg3}, + {remaining_path, RemainingPath}, {opts, Opts}}), + { + error, + #{ + <<"status">> => 422, + <<"body">> => Msg3, + <<"key">> => hb_maps:get(<<"path">>, Msg2, <<"Key unknown.">>, Opts), + <<"remaining-path">> => RemainingPath + } + }. +``` + +### error_execution + +Handle an error in a device call. + +```erlang +error_execution(ExecGroup, Msg2, Whence, {Class, Exception, Stacktrace}, Opts) -> + Error = {error, Whence, {Class, Exception, Stacktrace}}, + hb_persistent:unregister_notify(ExecGroup, Msg2, Error, Opts), + ?event(ao_core, {handle_error, Error, {opts, Opts}}, Opts), + case hb_opts:get(error_strategy, throw, Opts) of + throw -> erlang:raise(Class, Exception, Stacktrace); + _ -> Error + end. +``` + +### maybe_force_message + +Force the result of a device call into a message if the result is not + +```erlang +maybe_force_message({Status, Res}, Opts) -> + case hb_opts:get(force_message, false, Opts) of + true -> force_message({Status, Res}, Opts); + false -> {Status, Res} + end; +``` + +### maybe_force_message + +Force the result of a device call into a message if the result is not + +```erlang +maybe_force_message(Res, Opts) -> + maybe_force_message({ok, Res}, Opts). +``` + +### force_message + +```erlang +force_message({Status, Res}, Opts) when is_list(Res) -> + force_message({Status, normalize_keys(Res, Opts)}, Opts); +``` + +### force_message + +```erlang +force_message({Status, Subres = {resolve, _}}, _Opts) -> + {Status, Subres}; +``` + +### force_message + +```erlang +force_message({Status, Literal}, _Opts) when not is_map(Literal) -> + ?event({force_message_from_literal, Literal}), + {Status, #{ <<"ao-result">> => <<"body">>, <<"body">> => Literal }}; +``` + +### force_message + +```erlang +force_message({Status, M = #{ <<"status">> := Status, <<"body">> := Body }}, _Opts) + when map_size(M) == 2 -> + ?event({force_message_from_literal_with_status, M}), + {Status, #{ + <<"status">> => Status, + <<"ao-result">> => <<"body">>, + <<"body">> => Body + }}; +``` + +### force_message + +```erlang +force_message({Status, Map}, _Opts) -> + ?event({force_message_from_map, Map}), + {Status, Map}. +``` + +### get + +Shortcut for resolving a key in a message without its status if it is + +```erlang +get(Path, Msg) -> + get(Path, Msg, #{}). +``` + +### get + +```erlang +get(Path, Msg, Opts) -> + get(Path, Msg, not_found, Opts). +``` + +### get + +```erlang +get(Path, {as, Device, Msg}, Default, Opts) -> + get( + Path, + set( + Msg, + #{ <<"device">> => Device }, + internal_opts(Opts) + ), + Default, + Opts + ); +``` + +### get + +```erlang +get(Path, Msg, Default, Opts) -> + case resolve(Msg, #{ <<"path">> => Path }, Opts#{ spawn_worker => false }) of + {ok, Value} -> Value; + {error, _} -> Default + end. +``` + +### get_first + +take a sequence of base messages and paths, then return the value of the + +```erlang +get_first(Paths, Opts) -> get_first(Paths, not_found, Opts). +``` + +### get_first + +take a sequence of base messages and paths, then return the value of the + +```erlang +get_first([], Default, _Opts) -> Default; +``` + +### get_first + +take a sequence of base messages and paths, then return the value of the + +```erlang +get_first([{Base, Path}|Msgs], Default, Opts) -> + case get(Path, Base, Opts) of + not_found -> get_first(Msgs, Default, Opts); + Value -> Value + end. +``` + +### keys + +Shortcut to get the list of keys from a message. + +```erlang +keys(Msg) -> keys(Msg, #{}). +``` + +### keys + +Shortcut to get the list of keys from a message. + +```erlang +keys(Msg, Opts) -> keys(Msg, Opts, keep). +``` + +### keys + +Shortcut to get the list of keys from a message. + +```erlang +keys(Msg, Opts, keep) -> + % There is quite a lot of AO-Core-specific machinery here. We: + % 1. `get' the keys from the message, via AO-Core in order to trigger the + % `keys' function on its device. +``` + +### keys + +```erlang +keys(Msg, Opts, remove) -> + lists:filter( + fun(Key) -> not lists:member(Key, ?AO_CORE_KEYS) end, + keys(Msg, Opts, keep) + ). +``` + +### set + +Shortcut for setting a key in the message using its underlying device. + +```erlang +set(RawMsg1, RawMsg2, Opts) when is_map(RawMsg2) -> + Msg1 = normalize_keys(RawMsg1, Opts), + Msg2 = hb_maps:without([<<"hashpath">>, <<"priv">>], normalize_keys(RawMsg2, Opts), Opts), + ?event(ao_internal, {set_called, {msg1, Msg1}, {msg2, Msg2}}, Opts), + % Get the next key to set. +``` + +### set + +```erlang +set(Msg1, Key, Value, Opts) -> + % For an individual key, we run deep_set with the key as the path. +``` + +### deep_set + +Recursively search a map, resolving keys, and set the value of the key + +```erlang +deep_set(Msg, [], Value, Opts) when is_map(Msg) or is_list(Msg) -> + device_set(Msg, <<"/">>, Value, Opts); +``` + +### deep_set + +Recursively search a map, resolving keys, and set the value of the key + +```erlang +deep_set(_Msg, [], Value, _Opts) -> + Value; +``` + +### deep_set + +Recursively search a map, resolving keys, and set the value of the key + +```erlang +deep_set(Msg, [Key], Value, Opts) -> + device_set(Msg, Key, Value, Opts); +``` + +### deep_set + +Recursively search a map, resolving keys, and set the value of the key + +```erlang +deep_set(Msg, [Key|Rest], Value, Opts) -> + case resolve(Msg, Key, Opts) of + {ok, SubMsg} -> + ?event( + {traversing_deeper_to_set, + {current_key, Key}, + {current_value, SubMsg}, + {rest, Rest} + } + ), + Res = device_set(Msg, Key, deep_set(SubMsg, Rest, Value, Opts), <<"explicit">>, Opts), + ?event({deep_set_result, {msg, Msg}, {key, Key}, {res, Res}}), + Res; + _ -> + ?event( + {creating_new_map, + {current_key, Key}, + {rest, Rest} + } + ), + Msg#{ Key => deep_set(#{}, Rest, Value, Opts) } + end. +``` + +### device_set + +Call the device's `set` function. + +```erlang +device_set(Msg, Key, Value, Opts) -> + device_set(Msg, Key, Value, <<"deep">>, Opts). +``` + +### device_set + +Call the device's `set` function. + +```erlang +device_set(Msg, Key, Value, Mode, Opts) -> + ReqWithoutMode = + case Key of + <<"path">> -> + #{ <<"path">> => <<"set_path">>, <<"value">> => Value }; + <<"/">> when is_map(Value) -> + % The value is a map and it is to be `set' at the root of the + % message. Subsequently, we call the device's `set' function + % with all of the keys found in the message, leading it to be + % merged into the message. +``` + +### remove + +Remove a key from a message, using its underlying device. + +```erlang +remove(Msg, Key) -> remove(Msg, Key, #{}). +``` + +### remove + +Remove a key from a message, using its underlying device. + +```erlang +remove(Msg, Key, Opts) -> + hb_util:ok( + resolve( + Msg, + #{ <<"path">> => <<"remove">>, <<"item">> => Key }, + internal_opts(Opts) + ), + Opts + ). +``` + +### truncate_args + +Truncate the arguments of a function to the number of arguments it + +```erlang +truncate_args(Fun, Args) -> + {arity, Arity} = erlang:fun_info(Fun, arity), + lists:sublist(Args, Arity). +``` + +### message_to_fun + +Calculate the Erlang function that should be called to get a value for + +```erlang +message_to_fun(Msg, Key, Opts) -> + % Get the device module from the message. +``` + +### message_to_device + +Extract the device module from a message. + +```erlang +message_to_device(Msg, Opts) -> + case dev_message:get(<<"device">>, Msg, Opts) of + {error, not_found} -> + % The message does not specify a device, so we use the default device. +``` + +### info_handler_to_fun + +Parse a handler key given by a device's `info`. + +```erlang +info_handler_to_fun(Handler, _Msg, _Key, _Opts) when is_function(Handler) -> + {add_key, Handler}; +``` + +### info_handler_to_fun + +Parse a handler key given by a device's `info`. + +```erlang +info_handler_to_fun(HandlerMap, Msg, Key, Opts) -> + case hb_maps:find(excludes, HandlerMap, Opts) of + {ok, Exclude} -> + case lists:member(Key, Exclude) of + true -> + {ok, MsgWithoutDevice} = + dev_message:remove(Msg, #{ item => device }, Opts), + message_to_fun( + MsgWithoutDevice#{ <<"device">> => default_module() }, + Key, + Opts + ); + false -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} + end; + error -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} + end. +``` + +### find_exported_function + +Find the function with the highest arity that has the given name, if it + +```erlang +find_exported_function(Msg, Dev, Key, MaxArity, Opts) when is_map(Dev) -> + case hb_maps:get(normalize_key(Key), normalize_keys(Dev, Opts), not_found, Opts) of + not_found -> not_found; + Fun when is_function(Fun) -> + case erlang:fun_info(Fun, arity) of + {arity, Arity} when Arity =< MaxArity -> + case is_exported(Msg, Dev, Key, Opts) of + true -> {ok, Fun}; + false -> not_found + end; + _ -> not_found + end + end; +``` + +### find_exported_function + +Find the function with the highest arity that has the given name, if it + +```erlang +find_exported_function(_Msg, _Mod, _Key, Arity, _Opts) when Arity < 0 -> + not_found; +``` + +### find_exported_function + +Find the function with the highest arity that has the given name, if it + +```erlang +find_exported_function(Msg, Mod, Key, Arity, Opts) when not is_atom(Key) -> + try hb_util:key_to_atom(Key, false) of + KeyAtom -> find_exported_function(Msg, Mod, KeyAtom, Arity, Opts) + catch _:_ -> not_found + end; +``` + +### find_exported_function + +Find the function with the highest arity that has the given name, if it + +```erlang +find_exported_function(Msg, Mod, Key, Arity, Opts) -> + case erlang:function_exported(Mod, Key, Arity) of + true -> + case is_exported(Msg, Mod, Key, Opts) of + true -> {ok, fun Mod:Key/Arity}; + false -> not_found + end; + false -> + find_exported_function(Msg, Mod, Key, Arity - 1, Opts) + end. +``` + +### is_exported + +Check if a device is guarding a key via its `exports` list. Defaults to + +```erlang +is_exported(_Msg, _Dev, info, _Opts) -> true; +``` + +### is_exported + +Check if a device is guarding a key via its `exports` list. Defaults to + +```erlang +is_exported(Msg, Dev, Key, Opts) -> + is_exported(info(Dev, Msg, Opts), Key, Opts). +``` + +### is_exported + +```erlang +is_exported(_, info, _Opts) -> true; +``` + +### is_exported + +```erlang +is_exported(Info = #{ excludes := Excludes }, Key, Opts) -> + case lists:member(normalize_key(Key), lists:map(fun normalize_key/1, Excludes)) of + true -> false; + false -> is_exported(hb_maps:remove(excludes, Info, Opts), Key, Opts) + end; +``` + +### is_exported + +```erlang +is_exported(#{ exports := Exports }, Key, _Opts) -> + lists:member(normalize_key(Key), lists:map(fun normalize_key/1, Exports)); +``` + +### is_exported + +Convert a key to a binary in normalized form. + +```erlang +is_exported(_Info, _Key, _Opts) -> true. +``` + +### normalize_key + +Convert a key to a binary in normalized form. + +```erlang +normalize_key(Key) -> normalize_key(Key, #{}). +``` + +### normalize_key + +Convert a key to a binary in normalized form. + +```erlang +normalize_key(Key, _Opts) when is_binary(Key) -> Key; +``` + +### normalize_key + +Convert a key to a binary in normalized form. + +```erlang +normalize_key(Key, _Opts) when is_atom(Key) -> atom_to_binary(Key); +``` + +### normalize_key + +Convert a key to a binary in normalized form. + +```erlang +normalize_key(Key, _Opts) when is_integer(Key) -> integer_to_binary(Key); +``` + +### normalize_key + +Convert a key to a binary in normalized form. + +```erlang +normalize_key(Key, _Opts) when is_list(Key) -> + case hb_util:is_string_list(Key) of + true -> normalize_key(list_to_binary(Key)); + false -> + iolist_to_binary( + lists:join( + <<"/">>, + lists:map(fun normalize_key/1, Key) + ) + ) + end. +``` + +### normalize_keys + +Ensure that a message is processable by the AO-Core resolver: No lists. + +```erlang +normalize_keys(Msg) -> normalize_keys(Msg, #{}). +``` + +### normalize_keys + +Ensure that a message is processable by the AO-Core resolver: No lists. + +```erlang +normalize_keys(Msg1, Opts) when is_list(Msg1) -> + normalize_keys( + hb_maps:from_list( + lists:zip( + lists:seq(1, length(Msg1)), + Msg1 + ) + ), + Opts + ); +``` + +### normalize_keys + +Ensure that a message is processable by the AO-Core resolver: No lists. + +```erlang +normalize_keys(Map, Opts) when is_map(Map) -> + hb_maps:from_list( + lists:map( + fun({Key, Value}) when is_map(Value) -> + {hb_ao:normalize_key(Key), Value}; + ({Key, Value}) -> + {hb_ao:normalize_key(Key), Value} + end, + hb_maps:to_list(Map, Opts) + ) + ); +``` + +### normalize_keys + +Ensure that a message is processable by the AO-Core resolver: No lists. +Load a device module from its name or a message ID. + +```erlang +normalize_keys(Other, _Opts) -> Other. +``` + +### load_device + +Ensure that a message is processable by the AO-Core resolver: No lists. +Load a device module from its name or a message ID. + +```erlang +load_device(Map, _Opts) when is_map(Map) -> {ok, Map}; +``` + +### load_device + +Ensure that a message is processable by the AO-Core resolver: No lists. +Load a device module from its name or a message ID. + +```erlang +load_device(ID, _Opts) when is_atom(ID) -> + try ID:module_info(), {ok, ID} + catch _:_ -> {error, not_loadable} + end; +``` + +### load_device + +Ensure that a message is processable by the AO-Core resolver: No lists. +Load a device module from its name or a message ID. + +```erlang +load_device(ID, Opts) when ?IS_ID(ID) -> + ?event(device_load, {requested_load, {id, ID}}, Opts), + case hb_opts:get(load_remote_devices, false, Opts) of + false -> + {error, remote_devices_disabled}; + true -> + ?event(device_load, {loading_from_cache, {id, ID}}, Opts), + {ok, Msg} = hb_cache:read(ID, Opts), + ?event(device_load, {received_device, {id, ID}, {msg, Msg}}, Opts), + TrustedSigners = hb_opts:get(trusted_device_signers, [], Opts), + Trusted = + lists:any( + fun(Signer) -> + lists:member(Signer, TrustedSigners) + end, + hb_message:signers(Msg, Opts) + ), + ?event(device_load, + {verifying_device_trust, + {id, ID}, + {trusted, Trusted}, + {signers, hb_message:signers(Msg, Opts)} + }, + Opts + ), + case Trusted of + false -> {error, device_signer_not_trusted}; + true -> + ?event(device_load, {loading_device, {id, ID}}, Opts), + case hb_maps:get(<<"content-type">>, Msg, undefined, Opts) of + <<"application/beam">> -> + case verify_device_compatibility(Msg, Opts) of + ok -> + ModName = + hb_util:key_to_atom( + hb_maps:get( + <<"module-name">>, + Msg, + undefined, + Opts + ), + new_atoms + ), + LoadRes = + erlang:load_module( + ModName, + hb_maps:get( + <<"body">>, + Msg, + undefined, + Opts + ) + ), + case LoadRes of + {module, _} -> + {ok, ModName}; + {error, Reason} -> + {error, {device_load_failed, Reason}} + end; + {error, Reason} -> + {error, {device_load_failed, Reason}} + end; + Other -> + {error, + {device_load_failed, + {incompatible_content_type, Other}, + {expected, <<"application/beam">>}, + {found, Other} + } + } + end + end + end; +``` + +### load_device + +Ensure that a message is processable by the AO-Core resolver: No lists. +Load a device module from its name or a message ID. + +```erlang +load_device(ID, Opts) -> + NormKey = + case is_atom(ID) of + true -> ID; + false -> normalize_key(ID) + end, + case lists:search( + fun (#{ <<"name">> := Name }) -> Name =:= NormKey end, + Preloaded = hb_opts:get(preloaded_devices, [], Opts) + ) of + false -> {error, {module_not_admissable, NormKey, Preloaded}}; + {value, #{ <<"module">> := Mod }} -> load_device(Mod, Opts) + end. +``` + +### verify_device_compatibility + +Verify that a device is compatible with the current machine. + +```erlang +verify_device_compatibility(Msg, Opts) -> + ?event(device_load, {verifying_device_compatibility, {msg, Msg}}, Opts), + Required = + lists:filtermap( + fun({<<"requires-", Key/binary>>, Value}) -> + {true, + { + hb_util:key_to_atom( + hb_ao:normalize_key(Key), + new_atoms + ), + hb_cache:ensure_loaded(Value, Opts) + } + }; + (_) -> false + end, + hb_maps:to_list(Msg, Opts) + ), + ?event(device_load, + {discerned_requirements, + {required, Required}, + {msg, Msg} + }, + Opts + ), + FailedToMatch = + lists:filtermap( + fun({Property, Value}) -> + % The values of these properties are _not_ 'keys', but we normalize + % them as such in order to make them comparable. +``` + +### info + +Get the info map for a device, optionally giving it a message if the + +```erlang +info(Msg, Opts) -> + info(message_to_device(Msg, Opts), Msg, Opts). +``` + +### info + +```erlang +info(DevMod, Msg, Opts) -> + %?event({calculating_info, {dev, DevMod}, {msg, Msg}}), + case find_exported_function(Msg, DevMod, info, 2, Opts) of + {ok, Fun} -> + Res = apply(Fun, truncate_args(Fun, [Msg, Opts])), + % ?event({ + % info_result, + % {dev, DevMod}, + % {args, truncate_args(Fun, [Msg])}, + % {result, Res} + % }), + Res; + not_found -> #{} + end. +``` + +### default_module + +The default device is the identity device, which simply returns the +The execution options that are used internally by this module + +```erlang +default_module() -> dev_message. +``` + +### internal_opts + +The default device is the identity device, which simply returns the +The execution options that are used internally by this module + +```erlang +internal_opts(Opts) -> + hb_maps:merge(Opts, #{ + topic => hb_opts:get(topic, ao_internal, Opts), + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>], + spawn_worker => false, + await_inprogress => false +``` + +--- + +*Generated from [hb_ao.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_ao.erl)* diff --git a/docs/book/src/hb_ao_test_vectors.erl.md b/docs/book/src/hb_ao_test_vectors.erl.md new file mode 100644 index 000000000..9a27b1619 --- /dev/null +++ b/docs/book/src/hb_ao_test_vectors.erl.md @@ -0,0 +1,892 @@ +# hb_ao_test_vectors + +[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_ao_test_vectors.erl) + +Uses a series of different `Opts` values to test the resolution engine's +execution under different circumstances. + +--- + +### run_test + +Uses a series of different `Opts` values to test the resolution engine's +Easy hook to make a test executable via the command line: + +```erlang +run_test() -> + multiple_as_subresolutions_test(#{}). +``` + +### suite_test_ + +Run each test in the file with each set of options. Start and reset + +```erlang +suite_test_() -> + hb_test_utils:suite_with_opts(test_suite(), test_opts()). +``` + +### benchmark_test_ + +```erlang +benchmark_test_() -> + hb_test_utils:suite_with_opts(benchmark_suite(), test_opts()). +``` + +### test_suite + +```erlang +test_suite() -> + [ + {resolve_simple, "resolve simple", + fun resolve_simple_test/1}, + {resolve_id, "resolve id", + fun resolve_id_test/1}, + {start_as, "start as", + fun start_as_test/1}, + {start_as_with_parameters, "start as with parameters", + fun start_as_with_parameters_test/1}, + {load_as, "load as", + fun load_as_test/1}, + {as_path, "as path", + fun as_path_test/1}, + {continue_as, "continue as", + fun continue_as_test/1}, + {multiple_as_subresolutions, "multiple as subresolutions", + fun multiple_as_subresolutions_test/1}, + {resolve_key_twice, "resolve key twice", + fun resolve_key_twice_test/1}, + {resolve_from_multiple_keys, "resolve from multiple keys", + fun resolve_from_multiple_keys_test/1}, + {resolve_path_element, "resolve path element", + fun resolve_path_element_test/1}, + {resolve_binary_key, "resolve binary key", + fun resolve_binary_key_test/1}, + {key_to_binary, "key to binary", + fun key_to_binary_test/1}, + {key_from_id_device_with_args, "key from id device with args", + fun key_from_id_device_with_args_test/1}, + {device_with_handler_function, "device with handler function", + fun device_with_handler_function_test/1}, + {device_with_default_handler_function, + "device with default handler function", + fun device_with_default_handler_function_test/1}, + {basic_get, "basic get", + fun basic_get_test/1}, + {recursive_get, "recursive get", + fun recursive_get_test/1}, + {deep_recursive_get, "deep recursive get", + fun deep_recursive_get_test/1}, + {basic_set, "basic set", + fun basic_set_test/1}, + {get_with_device, "get with device", + fun get_with_device_test/1}, + {get_as_with_device, "get as with device", + fun get_as_with_device_test/1}, + {set_with_device, "set with device", + fun set_with_device_test/1}, + {deep_set, "deep set", + fun deep_set_test/1}, + {deep_set_with_device, "deep set with device", + fun deep_set_with_device_test/1}, + {device_exports, "device exports", + fun device_exports_test/1}, + {device_excludes, "device excludes", + fun device_excludes_test/1}, + {denormalized_device_key, "denormalized device key", + fun denormalized_device_key_test/1}, + {list_transform, "list transform", + fun list_transform_test/1}, + {step_hook, "step hook", + fun step_hook_test/1} + ]. +``` + +### benchmark_suite + +```erlang +benchmark_suite() -> + [ + {benchmark_simple, "simple resolution benchmark", + fun benchmark_simple_test/1}, + {benchmark_multistep, "multistep resolution benchmark", + fun benchmark_multistep_test/1}, + {benchmark_get, "get benchmark", + fun benchmark_get_test/1}, + {benchmark_set, "single value set benchmark", + fun benchmark_set_test/1}, + {benchmark_set_multiple, "set two keys benchmark", + fun benchmark_set_multiple_test/1}, + {benchmark_set_multiple_deep, "set two keys deep benchmark", + fun benchmark_set_multiple_deep_test/1} + ]. +``` + +### test_opts + +```erlang +test_opts() -> + [ + #{ + name => normal, + desc => "Default opts", + opts => #{}, + skip => [] + }, + #{ + name => without_hashpath, + desc => "Default without hashpath", + opts => #{ + hashpath => ignore + }, + skip => [] + }, + #{ + name => no_cache, + desc => "No cache read or write", + opts => #{ + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>], + spawn_worker => false, + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/fs">> + } + }, + skip => [load_as] + }, + #{ + name => only_store, + desc => "Store, don't read", + opts => #{ + hashpath => update, + cache_control => [<<"no-cache">>], + spawn_worker => false, + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/fs">> + } + }, + skip => [ + denormalized_device_key, + deep_set_with_device, + load_as + ], + reset => false + }, + #{ + name => only_if_cached, + desc => "Only read, don't exec", + opts => #{ + hashpath => ignore, + cache_control => [<<"only-if-cached">>], + spawn_worker => false, + store => #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-TEST/fs">> + } + }, + skip => [ + % Exclude tests that return a list on its own for now, as raw + % lists cannot be cached yet. +``` + +### exec_dummy_device + +Ensure that we can read a device from the cache then execute it. By + +```erlang +exec_dummy_device(SigningWallet, Opts) -> + % Compile the test device and store it in an accessible cache to the execution + % environment. +``` + +### load_device_test + +```erlang +load_device_test() -> + % Establish an execution environment which trusts the device author. +``` + +### untrusted_load_device_test + +```erlang +untrusted_load_device_test() -> + % Establish an execution environment which does not trust the device author. +``` + +### resolve_simple_test + +```erlang +resolve_simple_test(Opts) -> + Res = hb_ao:resolve(#{ <<"a">> => <<"RESULT">> }, <<"a">>, Opts), + ?assertEqual({ok, <<"RESULT">>}, Res). +``` + +### resolve_id_test + +```erlang +resolve_id_test(Opts) -> + ?assertMatch( + ID when byte_size(ID) == 43, + hb_ao:get(id, #{ test_key => <<"1">> }, Opts) + ). +``` + +### resolve_key_twice_test + +```erlang +resolve_key_twice_test(Opts) -> + % Ensure that the same message can be resolved again. +``` + +### resolve_from_multiple_keys_test + +```erlang +resolve_from_multiple_keys_test(Opts) -> + ?assertEqual( + {ok, [<<"a">>]}, + hb_ao:resolve(#{ <<"a">> => <<"1">>, <<"priv_a">> => <<"2">> }, <<"keys">>, Opts) + ). +``` + +### resolve_path_element_test + +```erlang +resolve_path_element_test(Opts) -> + ?assertEqual( + {ok, [<<"test_path">>]}, + hb_ao:resolve(#{ <<"path">> => [<<"test_path">>] }, <<"path">>, Opts) + ), + ?assertEqual( + {ok, [<<"a">>]}, + hb_ao:resolve(#{ <<"Path">> => [<<"a">>] }, <<"Path">>, Opts) + ). +``` + +### key_to_binary_test + +```erlang +key_to_binary_test(Opts) -> + ?assertEqual(<<"a">>, hb_ao:normalize_key(a, Opts)), + ?assertEqual(<<"a">>, hb_ao:normalize_key(<<"a">>, Opts)), + ?assertEqual(<<"a">>, hb_ao:normalize_key("a", Opts)). +``` + +### resolve_binary_key_test + +```erlang +resolve_binary_key_test(Opts) -> + ?assertEqual( + {ok, <<"RESULT">>}, + hb_ao:resolve(#{ a => <<"RESULT">> }, <<"a">>, Opts) + ), + ?assertEqual( + {ok, <<"1">>}, + hb_ao:resolve( + #{ + <<"Test-Header">> => <<"1">> + }, + <<"Test-Header">>, + Opts + ) + ). +``` + +### generate_device_with_keys_using_args + +Generates a test device with three keys, each of which uses + +```erlang +generate_device_with_keys_using_args() -> + #{ + key_using_only_state => + fun(State) -> + {ok, + <<(hb_maps:get(<<"state_key">>, State))/binary>> + } + end, + key_using_state_and_msg => + fun(State, Msg) -> + {ok, + << + (hb_maps:get(<<"state_key">>, State))/binary, + (hb_maps:get(<<"msg_key">>, Msg))/binary + >> + } + end, + key_using_all => + fun(State, Msg, Opts) -> + {ok, + << + (hb_maps:get(<<"state_key">>, State, undefined, Opts))/binary, + (hb_maps:get(<<"msg_key">>, Msg, undefined, Opts))/binary, + (hb_maps:get(<<"opts_key">>, Opts, undefined, Opts))/binary + >> + } + end + }. +``` + +### gen_default_device + +Create a simple test device that implements the default handler. + +```erlang +gen_default_device() -> + #{ + info => + fun() -> + #{ + default => + fun(_, _State) -> + {ok, <<"DEFAULT">>} + end + } + end, + <<"state_key">> => + fun(_) -> + {ok, <<"STATE">>} + end + }. +``` + +### gen_handler_device + +Create a simple test device that implements the handler key. + +```erlang +gen_handler_device() -> + #{ + info => + fun() -> + #{ + handler => + fun(<<"set">>, M1, M2, Opts) -> + dev_message:set(M1, M2, Opts); + (_, _, _, _) -> + {ok, <<"HANDLER VALUE">>} + end + } + end + }. +``` + +### key_from_id_device_with_args_test + +Test that arguments are passed to a device key as expected. + +```erlang +key_from_id_device_with_args_test(Opts) -> + Msg = + #{ + device => generate_device_with_keys_using_args(), + state_key => <<"1">> + }, + ?assertEqual( + {ok, <<"1">>}, + hb_ao:resolve( + Msg, + #{ + <<"path">> => <<"key_using_only_state">>, + <<"msg_key">> => <<"2">> % Param message, which is ignored + }, + Opts + ) + ), + ?assertEqual( + {ok, <<"13">>}, + hb_ao:resolve( + Msg, + #{ + <<"path">> => <<"key_using_state_and_msg">>, + <<"msg_key">> => <<"3">> % Param message, with value to add + }, + Opts + ) + ), + ?assertEqual( + {ok, <<"1337">>}, + hb_ao:resolve( + Msg, + #{ + <<"path">> => <<"key_using_all">>, + <<"msg_key">> => <<"3">> % Param message + }, + Opts#{ + <<"opts_key">> => <<"37">>, + <<"cache_control">> => [<<"no-cache">>, <<"no-store">>] + } + ) + ). +``` + +### device_with_handler_function_test + +```erlang +device_with_handler_function_test(Opts) -> + Msg = + #{ + device => gen_handler_device(), + test_key => <<"BAD">> + }, + ?assertEqual( + {ok, <<"HANDLER VALUE">>}, + hb_ao:resolve(Msg, <<"test_key">>, Opts) + ). +``` + +### device_with_default_handler_function_test + +```erlang +device_with_default_handler_function_test(Opts) -> + Msg = + #{ + device => gen_default_device() + }, + ?assertEqual( + {ok, <<"STATE">>}, + hb_ao:resolve(Msg, <<"state_key">>, Opts) + ), + ?assertEqual( + {ok, <<"DEFAULT">>}, + hb_ao:resolve(Msg, <<"any_random_key">>, Opts) + ). +``` + +### basic_get_test + +```erlang +basic_get_test(Opts) -> + Msg = #{ <<"key1">> => <<"value1">>, <<"key2">> => <<"value2">> }, + ?assertEqual(<<"value1">>, hb_ao:get(<<"key1">>, Msg, Opts)), + ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), + ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), + ?assertEqual(<<"value2">>, hb_ao:get([<<"key2">>], Msg, Opts)). +``` + +### recursive_get_test + +```erlang +recursive_get_test(Opts) -> + Msg = #{ + <<"key1">> => <<"value1">>, + <<"key2">> => #{ + <<"key3">> => <<"value3">>, + <<"key4">> => #{ + <<"key5">> => <<"value5">>, + <<"key6">> => #{ + <<"key7">> => <<"value7">> + } + } + } + }, + ?assertEqual( + {ok, <<"value1">>}, + hb_ao:resolve(Msg, #{ <<"path">> => <<"key1">> }, Opts) + ), + ?assertEqual(<<"value1">>, hb_ao:get(<<"key1">>, Msg, Opts)), + ?assertEqual( + {ok, <<"value3">>}, + hb_ao:resolve(Msg, #{ <<"path">> => [<<"key2">>, <<"key3">>] }, Opts) + ), + ?assertEqual(<<"value3">>, hb_ao:get([<<"key2">>, <<"key3">>], Msg, Opts)), + ?assertEqual(<<"value3">>, hb_ao:get(<<"key2/key3">>, Msg, Opts)). +``` + +### deep_recursive_get_test + +```erlang +deep_recursive_get_test(Opts) -> + Msg = #{ + <<"key1">> => <<"value1">>, + <<"key2">> => #{ + <<"key3">> => <<"value3">>, + <<"key4">> => #{ + <<"key5">> => <<"value5">>, + <<"key6">> => #{ + <<"key7">> => <<"value7">> + } + } + } + }, + ?assertEqual(<<"value7">>, hb_ao:get(<<"key2/key4/key6/key7">>, Msg, Opts)). +``` + +### basic_set_test + +```erlang +basic_set_test(Opts) -> + Msg = #{ <<"key1">> => <<"value1">>, <<"key2">> => <<"value2">> }, + UpdatedMsg = hb_ao:set(Msg, #{ <<"key1">> => <<"new_value1">> }, Opts), + ?event({set_key_complete, {key, <<"key1">>}, {value, <<"new_value1">>}}), + ?assertEqual(<<"new_value1">>, hb_ao:get(<<"key1">>, UpdatedMsg, Opts)), + ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, UpdatedMsg, Opts)). +``` + +### get_with_device_test + +```erlang +get_with_device_test(Opts) -> + Msg = + #{ + <<"device">> => generate_device_with_keys_using_args(), + <<"state_key">> => <<"STATE">> + }, + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), + ?assertEqual(<<"STATE">>, hb_ao:get(<<"key_using_only_state">>, Msg, Opts)). +``` + +### get_as_with_device_test + +```erlang +get_as_with_device_test(Opts) -> + Msg = + #{ + <<"device">> => gen_handler_device(), + <<"test_key">> => <<"ACTUAL VALUE">> + }, + ?assertEqual( + <<"HANDLER VALUE">>, + hb_ao:get(test_key, Msg, Opts) + ), + ?assertEqual( + <<"ACTUAL VALUE">>, + hb_ao:get(test_key, {as, dev_message, Msg}, Opts) + ). +``` + +### set_with_device_test + +```erlang +set_with_device_test(Opts) -> + Msg = + #{ + <<"device">> => + #{ + <<"set">> => + fun(State, _Msg) -> + Acc = hb_maps:get(<<"set_count">>, State, <<"">>, Opts), + {ok, + State#{ + <<"set_count">> => << Acc/binary, "." >> + } + } + end + }, + <<"state_key">> => <<"STATE">> + }, + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), + SetOnce = hb_ao:set(Msg, #{ <<"state_key">> => <<"SET_ONCE">> }, Opts), + ?assertEqual(<<".">>, hb_ao:get(<<"set_count">>, SetOnce, Opts)), + SetTwice = hb_ao:set(SetOnce, #{ <<"state_key">> => <<"SET_TWICE">> }, Opts), + ?assertEqual(<<"..">>, hb_ao:get(<<"set_count">>, SetTwice, Opts)), + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, SetTwice, Opts)). +``` + +### deep_set_test + +```erlang +deep_set_test(Opts) -> + % First validate second layer changes are handled correctly. +``` + +### deep_set_new_messages_test + +```erlang +deep_set_new_messages_test() -> + Opts = hb_maps:get(opts, hd(test_opts())), + % Test that new messages are created when the path does not exist. +``` + +### deep_set_with_device_test + +```erlang +deep_set_with_device_test(Opts) -> + Device = #{ + set => + fun(Msg1, Msg2) -> + % A device where the set function modifies the key + % and adds a modified flag. +``` + +### device_exports_test + +```erlang +device_exports_test(Opts) -> + Msg = #{ <<"device">> => dev_message }, + ?assert(hb_ao:is_exported(Msg, dev_message, info, Opts)), + ?assert(hb_ao:is_exported(Msg, dev_message, set, Opts)), + ?assert( + hb_ao:is_exported( + Msg, + dev_message, + not_explicitly_exported, + Opts + ) + ), + Dev = #{ + info => fun() -> #{ exports => [set] } end, + set => fun(_, _) -> {ok, <<"SET">>} end + }, + Msg2 = #{ <<"device">> => Dev }, + ?assert(hb_ao:is_exported(Msg2, Dev, info, Opts)), + ?assert(hb_ao:is_exported(Msg2, Dev, set, Opts)), + ?assert(not hb_ao:is_exported(Msg2, Dev, not_exported, Opts)), + Dev2 = #{ + info => + fun() -> + #{ + exports => [test1, <<"test2">>], + handler => + fun() -> + {ok, <<"Handler-Value">>} + end + } + end + }, + Msg3 = #{ <<"device">> => Dev2, <<"test1">> => <<"BAD1">>, <<"test3">> => <<"GOOD3">> }, + ?assertEqual(<<"Handler-Value">>, hb_ao:get(<<"test1">>, Msg3, Opts)), + ?assertEqual(<<"Handler-Value">>, hb_ao:get(<<"test2">>, Msg3, Opts)), + ?assertEqual(<<"GOOD3">>, hb_ao:get(<<"test3">>, Msg3, Opts)), + ?assertEqual(<<"GOOD4">>, + hb_ao:get( + <<"test4">>, + hb_ao:set(Msg3, <<"test4">>, <<"GOOD4">>, Opts) + ) + ), + ?assertEqual(not_found, hb_ao:get(<<"test5">>, Msg3, Opts)). +``` + +### device_excludes_test + +```erlang +device_excludes_test(Opts) -> + % Create a device that returns an identifiable message for any key, but also + % sets excludes to [set], such that the message can be modified using the + % default handler. +``` + +### denormalized_device_key_test + +```erlang +denormalized_device_key_test(Opts) -> + Msg = #{ <<"device">> => dev_test }, + ?assertEqual(dev_test, hb_ao:get(device, Msg, Opts)), + ?assertEqual(dev_test, hb_ao:get(<<"device">>, Msg, Opts)), + ?assertEqual({module, dev_test}, + erlang:fun_info( + element(3, hb_ao:message_to_fun(Msg, test_func, Opts)), + module + ) + ). +``` + +### list_transform_test + +```erlang +list_transform_test(Opts) -> + Msg = [<<"A">>, <<"B">>, <<"C">>, <<"D">>, <<"E">>], + ?assertEqual(<<"A">>, hb_ao:get(1, Msg, Opts)), + ?assertEqual(<<"B">>, hb_ao:get(2, Msg, Opts)), + ?assertEqual(<<"C">>, hb_ao:get(3, Msg, Opts)), + ?assertEqual(<<"D">>, hb_ao:get(4, Msg, Opts)), + ?assertEqual(<<"E">>, hb_ao:get(5, Msg, Opts)). +``` + +### start_as_test + +```erlang +start_as_test(Opts) -> + ?assertEqual( + {ok, <<"GOOD_FUNCTION">>}, + hb_ao:resolve_many( + [ + {as, <<"test-device@1.0">>, #{ <<"path">> => <<>> }}, + #{ <<"path">> => <<"test_func">> } + ], + Opts + ) + ). +``` + +### start_as_with_parameters_test + +```erlang +start_as_with_parameters_test(Opts) -> + % Resolve a key on a message that has its device set with `as'. +``` + +### load_as_test + +```erlang +load_as_test(Opts) -> + % Load a message as a device with the `as' keyword. +``` + +### as_path_test + +```erlang +as_path_test(Opts) -> + % Create a message with the test device, which implements the test_func + % function. It normally returns `GOOD_FUNCTION'. +``` + +### continue_as_test + +```erlang +continue_as_test(Opts) -> + % Resolve a list of messages in sequence, swapping the device in the middle. +``` + +### multiple_as_subresolutions_test + +```erlang +multiple_as_subresolutions_test(Opts) -> + % Test that multiple as subresolutions in a sequence are handled correctly. +``` + +### step_hook_test + +```erlang +step_hook_test(InitOpts) -> + % Test that the step hook is called correctly. We do this by sending ourselves + % a message each time the hook is called. We also send a `reference', such + % that this test is uniquely identified and further/prior tests do not affect + % it. +``` + +### benchmark_simple_test + +```erlang +benchmark_simple_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> hb_ao:resolve(#{ <<"a">> => I }, <<"a">>, Opts) end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Single-step resolutions:">>, + ?BENCHMARK_ITERATIONS, + Time + ). +``` + +### benchmark_multistep_test + +```erlang +benchmark_multistep_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:resolve( + #{ + <<"iteration">> => I, + <<"a">> => #{ + <<"b">> => #{ <<"return">> => I } + } + }, + <<"a/b/return">>, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Multistep resolutions:">>, + ?BENCHMARK_ITERATIONS, + Time + ). +``` + +### benchmark_get_test + +```erlang +benchmark_get_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:get( + <<"a">>, + #{ <<"a">> => <<"1">>, <<"iteration">> => I }, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Get operations:">>, + ?BENCHMARK_ITERATIONS, + Time + ). +``` + +### benchmark_set_test + +```erlang +benchmark_set_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:set( + #{ <<"a">> => <<"1">>, <<"iteration">> => I }, + <<"a">>, + <<"2">>, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Single value set operations:">>, + ?BENCHMARK_ITERATIONS, + Time + ). +``` + +### benchmark_set_multiple_test + +```erlang +benchmark_set_multiple_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:set( + #{ <<"a">> => <<"1">>, <<"iteration">> => I }, + #{ <<"a">> => <<"1a">>, <<"b">> => <<"2">> }, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Set two keys operations:">>, + ?BENCHMARK_ITERATIONS, + Time + ). +``` + +### benchmark_set_multiple_deep_test + +```erlang +benchmark_set_multiple_deep_test(Opts) -> + Time = + hb_test_utils:benchmark_iterations( + fun(I) -> + hb_ao:set( + #{ <<"a">> => #{ <<"b">> => <<"1">> } }, + #{ <<"a">> => #{ <<"b">> => <<"2">>, <<"c">> => I } }, + Opts + ) + end, + ?BENCHMARK_ITERATIONS + ), + hb_test_utils:benchmark_print( + <<"Set two keys operations:">>, + ?BENCHMARK_ITERATIONS, + Time +``` + +--- + +*Generated from [hb_ao_test_vectors.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_ao_test_vectors.erl)* diff --git a/docs/book/src/hb_app.erl.md b/docs/book/src/hb_app.erl.md new file mode 100644 index 000000000..5ad52cd50 --- /dev/null +++ b/docs/book/src/hb_app.erl.md @@ -0,0 +1,37 @@ +# hb_app + +[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_app.erl) + +The main HyperBEAM application module. + +--- + +## Exported Functions + +- `start/2` +- `stop/1` + +--- + +### start + +The main HyperBEAM application module. + +```erlang +start(_StartType, _StartArgs) -> + hb:init(), + hb_sup:start_link(), + ok = dev_scheduler_registry:start(), + _TimestampServer = ar_timestamp:start(), + {ok, _} = hb_http_server:start(). +``` + +### stop + +```erlang +stop(_State) -> +``` + +--- + +*Generated from [hb_app.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_app.erl)* diff --git a/docs/book/src/hb_beamr.erl.md b/docs/book/src/hb_beamr.erl.md new file mode 100644 index 000000000..bc2f18a17 --- /dev/null +++ b/docs/book/src/hb_beamr.erl.md @@ -0,0 +1,443 @@ +# hb_beamr + +[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_beamr.erl) + +BEAMR: A WAMR wrapper for BEAM. +Beamr is a library that allows you to run WASM modules in BEAM, using the +Webassembly Micro Runtime (WAMR) as its engine. Each WASM module is +executed using a Linked-In Driver (LID) that is loaded into BEAM. It is +designed with a focus on supporting long-running WASM executions that +interact with Erlang functions and processes easily. +Because each WASM module runs as an independent async worker, if you plan +to run many instances in parallel, you should be sure to configure the +BEAM to have enough async worker threads enabled (see `erl +A N` in the +Erlang manuals). +The core API is simple: +`. +
+ start(WasmBinary) -> {ok, Port, Imports, Exports}
+ Where:
+ WasmBinary is the WASM binary to load.
+ Port is the port to the LID.
+ Imports is a list of tuples of the form {Module, Function,
+ Args, Signature}.
+ Exports is a list of tuples of the form {Function, Args,
+ Signature}.
+ stop(Port) -> ok
+ call(Port, FunctionName, Args) -> {ok, Result}
+ Where:
+ FunctionName is the name of the function to call.
+ Args is a list of Erlang terms (converted to WASM values by
+ BEAMR) that match the signature of the function.
+ Result is a list of Erlang terms (converted from WASM values).
+ call(Port, FunName, Args[, Import, State, Opts]) -> {ok, Res, NewState}
+ Where:
+ ImportFun is a function that will be called upon each import.
+ ImportFun must have an arity of 2: Taking an arbitrary `state`
+ term, and a map containing the `port`, `module`, `func`, `args`,
+ `signature`, and the `options` map of the import.
+ It must return a tuple of the form {ok, Response, NewState}.
+ serialize(Port) -> {ok, Mem}
+ Where:
+ Port is the port to the LID.
+ Mem is a binary representing the full WASM state.
+ deserialize(Port, Mem) -> ok
+ Where:
+ Port is the port to the LID.
+ Mem is a binary output of a previous `serialize/1` call.
+
+BEAMR was designed for use in the HyperBEAM project, but is suitable for
+deployment in other Erlang applications that need to run WASM modules. PRs
+are welcome.
+
+---
+
+## Exported Functions
+
+- `call/3`
+- `call/4`
+- `call/5`
+- `call/6`
+- `deserialize/2`
+- `serialize/1`
+- `start/1`
+- `start/2`
+- `stop/1`
+- `stub/3`
+- `wasm_send/2`
+
+---
+
+### load_driver
+
+BEAMR: A WAMR wrapper for BEAM.
+Load the driver for the WASM executor.
+
+```erlang
+load_driver() ->
+ case erl_ddll:load(code:priv_dir(hb), ?MODULE) of
+ ok -> ok;
+ {error, already_loaded} -> ok;
+ {error, Error} -> {error, Error}
+ end.
+```
+
+### start
+
+Start a WASM executor context. Yields a port to the LID, and the
+
+```erlang
+start(WasmBinary) when is_binary(WasmBinary) ->
+ start(WasmBinary, wasm).
+```
+
+### start
+
+```erlang
+start(WasmBinary, Mode) when is_binary(WasmBinary) ->
+ ?event({loading_module, {bytes, byte_size(WasmBinary)}, Mode}),
+ Self = self(),
+ WASM = spawn(
+ fun() ->
+ ok = load_driver(),
+ Port = open_port({spawn, "hb_beamr"}, []),
+ Port ! {self(), {command, term_to_binary({init, WasmBinary, Mode})}},
+ ?event({waiting_for_init_from, Port}),
+ worker(Port, Self)
+ end
+ ),
+ receive
+ {execution_result, Imports, Exports} ->
+ ?event(
+ {wasm_init_success,
+ {imports, Imports},
+ {exports, Exports}}),
+ {ok, WASM, Imports, Exports};
+ {error, Error} ->
+ ?event({wasm_init_error, Error}),
+ stop(WASM),
+ {error, Error}
+ end.
+```
+
+### worker
+
+A worker process that is responsible for handling a WASM instance.
+
+```erlang
+worker(Port, Listener) ->
+ receive
+ stop ->
+ ?event({stop_invoked_for_beamr, self()}),
+ case erlang:port_info(Port, id) of
+ undefined ->
+ ok;
+ _ ->
+ port_close(Port),
+ ok
+ end,
+ ok;
+ {wasm_send, NewListener, Message} ->
+ ?event({wasm_send, {listener, NewListener}, {message, Message}}),
+ Port ! {self(), Message},
+ worker(Port, NewListener);
+ WASMResult ->
+ ?event({wasm_result, {listener, Listener}, {result, WASMResult}}),
+ Listener ! WASMResult,
+ worker(Port, Listener)
+ end.
+```
+
+### wasm_send
+
+```erlang
+wasm_send(WASM, Message) when is_pid(WASM) ->
+ WASM ! {wasm_send, self(), Message},
+ ok.
+```
+
+### stop
+
+Stop a WASM executor context.
+
+```erlang
+stop(WASM) when is_pid(WASM) ->
+ WASM ! stop,
+ ok.
+```
+
+### call
+
+Call a function in the WASM executor (see moduledoc for more details).
+
+```erlang
+call(PID, FuncRef, Args) ->
+ {ok, Res, _} = call(PID, FuncRef, Args, fun stub/3),
+ {ok, Res}.
+```
+
+### call
+
+```erlang
+call(PID, FuncRef, Args, ImportFun) ->
+ call(PID, FuncRef, Args, ImportFun, #{}).
+```
+
+### call
+
+```erlang
+call(PID, FuncRef, Args, ImportFun, StateMsg) ->
+ call(PID, FuncRef, Args, ImportFun, StateMsg, #{}).
+```
+
+### call
+
+```erlang
+call(PID, FuncRef, Args, ImportFun, StateMsg, Opts)
+ when is_binary(FuncRef) ->
+ call(PID, binary_to_list(FuncRef), Args, ImportFun, StateMsg, Opts);
+```
+
+### call
+
+```erlang
+call(WASM, FuncRef, Args, ImportFun, StateMsg, Opts)
+ when is_pid(WASM)
+ andalso (is_list(FuncRef) or is_integer(FuncRef))
+ andalso is_list(Args)
+ andalso is_function(ImportFun)
+ andalso is_map(Opts) ->
+ case is_valid_arg_list(Args) of
+ true ->
+ ?event(
+ {call_started,
+ WASM,
+ FuncRef,
+ Args,
+ ImportFun,
+ StateMsg,
+ Opts}),
+ wasm_send(WASM,
+ {command,
+ term_to_binary(
+ case is_integer(FuncRef) of
+ true -> {indirect_call, FuncRef, Args};
+ false -> {call, FuncRef, Args}
+ end
+ )
+ }
+ ),
+ ?event({waiting_for_call_result, self(), WASM}),
+ monitor_call(WASM, ImportFun, StateMsg, Opts);
+ false ->
+ {error, {invalid_args, Args}}
+ end.
+```
+
+### stub
+
+Stub import function for the WASM executor.
+
+```erlang
+stub(Msg1, _Msg2, _Opts) ->
+ ?event(stub_stdlib_called),
+ {ok, [0], Msg1}.
+```
+
+### monitor_call
+
+Synchonously monitor the WASM executor for a call result and any
+
+```erlang
+monitor_call(WASM, ImportFun, StateMsg, Opts) ->
+ receive
+ {execution_result, Result} ->
+ ?event({call_result, Result}),
+ {ok, Result, StateMsg};
+ {import, Module, Func, Args, Signature} ->
+ ?event({import_called, Module, Func, Args, Signature}),
+ try
+ {ok, Res, StateMsg2} =
+ ImportFun(StateMsg,
+ #{
+ instance => WASM,
+ module => Module,
+ func => Func,
+ args => Args,
+ func_sig => Signature
+ },
+ Opts
+ ),
+ ?event({import_ret, Module, Func, {args, Args}, {res, Res}}),
+ dispatch_response(WASM, Res),
+ monitor_call(WASM, ImportFun, StateMsg2, Opts)
+ catch
+ Err:Reason:Stack ->
+ % Signal the WASM executor to stop.
+```
+
+### dispatch_response
+
+Check the type of an import response and dispatch it to a Beamr port.
+
+```erlang
+dispatch_response(WASM, Term) when is_pid(WASM) ->
+ case is_valid_arg_list(Term) of
+ true ->
+ wasm_send(WASM,
+ {command, term_to_binary({import_response, Term})});
+ false ->
+ throw({error, {invalid_response, Term}})
+ end;
+```
+
+### dispatch_response
+
+Check the type of an import response and dispatch it to a Beamr port.
+
+```erlang
+dispatch_response(_WASM, Term) ->
+ throw({error, {invalid_response, Term}}).
+```
+
+### is_valid_arg_list
+
+Check that a list of arguments is valid for a WASM function call.
+
+```erlang
+is_valid_arg_list(Args) when is_list(Args) ->
+ lists:all(fun(Arg) -> is_integer(Arg) or is_float(Arg) end, Args);
+```
+
+### is_valid_arg_list
+
+Check that a list of arguments is valid for a WASM function call.
+
+```erlang
+is_valid_arg_list(_) ->
+ false.
+```
+
+### serialize
+
+Serialize the WASM state to a binary.
+
+```erlang
+serialize(WASM) when is_pid(WASM) ->
+ ?event(starting_serialize),
+ {ok, Size} = hb_beamr_io:size(WASM),
+ ?event({image_size, Size}),
+ {ok, Mem} = hb_beamr_io:read(WASM, 0, Size),
+ ?event({finished_serialize, byte_size(Mem)}),
+ {ok, Mem}.
+```
+
+### deserialize
+
+Deserialize a WASM state from a binary.
+
+```erlang
+deserialize(WASM, Bin) when is_pid(WASM) andalso is_binary(Bin) ->
+ ?event(starting_deserialize),
+ Res = hb_beamr_io:write(WASM, 0, Bin),
+ ?event({finished_deserialize, Res}),
+ ok.
+```
+
+### driver_loads_test
+
+```erlang
+driver_loads_test() ->
+ ?assertEqual(ok, load_driver()).
+```
+
+### simple_wasm_test
+
+Test standalone `hb_beamr` correctly after loading a WASM module.
+
+```erlang
+simple_wasm_test() ->
+ {ok, File} = file:read_file("test/test.wasm"),
+ {ok, WASM, _Imports, _Exports} = start(File),
+ {ok, [Result]} = call(WASM, "fac", [5.0]),
+ ?assertEqual(120.0, Result).
+```
+
+### imported_function_test
+
+Test that imported functions can be called from the WASM module.
+
+```erlang
+imported_function_test() ->
+ {ok, File} = file:read_file("test/pow_calculator.wasm"),
+ {ok, WASM, _Imports, _Exports} = start(File),
+ {ok, [Result], _} =
+ call(WASM, <<"pow">>, [2, 5],
+ fun(Msg1, #{ args := [Arg1, Arg2] }, _Opts) ->
+ {ok, [Arg1 * Arg2], Msg1}
+ end),
+ ?assertEqual(32, Result).
+```
+
+### wasm64_test
+
+Test that WASM Memory64 modules load and execute correctly.
+
+```erlang
+wasm64_test() ->
+ {ok, File} = file:read_file("test/test-64.wasm"),
+ {ok, WASM, _ImportMap, _Exports} = start(File),
+ {ok, [Result]} = call(WASM, "fac", [5.0]),
+ ?assertEqual(120.0, Result).
+```
+
+### multiclient_test
+
+Ensure that processes outside of the initial one can interact with
+
+```erlang
+multiclient_test() ->
+ Self = self(),
+ ExecPID = spawn(fun() ->
+ receive {wasm, WASM} ->
+ {ok, [Result]} = call(WASM, "fac", [5.0]),
+ Self ! {result, Result}
+ end
+ end),
+ _StartPID = spawn(fun() ->
+ {ok, File} = file:read_file("test/test.wasm"),
+ {ok, WASM, _ImportMap, _Exports} = start(File),
+ ExecPID ! {wasm, WASM}
+ end),
+ receive
+ {result, Result} ->
+ ?assertEqual(120.0, Result)
+ end.
+```
+
+### benchmark_test
+
+```erlang
+benchmark_test() ->
+ BenchTime = 1,
+ {ok, File} = file:read_file("test/test-64.wasm"),
+ {ok, WASM, _ImportMap, _Exports} = start(File),
+ Iterations = hb_test_utils:benchmark(
+ fun() ->
+ {ok, [Result]} = call(WASM, "fac", [5.0]),
+ ?assertEqual(120.0, Result)
+ end,
+ BenchTime
+ ),
+ ?event(benchmark, {scheduled, Iterations}),
+ ?assert(Iterations > 1000),
+ hb_test_utils:benchmark_print(
+ <<"Direct beamr: Executed">>,
+ <<"calls">>,
+ Iterations,
+ BenchTime
+ ),
+```
+
+---
+
+*Generated from [hb_beamr.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_beamr.erl)*
diff --git a/docs/book/src/hb_beamr_io.erl.md b/docs/book/src/hb_beamr_io.erl.md
new file mode 100644
index 000000000..181ac9e38
--- /dev/null
+++ b/docs/book/src/hb_beamr_io.erl.md
@@ -0,0 +1,238 @@
+# hb_beamr_io
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_beamr_io.erl)
+
+Simple interface for memory management for Beamr instances.
+It allows for reading and writing to memory, as well as allocating and
+freeing memory by calling the WASM module's exported malloc and free
+functions.
+Unlike the majority of HyperBEAM modules, this module takes a defensive
+approach to type checking, breaking from the conventional Erlang style,
+such that failures are caught in the Erlang-side of functions rather than
+in the C/WASM-side.
+
+---
+
+## Exported Functions
+
+- `free/2`
+- `malloc/2`
+- `read_string/2`
+- `read/3`
+- `size/1`
+- `write_string/2`
+- `write/3`
+
+---
+
+### size
+
+Simple interface for memory management for Beamr instances.
+Get the size (in bytes) of the native memory allocated in the Beamr
+
+```erlang
+size(WASM) when is_pid(WASM) ->
+ hb_beamr:wasm_send(WASM, {command, term_to_binary({size})}),
+ receive
+ {execution_result, Size} ->
+ {ok, Size}
+ end.
+```
+
+### write
+
+Write a binary to the Beamr instance's native memory at a given offset.
+
+```erlang
+write(WASM, Offset, Data)
+ when is_pid(WASM)
+ andalso is_binary(Data)
+ andalso is_integer(Offset) ->
+ ?event(writing_to_mem),
+ hb_beamr:wasm_send(WASM, {command, term_to_binary({write, Offset, Data})}),
+ ?event(mem_written),
+ receive
+ ok -> ok;
+ {error, Error} -> {error, Error}
+ end.
+```
+
+### write_string
+
+Simple helper function to allocate space for (via malloc) and write a
+
+```erlang
+write_string(WASM, Data) when is_pid(WASM) andalso is_list(Data) ->
+ write_string(WASM, iolist_to_binary(Data));
+```
+
+### write_string
+
+Simple helper function to allocate space for (via malloc) and write a
+
+```erlang
+write_string(WASM, Data) when is_pid(WASM) andalso is_binary(Data) ->
+ DataSize = byte_size(Data) + 1,
+ String = <>,
+ case malloc(WASM, DataSize) of
+ {ok, Ptr} ->
+ case write(WASM, Ptr, String) of
+ ok -> {ok, Ptr};
+ {error, Error} -> {error, Error}
+ end;
+ Error -> Error
+ end.
+```
+
+### read
+
+Read a binary from the Beamr instance's native memory at a given offset
+
+```erlang
+read(WASM, Offset, Size)
+ when is_pid(WASM)
+ andalso is_integer(Offset)
+ andalso is_integer(Size) ->
+ ?event({read_request, {port, WASM}, {location, Offset}, {size, Size}}),
+ hb_beamr:wasm_send(WASM, {command, term_to_binary({read, Offset, Size})}),
+ ?event(read_req_sent),
+ receive
+ {execution_result, Result} ->
+ ?event(
+ {read_result,
+ {wasm, WASM},
+ {location, Offset},
+ {size, Size},
+ {result, Result}}),
+ {ok, Result};
+ {error, Error} ->
+ {error, Error}
+ end.
+```
+
+### read_string
+
+Simple helper function to read a string from the Beamr instance's native
+
+```erlang
+read_string(Port, Offset) -> read_string(Port, Offset, 8).
+```
+
+### read_string
+
+Simple helper function to read a string from the Beamr instance's native
+
+```erlang
+read_string(WASM, Offset, ChunkSize)
+ when is_pid(WASM)
+ andalso is_integer(Offset)
+ andalso is_integer(ChunkSize) ->
+ {ok, iolist_to_binary(do_read_string(WASM, Offset, ChunkSize))}.
+```
+
+### do_read_string
+
+```erlang
+do_read_string(WASM, Offset, ChunkSize) ->
+ {ok, Data} = read(WASM, Offset, ChunkSize),
+ case binary:split(Data, [<<0>>]) of
+ [Data|[]] -> [Data|do_read_string(WASM, Offset + ChunkSize, ChunkSize)];
+ [FinalData|_Remainder] -> [FinalData]
+ end.
+```
+
+### malloc
+
+Allocate space for (via an exported malloc function from the WASM) in
+
+```erlang
+malloc(WASM, Size) when is_pid(WASM) andalso is_integer(Size) ->
+ case hb_beamr:call(WASM, "malloc", [Size]) of
+ {ok, [0]} ->
+ ?event({malloc_failed, Size}),
+ {error, malloc_failed};
+ {ok, [Ptr]} ->
+ ?event({malloc_success, Ptr, Size}),
+ {ok, Ptr};
+ {error, Error} ->
+ {error, Error}
+ end.
+```
+
+### free
+
+Free space allocated in the Beamr instance's native memory via a
+
+```erlang
+free(WASM, Ptr) when is_pid(WASM) andalso is_integer(Ptr) ->
+ case hb_beamr:call(WASM, "free", [Ptr]) of
+ {ok, Res} ->
+ ?event({free_result, Res}),
+ ok;
+ {error, Error} ->
+ {error, Error}
+ end.
+```
+
+### size_test
+
+```erlang
+size_test() ->
+ WASMPageSize = 65536,
+ File1Pages = 1,
+ File2Pages = 193,
+ {ok, File} = file:read_file("test/test-print.wasm"),
+ {ok, WASM, _Imports, _Exports} = hb_beamr:start(File),
+ ?assertEqual({ok, WASMPageSize * File1Pages}, hb_beamr_io:size(WASM)),
+ hb_beamr:stop(WASM),
+ {ok, File2} = file:read_file("test/aos-2-pure-xs.wasm"),
+ {ok, WASM2, _Imports2, _Exports2} = hb_beamr:start(File2),
+ ?assertEqual({ok, WASMPageSize * File2Pages}, hb_beamr_io:size(WASM2)),
+ hb_beamr:stop(WASM2).
+```
+
+### write_test
+
+Test writing memory in and out of bounds.
+
+```erlang
+write_test() ->
+ % Load the `test-print' WASM module, which has a simple print function.
+```
+
+### read_test
+
+Test reading memory in and out of bounds.
+
+```erlang
+read_test() ->
+ % Our `test-print' module is hand-written in WASM, so we know that it
+ % has a `Hello, World!` string at precisely offset 66.
+```
+
+### malloc_test
+
+Test allocating and freeing memory.
+
+```erlang
+malloc_test() ->
+ {ok, File} = file:read_file("test/test-calling.wasm"),
+ {ok, WASM, _Imports, _Exports} = hb_beamr:start(File),
+ % Check that we can allocate memory inside the bounds of the WASM module.
+```
+
+### string_write_and_read_test
+
+Write and read strings to memory.
+
+```erlang
+string_write_and_read_test() ->
+ {ok, File} = file:read_file("test/test-calling.wasm"),
+ {ok, WASM, _Imports, _Exports} = hb_beamr:start(File),
+ {ok, Ptr} = write_string(WASM, <<"Hello, World!">>),
+ ?assertEqual({ok, <<"Hello, World!">>}, read_string(WASM, Ptr)).
+```
+
+---
+
+*Generated from [hb_beamr_io.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_beamr_io.erl)*
diff --git a/docs/book/src/hb_cache.erl.md b/docs/book/src/hb_cache.erl.md
new file mode 100644
index 000000000..356becb5d
--- /dev/null
+++ b/docs/book/src/hb_cache.erl.md
@@ -0,0 +1,967 @@
+# hb_cache
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_cache.erl)
+
+A cache of AO-Core protocol messages and compute results.
+HyperBEAM stores all paths in key value stores, abstracted by the `hb_store`
+module. Each store has its own storage backend, but each works with simple
+key-value pairs. Each store can write binary keys at paths, and link between
+paths.
+There are three layers to HyperBEAMs internal data representation on-disk:
+1. The raw binary data, written to the store at the hash of the content.
+ Storing binary paths in this way effectively deduplicates the data.
+2. The hashpath-graph of all content, stored as a set of links between
+ hashpaths, their keys, and the data that underlies them. This allows
+ all messages to share the same hashpath space, such that all requests
+ from users additively fill-in the hashpath space, minimizing duplicated
+ compute.
+3. Messages, referrable by their IDs (committed or uncommitted). These are
+ stored as a set of links commitment IDs and the uncommitted message.
+Before writing a message to the store, we convert it to Type-Annotated
+Binary Messages (TABMs), such that each of the keys in the message is
+either a map or a direct binary.
+Nested keys are lazily loaded from the stores, such that large deeply
+nested messages where only a small part of the data is actually used are
+not loaded into memory unnecessarily. In order to ensure that a message is
+loaded from the cache after a `read`, we can use the `ensure_loaded/1` and
+`ensure_all_loaded/1` functions. Ensure loaded will load the exact value
+that has been requested, while ensure all loaded will load the entire
+structure of the message into memory.
+Lazily loadable `links` are expressed as a tuple of the following form:
+`{link, ID, LinkOpts}`, where `ID` is the path to the data in the store,
+and `LinkOpts` is a map of suggested options to use when loading the data.
+In particular, this module ensures to stash the `store` option in `LinkOpts`,
+such that the `read` function can use the correct store without having to
+search unnecessarily. By providing an `Opts` argument to `ensure_loaded` or
+`ensure_all_loaded`, the caller can specify additional options to use when
+loading the data -- overriding the suggested options in the link.
+
+---
+
+## Exported Functions
+
+- `ensure_all_loaded/1`
+- `ensure_all_loaded/2`
+- `ensure_loaded/1`
+- `ensure_loaded/2`
+- `link/3`
+- `list_numbered/2`
+- `list/2`
+- `match/2`
+- `read_resolved/3`
+- `read/2`
+- `test_signed/1`
+- `test_unsigned/1`
+- `write_binary/3`
+- `write_hashpath/2`
+- `write/2`
+
+---
+
+### ensure_loaded
+
+A cache of AO-Core protocol messages and compute results.
+Ensure that a value is loaded from the cache if it is an ID or a link.
+
+```erlang
+ensure_loaded(Msg) ->
+ ensure_loaded(Msg, #{}).
+```
+
+### ensure_loaded
+
+```erlang
+ensure_loaded(Msg, Opts) ->
+ ensure_loaded([], Msg, Opts).
+```
+
+### ensure_loaded
+
+```erlang
+ensure_loaded(Ref, {Status, Msg}, Opts) when Status == ok; Status == error ->
+ {Status, ensure_loaded(Ref, Msg, Opts)};
+```
+
+### ensure_loaded
+
+```erlang
+ensure_loaded(Ref,
+ Lk = {link, ID, LkOpts = #{ <<"type">> := <<"link">>, <<"lazy">> := Lazy }},
+ RawOpts) ->
+ % The link is to a submessage; either in lazy (unresolved) form, or direct
+ % form.
+```
+
+### ensure_loaded
+
+```erlang
+ensure_loaded(Ref, Link = {link, ID, LinkOpts = #{ <<"lazy">> := true }}, RawOpts) ->
+ % If the user provided their own options, we merge them and _overwrite_
+ % the options that are already set in the link.
+```
+
+### ensure_loaded
+
+```erlang
+ensure_loaded(Ref, {link, ID, LinkOpts}, Opts) ->
+ ensure_loaded(Ref, {link, ID, LinkOpts#{ <<"lazy">> => true}}, Opts);
+```
+
+### ensure_loaded
+
+```erlang
+ensure_loaded(_Ref, Msg, _Opts) when not ?IS_LINK(Msg) ->
+ Msg.
+```
+
+### report_ensure_loaded_not_found
+
+Report that a value was not found in the cache. If a key is provided,
+
+```erlang
+report_ensure_loaded_not_found(Ref, Lk, Opts) ->
+ ?event(link_error, {link_not_resolvable, {ref, Ref}, {link, Lk}, {opts, Opts}}),
+ throw(
+ {necessary_message_not_found,
+ hb_path:to_binary(lists:reverse(Ref)),
+ hb_link:format_unresolved(Lk, Opts, 0)
+ }
+ ).
+```
+
+### ensure_all_loaded
+
+Ensure that all of the components of a message (whether a map, list,
+
+```erlang
+ensure_all_loaded(Msg) ->
+ ensure_all_loaded(Msg, #{}).
+```
+
+### ensure_all_loaded
+
+```erlang
+ensure_all_loaded(Msg, Opts) ->
+ ensure_all_loaded([], Msg, Opts).
+```
+
+### ensure_all_loaded
+
+```erlang
+ensure_all_loaded(Ref, Link, Opts) when ?IS_LINK(Link) ->
+ ensure_all_loaded(Ref, ensure_loaded(Ref, Link, Opts), Opts);
+```
+
+### ensure_all_loaded
+
+```erlang
+ensure_all_loaded(Ref, Msg, Opts) when is_map(Msg) ->
+ maps:map(fun(K, V) -> ensure_all_loaded([K|Ref], V, Opts) end, Msg);
+```
+
+### ensure_all_loaded
+
+```erlang
+ensure_all_loaded(Ref, Msg, Opts) when is_list(Msg) ->
+ lists:map(
+ fun({N, V}) -> ensure_all_loaded([N|Ref], V, Opts) end,
+ hb_util:number(Msg)
+ );
+```
+
+### ensure_all_loaded
+
+```erlang
+ensure_all_loaded(Ref, Msg, Opts) ->
+ ensure_loaded(Ref, Msg, Opts).
+```
+
+### list_numbered
+
+List all items in a directory, assuming they are numbered.
+
+```erlang
+list_numbered(Path, Opts) ->
+ SlotDir = hb_store:path(hb_opts:get(store, no_viable_store, Opts), Path),
+ [ hb_util:int(Name) || Name <- list(SlotDir, Opts) ].
+```
+
+### list
+
+List all items under a given path.
+
+```erlang
+list(Path, Opts) when is_map(Opts) and not is_map_key(<<"store-module">>, Opts) ->
+ case hb_opts:get(store, no_viable_store, Opts) of
+ not_found -> [];
+ Store ->
+ list(Path, Store)
+ end;
+```
+
+### list
+
+List all items under a given path.
+
+```erlang
+list(Path, Store) ->
+ ResolvedPath = hb_store:resolve(Store, Path),
+ case hb_store:list(Store, ResolvedPath) of
+ {ok, Names} -> Names;
+ {error, _} -> [];
+ not_found -> []
+ end.
+```
+
+### match
+
+Match a template message against the cache, returning a list of IDs
+
+```erlang
+match(MatchSpec, Opts) ->
+ Spec = hb_message:convert(MatchSpec, tabm, <<"structured@1.0">>, Opts),
+ ConvertedMatchSpec =
+ maps:map(
+ fun(_, Value) ->
+ generate_binary_path(Value, Opts)
+ end,
+ maps:without([<<"ao-types">>], hb_ao:normalize_keys(Spec, Opts))
+ ),
+ case hb_store:match(hb_opts:get(store, no_viable_store, Opts), ConvertedMatchSpec) of
+ {ok, Matches} -> {ok, Matches};
+ _ -> not_found
+ end.
+```
+
+### generate_binary_path
+
+Generate the path at which a binary value should be stored.
+Write a message to the cache. For raw binaries, we write the data at
+
+```erlang
+generate_binary_path(Bin, Opts) ->
+ Hashpath = hb_path:hashpath(Bin, Opts),
+ <<"data/", Hashpath/binary>>.
+```
+
+### write
+
+Generate the path at which a binary value should be stored.
+Write a message to the cache. For raw binaries, we write the data at
+
+```erlang
+write(RawMsg, Opts) when is_map(RawMsg) ->
+ {ok, Msg} = hb_message:with_only_committed(RawMsg, Opts),
+ TABM = hb_message:convert(Msg, tabm, <<"structured@1.0">>, Opts),
+ ?event(debug_cache, {writing_full_message, {msg, TABM}}),
+ try
+ do_write_message(
+ TABM,
+ hb_opts:get(store, no_viable_store, Opts),
+ Opts
+ )
+ catch
+ Type:Reason:Stacktrace ->
+ ?event(error,
+ {cache_write_error,
+ {type, Type},
+ {reason, Reason},
+ {stacktrace, {trace, Stacktrace}}
+ },
+ Opts
+ ),
+ erlang:raise(Type, Reason, Stacktrace)
+ end;
+```
+
+### write
+
+Generate the path at which a binary value should be stored.
+Write a message to the cache. For raw binaries, we write the data at
+
+```erlang
+write(List, Opts) when is_list(List) ->
+ write(hb_message:convert(List, tabm, <<"structured@1.0">>, Opts), Opts);
+```
+
+### write
+
+Generate the path at which a binary value should be stored.
+Write a message to the cache. For raw binaries, we write the data at
+
+```erlang
+write(Bin, Opts) when is_binary(Bin) ->
+ do_write_message(Bin, hb_opts:get(store, no_viable_store, Opts), Opts).
+```
+
+### do_write_message
+
+```erlang
+do_write_message(Bin, Store, Opts) when is_binary(Bin) ->
+ % Write the binary in the store at its calculated content-hash.
+```
+
+### do_write_message
+
+```erlang
+do_write_message(List, Store, Opts) when is_list(List) ->
+ do_write_message(
+ hb_message:convert(List, tabm, <<"structured@1.0">>, Opts),
+ Store,
+ Opts
+ );
+```
+
+### do_write_message
+
+```erlang
+do_write_message(Msg, Store, Opts) when is_map(Msg) ->
+ ?event(debug_cache, {writing_message, Msg}),
+ % Calculate the IDs of the message.
+```
+
+### write_key
+
+Write a single key for a message into the store.
+
+```erlang
+write_key(Base, <<"commitments">>, _HPAlg, RawCommitments, Store, Opts) ->
+ % The commitments are a special case: We calculate the single-part hashpath
+ % for the `baseID/commitments` key, then write each commitment to the store
+ % and link it to `baseCommHP/commitmentID`.
+```
+
+### write_key
+
+```erlang
+write_key(Base, Key, HPAlg, Value, Store, Opts) ->
+ KeyHashPath =
+ hb_path:hashpath(
+ Base,
+ hb_path:to_binary(Key),
+ HPAlg,
+ Opts
+ ),
+ {ok, Path} = do_write_message(Value, Store, Opts),
+ hb_store:make_link(Store, Path, KeyHashPath),
+ {ok, Path}.
+```
+
+### prepare_commitments
+
+The `structured@1.0` encoder does not typically encode `commitments`,
+
+```erlang
+prepare_commitments(RawCommitments, Opts) ->
+ Commitments = ensure_all_loaded(RawCommitments, Opts),
+ maps:map(
+ fun(_, StructuredCommitment) ->
+ hb_message:convert(StructuredCommitment, tabm, Opts)
+ end,
+ Commitments
+ ).
+```
+
+### commitment_path
+
+Generate the commitment path for a given base path.
+Calculate the IDs for a message.
+
+```erlang
+commitment_path(Base, Opts) ->
+ hb_path:hashpath(<+ Arweave TX/ANS-104 ==> dev_codec_ans104:from/1 ==> TABM + HTTP Signed Message ==> dev_codec_httpsig_conv:from/1 ==> TABM + Flat Maps ==> dev_codec_flat:from/1 ==> TABM + TABM ==> dev_codec_structured:to/1 ==> AO-Core Message + AO-Core Message ==> dev_codec_structured:from/1 ==> TABM + TABM ==> dev_codec_ans104:to/1 ==> Arweave TX/ANS-104 + TABM ==> dev_codec_httpsig_conv:to/1 ==> HTTP Signed Message + TABM ==> dev_codec_flat:to/1 ==> Flat Maps + ... ++Additionally, this module provides a number of utility functions for +manipulating messages. For example, `hb_message:sign/2` to sign a message of +arbitrary type, or `hb_formatter:format_msg/1` to print an AO-Core/TABM message in +a human-readable format. +The `hb_cache` module is responsible for storing and retrieving messages in +the HyperBEAM stores configured on the node. Each store has its own storage +backend, but each works with simple key-value pairs. Subsequently, the +`hb_cache` module uses TABMs as the internal format for storing and +retrieving messages. +Test vectors to ensure the functioning of this module and the codecs that +interact with it are found in `hb_message_test_vectors.erl`. + +--- + +## Exported Functions + +- `commit/2` +- `commit/3` +- `commitment_devices/2` +- `commitment/2` +- `commitment/3` +- `commitments/3` +- `committed/3` +- `convert/3` +- `convert/4` +- `default_tx_list/0` +- `diff/3` +- `filter_default_keys/1` +- `find_target/3` +- `id/1` +- `id/2` +- `id/3` +- `is_signed_key/3` +- `match/2` +- `match/3` +- `match/4` +- `minimize/1` +- `normalize_commitments/2` +- `print/1` +- `signers/2` +- `type/1` +- `uncommitted/1` +- `uncommitted/2` +- `verify/1` +- `verify/2` +- `verify/3` +- `with_commitments/3` +- `with_only_committed/2` +- `with_only_committers/2` +- `with_only_committers/3` +- `without_commitments/3` +- `without_unless_signed/3` + +--- + +### convert + +This module acts an adapter between messages, as modeled in the +Convert a message from one format to another. Taking a message in the + +```erlang +convert(Msg, TargetFormat, Opts) -> + convert(Msg, TargetFormat, <<"structured@1.0">>, Opts). +``` + +### convert + +This module acts an adapter between messages, as modeled in the +Convert a message from one format to another. Taking a message in the + +```erlang +convert(Msg, TargetFormat, tabm, Opts) -> + OldPriv = + if is_map(Msg) -> maps:get(<<"priv">>, Msg, #{}); + true -> #{} + end, + from_tabm(Msg, TargetFormat, OldPriv, Opts); +``` + +### convert + +This module acts an adapter between messages, as modeled in the +Convert a message from one format to another. Taking a message in the + +```erlang +convert(Msg, TargetFormat, SourceFormat, Opts) -> + OldPriv = + if is_map(Msg) -> maps:get(<<"priv">>, Msg, #{}); + true -> #{} + end, + TABM = + to_tabm( + case is_map(Msg) of + true -> hb_maps:without([<<"priv">>], Msg, Opts); + false -> Msg + end, + SourceFormat, + Opts + ), + case TargetFormat of + tabm -> restore_priv(TABM, OldPriv, Opts); + _ -> from_tabm(TABM, TargetFormat, OldPriv, Opts) + end. +``` + +### to_tabm + +```erlang +to_tabm(Msg, SourceFormat, Opts) -> + {SourceCodecMod, Params} = conversion_spec_to_req(SourceFormat, Opts), + % We use _from_ here because the codecs are labelled from the perspective + % of their own format. `dev_codec_ans104:from/1' will convert _from_ + % an ANS-104 message _into_ a TABM. +``` + +### from_tabm + +```erlang +from_tabm(Msg, TargetFormat, OldPriv, Opts) -> + {TargetCodecMod, Params} = conversion_spec_to_req(TargetFormat, Opts), + % We use the _to_ function here because each of the codecs we may call in + % this step are labelled from the perspective of the target format. For + % example, `dev_codec_httpsig:to/1' will convert _from_ a TABM to an + % HTTPSig message. +``` + +### restore_priv + +Add the existing `priv` sub-map back to a converted message, honoring + +```erlang +restore_priv(Msg, EmptyPriv, _Opts) when map_size(EmptyPriv) == 0 -> Msg; +``` + +### restore_priv + +Add the existing `priv` sub-map back to a converted message, honoring +Get a codec device and request params from the given conversion request. + +```erlang +restore_priv(Msg, OldPriv, Opts) -> + MsgPriv = hb_maps:get(<<"priv">>, Msg, #{}, Opts), + ?event({restoring_priv, {msg_priv, MsgPriv}, {old_priv, OldPriv}}), + NewPriv = hb_util:deep_merge(MsgPriv, OldPriv, Opts), + ?event({new_priv, NewPriv}), + Msg#{ <<"priv">> => NewPriv }. +``` + +### conversion_spec_to_req + +Add the existing `priv` sub-map back to a converted message, honoring +Get a codec device and request params from the given conversion request. + +```erlang +conversion_spec_to_req(Spec, Opts) when is_binary(Spec) or (Spec == tabm) -> + conversion_spec_to_req(#{ <<"device">> => Spec }, Opts); +``` + +### conversion_spec_to_req + +Add the existing `priv` sub-map back to a converted message, honoring +Get a codec device and request params from the given conversion request. + +```erlang +conversion_spec_to_req(Spec, Opts) -> + try + Device = + hb_maps:get( + <<"device">>, + Spec, + no_codec_device_in_conversion_spec, + Opts + ), + { + case Device of + tabm -> tabm; + _ -> + hb_ao:message_to_device( + #{ + <<"device">> => Device + }, + Opts + ) + end, + hb_maps:without([<<"device">>], Spec, Opts) + } + catch _:_ -> + throw({message_codec_not_extractable, Spec}) + end. +``` + +### id + +Return the ID of a message. + +```erlang +id(Msg) -> id(Msg, uncommitted). +``` + +### id + +Return the ID of a message. + +```erlang +id(Msg, Opts) when is_map(Opts) -> id(Msg, uncommitted, Opts); +``` + +### id + +Return the ID of a message. + +```erlang +id(Msg, Committers) -> id(Msg, Committers, #{}). +``` + +### id + +Return the ID of a message. + +```erlang +id(Msg, RawCommitters, Opts) -> + CommSpec = + case RawCommitters of + none -> #{ <<"committers">> => <<"none">> }; + uncommitted -> #{ <<"committers">> => <<"none">> }; + unsigned -> #{ <<"committers">> => <<"none">> }; + all -> #{ <<"committers">> => <<"all">> }; + signed -> #{ <<"committers">> => <<"all">> }; + List when is_list(List) -> #{ <<"committers">> => List } + end, + ?event({getting_id, {msg, Msg}, {spec, CommSpec}}), + {ok, ID} = + dev_message:id( + Msg, + CommSpec#{ <<"path">> => <<"id">> }, + Opts + ), + hb_util:human_id(ID). +``` + +### normalize_commitments + +Normalize the IDs in a message, ensuring that there is at least one + +```erlang +normalize_commitments(Msg, Opts) when is_map(Msg) -> + NormMsg = + maps:map( + fun(Key, Val) when Key == <<"commitments">> orelse Key == <<"priv">> -> + Val; + (_Key, Val) -> normalize_commitments(Val, Opts) + end, + Msg + ), + case hb_maps:get(<<"commitments">>, NormMsg, not_found, Opts) of + not_found -> + {ok, #{ <<"commitments">> := Commitments }} = + dev_message:commit( + NormMsg, + #{ <<"type">> => <<"unsigned">> }, + Opts + ), + NormMsg#{ <<"commitments">> => Commitments }; + _ -> NormMsg + end; +``` + +### normalize_commitments + +Normalize the IDs in a message, ensuring that there is at least one + +```erlang +normalize_commitments(Msg, Opts) when is_list(Msg) -> + lists:map(fun(X) -> normalize_commitments(X, Opts) end, Msg); +``` + +### normalize_commitments + +Normalize the IDs in a message, ensuring that there is at least one + +```erlang +normalize_commitments(Msg, _Opts) -> + Msg. +``` + +### with_only_committed + +Return a message with only the committed keys. If no commitments are + +```erlang +with_only_committed(Msg, Opts) when is_map(Msg) -> + ?event({with_only_committed, {msg, Msg}, {opts, Opts}}), + Comms = hb_maps:get(<<"commitments">>, Msg, not_found, Opts), + case is_map(Msg) andalso Comms /= not_found of + true -> + try + CommittedKeys = + hb_message:committed( + Msg, + #{ <<"commitments">> => <<"all">> }, + Opts + ), + % Add the ao-body-key to the committed list if it is not + % already present. +``` + +### with_only_committed + +```erlang +with_only_committed(Msg, _) -> + % If the message is not a map, it cannot be signed. +``` + +### with_links + +Filter keys from a map that do not match either the list of keys or + +```erlang +with_links(Keys, Map, Opts) -> + hb_maps:with( + Keys ++ + lists:map( + fun(Key) -> + <<(hb_link:remove_link_specifier(Key))/binary, "+link">> + end, + Keys + ), + Map, + Opts + ). +``` + +### with_only_committers + +Return the message with only the specified committers attached. + +```erlang +with_only_committers(Msg, Committers) -> + with_only_committers(Msg, Committers, #{}). +``` + +### with_only_committers + +```erlang +with_only_committers(Msg, Committers, Opts) when is_map(Msg) -> + NewCommitments = + hb_maps:filter( + fun(_, #{ <<"committer">> := Committer }) -> + lists:member(Committer, Committers); + (_, _) -> false + end, + hb_maps:get(<<"commitments">>, Msg, #{}, Opts), + Opts + ), + Msg#{ <<"commitments">> => NewCommitments }; +``` + +### with_only_committers + +```erlang +with_only_committers(Msg, _Committers, _Opts) -> + throw({unsupported_message_type, Msg}). +``` + +### is_signed_key + +Determine whether a specific key is part of a message's commitments. + +```erlang +is_signed_key(Key, Msg, Opts) -> + lists:member(Key, hb_message:committed(Msg, all, Opts)). +``` + +### without_unless_signed + +Remove the any of the given keys that are not signed from a message. + +```erlang +without_unless_signed(Key, Msg, Opts) when not is_list(Key) -> + without_unless_signed([Key], Msg, Opts); +``` + +### without_unless_signed + +Remove the any of the given keys that are not signed from a message. + +```erlang +without_unless_signed(Keys, Msg, Opts) -> + SignedKeys = hb_message:committed(Msg, all, Opts), + maps:without( + lists:filter(fun(K) -> not lists:member(K, SignedKeys) end, Keys), + Msg + ). +``` + +### commit + +Sign a message with the given wallet. + +```erlang +commit(Msg, WalletOrOpts) -> + commit( + Msg, + WalletOrOpts, + hb_opts:get( + commitment_device, + no_viable_commitment_device, + case is_map(WalletOrOpts) of + true -> WalletOrOpts; + false -> #{ priv_wallet => WalletOrOpts } + end + ) + ). +``` + +### commit + +```erlang +commit(Msg, Wallet, Format) when not is_map(Wallet) -> + commit(Msg, #{ priv_wallet => Wallet }, Format); +``` + +### commit + +```erlang +commit(Msg, Opts, CodecName) when is_binary(CodecName) -> + commit(Msg, Opts, #{ <<"commitment-device">> => CodecName }); +``` + +### commit + +```erlang +commit(Msg, Opts, Spec) -> + {ok, Signed} = + dev_message:commit( + Msg, + Spec#{ + <<"commitment-device">> => + case hb_maps:get(<<"commitment-device">>, Spec, none, Opts) of + none -> + case hb_maps:get(<<"device">>, Spec, none, Opts) of + none -> + throw( + { + no_commitment_device_in_codec_spec, + Spec + } + ); + Device -> Device + end; + CommitmentDevice -> CommitmentDevice + end + }, + Opts + ), + Signed. +``` + +### committed + +Return the list of committed keys from a message. + +```erlang +committed(Msg, all, Opts) -> + committed(Msg, #{ <<"committers">> => <<"all">> }, Opts); +``` + +### committed + +Return the list of committed keys from a message. + +```erlang +committed(Msg, none, Opts) -> + committed(Msg, #{ <<"committers">> => <<"none">> }, Opts); +``` + +### committed + +Return the list of committed keys from a message. + +```erlang +committed(Msg, List, Opts) when is_list(List) -> + committed(Msg, #{ <<"commitments">> => List }, Opts); +``` + +### committed + +Return the list of committed keys from a message. + +```erlang +committed(Msg, CommittersMsg, Opts) -> + ?event( + {committed, + {msg, {explicit, Msg}}, + {committers_msg, {explicit, CommittersMsg}}, + {opts, Opts} + } + ), + {ok, CommittedKeys} = dev_message:committed(Msg, CommittersMsg, Opts), + CommittedKeys. +``` + +### verify + +wrapper function to verify a message. + +```erlang +verify(Msg) -> verify(Msg, all). +``` + +### verify + +wrapper function to verify a message. + +```erlang +verify(Msg, Committers) -> + verify(Msg, Committers, #{}). +``` + +### verify + +```erlang +verify(Msg, all, Opts) -> + verify(Msg, <<"all">>, Opts); +``` + +### verify + +```erlang +verify(Msg, signers, Opts) -> + verify(Msg, hb_message:signers(Msg, Opts), Opts); +``` + +### verify + +```erlang +verify(Msg, Committers, Opts) when not is_map(Committers) -> + verify( + Msg, + #{ + <<"committers">> => + case ?IS_ID(Committers) of + true -> [Committers]; + false -> Committers + end + }, + Opts + ); +``` + +### verify + +```erlang +verify(Msg, Spec, Opts) -> + ?event(verify, {verify, {spec, Spec}}), + {ok, Res} = + dev_message:verify( + Msg, + Spec, + Opts + ), + Res. +``` + +### uncommitted + +Return the unsigned version of a message in AO-Core format. + +```erlang +uncommitted(Msg) -> uncommitted(Msg, #{}). +``` + +### uncommitted + +Return the unsigned version of a message in AO-Core format. + +```erlang +uncommitted(Bin, _Opts) when is_binary(Bin) -> Bin; +``` + +### uncommitted + +Return the unsigned version of a message in AO-Core format. +Return all of the committers on a message that have 'normal', 256 bit, + +```erlang +uncommitted(Msg, Opts) -> + hb_maps:remove(<<"commitments">>, Msg, Opts). +``` + +### signers + +Return the unsigned version of a message in AO-Core format. +Return all of the committers on a message that have 'normal', 256 bit, + +```erlang +signers(Msg, Opts) -> + hb_util:ok(dev_message:committers(Msg, #{}, Opts)). +``` + +### print + +Pretty-print a message. + +```erlang +print(Msg) -> print(Msg, 0). +``` + +### print + +Pretty-print a message. +Return the type of an encoded message. + +```erlang +print(Msg, Indent) -> + io:format(standard_error, "~s", [lists:flatten(hb_format:message(Msg, #{}, Indent))]). +``` + +### type + +Pretty-print a message. +Return the type of an encoded message. + +```erlang +type(TX) when is_record(TX, tx) -> tx; +``` + +### type + +Pretty-print a message. +Return the type of an encoded message. + +```erlang +type(Binary) when is_binary(Binary) -> binary; +``` + +### type + +Pretty-print a message. +Return the type of an encoded message. + +```erlang +type(Msg) when is_map(Msg) -> + IsDeep = lists:any( + fun({_, Value}) -> is_map(Value) end, + lists:filter( + fun({Key, _}) -> not hb_private:is_private(Key) end, + hb_maps:to_list(Msg) + ) + ), + case IsDeep of + true -> deep; + false -> shallow + end. +``` + +### match + +Check if two maps match, including recursively checking nested maps. + +```erlang +match(Map1, Map2) -> + match(Map1, Map2, strict). +``` + +### match + +```erlang +match(Map1, Map2, Mode) -> + match(Map1, Map2, Mode, #{}). +``` + +### match + +```erlang +match(Map1, Map2, Mode, Opts) -> + try unsafe_match(Map1, Map2, Mode, [], Opts) + catch _:Details -> Details + end. +``` + +### unsafe_match + +Match two maps, returning `true` if they match, or throwing an error + +```erlang +unsafe_match(Map1, Map2, Mode, Path, Opts) -> + Keys1 = + hb_maps:keys( + NormMap1 = hb_util:lower_case_key_map(minimize( + normalize(hb_ao:normalize_keys(Map1, Opts), Opts), + [<<"content-type">>, <<"ao-body-key">>] + ), Opts) + ), + Keys2 = + hb_maps:keys( + NormMap2 = hb_util:lower_case_key_map(minimize( + normalize(hb_ao:normalize_keys(Map2, Opts), Opts), + [<<"content-type">>, <<"ao-body-key">>] + ), Opts) + ), + PrimaryKeysPresent = + (Mode == primary) andalso + lists:all( + fun(Key) -> lists:member(Key, Keys1) end, + Keys1 + ), + ?event(match, + {match, + {keys1, Keys1}, + {keys2, Keys2}, + {mode, Mode}, + {primary_keys_present, PrimaryKeysPresent}, + {msg1, Map1}, + {msg2, Map2} + } + ), + case (Keys1 == Keys2) or (Mode == only_present) or PrimaryKeysPresent of + true -> + lists:all( + fun(Key) -> + ?event(match, {matching_key, Key}), + Val1 = + hb_ao:normalize_keys( + hb_maps:get(Key, NormMap1, not_found, Opts), + Opts + ), + Val2 = + hb_ao:normalize_keys( + hb_maps:get(Key, NormMap2, not_found, Opts), + Opts + ), + BothPresent = (Val1 =/= not_found) and (Val2 =/= not_found), + case (not BothPresent) and (Mode == only_present) of + true -> true; + false -> + case is_map(Val1) andalso is_map(Val2) of + true -> + unsafe_match(Val1, Val2, Mode, Path ++ [Key], Opts); + false -> + case {Val1, Val2} of + {V, V} -> true; + {V, '_'} when V =/= not_found -> true; + {'_', V} when V =/= not_found -> true; + {'_', '_'} -> true; + _ -> + throw( + {value_mismatch, + hb_format:short_id( + hb_path:to_binary( + Path ++ [Key] + ) + ), + {val1, Val1}, + {val2, Val2} + } + ) + end + end + end + end, + Keys1 + ); + false -> + throw( + {keys_mismatch, + {path, hb_format:short_id(hb_path:to_binary(Path))}, + {keys1, Keys1}, + {keys2, Keys2} + } + ) + end. +``` + +### matchable_keys + +```erlang +matchable_keys(Map) -> + lists:sort(lists:map(fun hb_ao:normalize_key/1, hb_maps:keys(Map))). +``` + +### diff + +Return the numeric differences between two messages, matching deeply + +```erlang +diff(Msg1, Msg2, Opts) when is_map(Msg1) andalso is_map(Msg2) -> + maps:filtermap( + fun(Key, Val2) -> + case hb_maps:get(Key, Msg1, not_found, Opts) of + Val2 -> + % The key is present in both maps, and the values match. +``` + +### diff + +```erlang +diff(_Val1, _Val2, _Opts) -> + not_found. +``` + +### with_commitments + +Filter messages that do not match the 'spec' given. The underlying match + +```erlang +with_commitments(ID, Msg, Opts) when ?IS_ID(ID) -> + with_commitments([ID], Msg, Opts); +``` + +### with_commitments + +Filter messages that do not match the 'spec' given. The underlying match + +```erlang +with_commitments(Spec, Msg = #{ <<"commitments">> := Commitments }, Opts) -> + ?event({with_commitments, {spec, Spec}, {commitments, Commitments}}), + FilteredCommitments = + hb_maps:filter( + fun(ID, CommMsg) -> + if is_list(Spec) -> + lists:member(ID, Spec); + is_map(Spec) -> + match(Spec, CommMsg, primary, Opts) == true + end + end, + Commitments, + Opts + ), + ?event({with_commitments, {filtered_commitments, FilteredCommitments}}), + Msg#{ <<"commitments">> => FilteredCommitments }; +``` + +### with_commitments + +Filter messages that do not match the 'spec' given. The underlying match + +```erlang +with_commitments(_Spec, Msg, _Opts) -> + Msg. +``` + +### without_commitments + +Filter messages that match the 'spec' given. Inverts the `with_commitments/2` + +```erlang +without_commitments(Spec, Msg = #{ <<"commitments">> := Commitments }, Opts) -> + ?event({without_commitments, {spec, Spec}, {msg, Msg}, {commitments, Commitments}}), + FilteredCommitments = + hb_maps:without( + hb_maps:keys( + hb_maps:get( + <<"commitments">>, + with_commitments(Spec, Msg, Opts), + #{}, + Opts + ) + ), + Commitments + ), + ?event({without_commitments, {filtered_commitments, FilteredCommitments}}), + Msg#{ <<"commitments">> => FilteredCommitments }; +``` + +### without_commitments + +Filter messages that match the 'spec' given. Inverts the `with_commitments/2` + +```erlang +without_commitments(_Spec, Msg, _Opts) -> + Msg. +``` + +### commitment + +Extract a commitment from a message given a `committer` or `commitment` + +```erlang +commitment(ID, Msg) -> + commitment(ID, Msg, #{}). +``` + +### commitment + +```erlang +commitment(ID, Link, Opts) when ?IS_LINK(Link) -> + commitment(ID, hb_cache:ensure_loaded(Link, Opts), Opts); +``` + +### commitment + +```erlang +commitment(ID, #{ <<"commitments">> := Commitments }, Opts) + when is_binary(ID), is_map_key(ID, Commitments) -> + hb_maps:get( + ID, + Commitments, + not_found, + Opts + ); +``` + +### commitment + +```erlang +commitment(Spec, Msg, Opts) -> + Matches = commitments(Spec, Msg, Opts), + ?event(debug_commitment, {commitment, {spec, Spec}, {matches, Matches}}), + if + map_size(Matches) == 0 -> not_found; + map_size(Matches) == 1 -> + CommID = hd(hb_maps:keys(Matches)), + {ok, CommID, hb_util:ok(hb_maps:find(CommID, Matches, Opts))}; + true -> + ?event(commitment, {multiple_matches, {matches, Matches}}), + multiple_matches + end; +``` + +### commitment + +```erlang +commitment(_Spec, _Msg, _Opts) -> + % The message has no commitments, so the spec can never match. +``` + +### commitments + +Return a list of all commitments that match the spec. + +```erlang +commitments(ID, Link, Opts) when ?IS_LINK(Link) -> + commitments(ID, hb_cache:ensure_loaded(Link, Opts), Opts); +``` + +### commitments + +Return a list of all commitments that match the spec. + +```erlang +commitments(CommitterID, Msg, Opts) when is_binary(CommitterID) -> + commitments(#{ <<"committer">> => CommitterID }, Msg, Opts); +``` + +### commitments + +Return a list of all commitments that match the spec. + +```erlang +commitments(Spec, #{ <<"commitments">> := Commitments }, Opts) -> + hb_maps:filtermap( + fun(_ID, CommMsg) -> + case match(Spec, CommMsg, primary, Opts) of + true -> {true, CommMsg}; + _ -> false + end + end, + Commitments, + Opts + ); +``` + +### commitments + +Return a list of all commitments that match the spec. + +```erlang +commitments(_Spec, _Msg, _Opts) -> + #{}. +``` + +### commitment_devices + +Return the devices for which there are commitments on a message. + +```erlang +commitment_devices(#{ <<"commitments">> := Commitments }, Opts) -> + lists:map( + fun(CommMsg) -> + hb_ao:get(<<"commitment-device">>, CommMsg, Opts) + end, + maps:values(Commitments) + ); +``` + +### commitment_devices + +Return the devices for which there are commitments on a message. + +```erlang +commitment_devices(_Msg, _Opts) -> + []. +``` + +### find_target + +Implements a standard pattern in which the target for an operation is + +```erlang +find_target(Self, Req, Opts) -> + GetOpts = Opts#{ + hashpath => ignore, + cache_control => [<<"no-cache">>, <<"no-store">>] + }, + {ok, + case hb_maps:get(<<"target">>, Req, <<"self">>, GetOpts) of + <<"self">> -> Self; + Key -> + hb_maps:get( + Key, + Req, + hb_maps:get(<<"body">>, Req, GetOpts), + GetOpts + ) + end + }. +``` + +### minimize + +Remove keys from the map that can be regenerated. Optionally takes an + +```erlang +minimize(Msg) -> minimize(Msg, []). +``` + +### minimize + +Remove keys from the map that can be regenerated. Optionally takes an + +```erlang +minimize(RawVal, _) when not is_map(RawVal) -> RawVal; +``` + +### minimize + +Remove keys from the map that can be regenerated. Optionally takes an + +```erlang +minimize(Map, ExtraKeys) -> + NormKeys = + lists:map(fun hb_ao:normalize_key/1, ?REGEN_KEYS) + ++ lists:map(fun hb_ao:normalize_key/1, ExtraKeys), + maps:filter( + fun(Key, _) -> + (not lists:member(hb_ao:normalize_key(Key), NormKeys)) + andalso (not hb_private:is_private(Key)) + end, + maps:map(fun(_K, V) -> minimize(V) end, Map) + ). +``` + +### normalize + +Return a map with only the keys that necessary, without those that can + +```erlang +normalize(Map, Opts) when is_map(Map) orelse is_list(Map) -> + NormalizedMap = hb_ao:normalize_keys(Map, Opts), + FilteredMap = filter_default_keys(NormalizedMap), + hb_maps:with(matchable_keys(FilteredMap), FilteredMap); +``` + +### normalize + +Return a map with only the keys that necessary, without those that can + +```erlang +normalize(Other, _Opts) -> + Other. +``` + +### filter_default_keys + +Remove keys from a map that have the default values found in the tx + +```erlang +filter_default_keys(Map) -> + DefaultsMap = default_tx_message(), + maps:filter( + fun(Key, Value) -> + case hb_maps:find(hb_ao:normalize_key(Key), DefaultsMap) of + {ok, Value} -> false; + _ -> true + end + end, + Map + ). +``` + +### default_tx_message + +Get the normalized fields and default values of the tx record. + +```erlang +default_tx_message() -> + hb_maps:from_list(default_tx_list()). +``` + +### default_tx_list + +Get the ordered list of fields as AO-Core keys and default values of + +```erlang +default_tx_list() -> + Keys = lists:map(fun hb_ao:normalize_key/1, record_info(fields, tx)), +``` + +--- + +*Generated from [hb_message.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_message.erl)* diff --git a/docs/book/src/hb_message_test_vectors.erl.md b/docs/book/src/hb_message_test_vectors.erl.md new file mode 100644 index 000000000..36e40e41e --- /dev/null +++ b/docs/book/src/hb_message_test_vectors.erl.md @@ -0,0 +1,1736 @@ +# hb_message_test_vectors + +[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_message_test_vectors.erl) + +A battery of test vectors for message codecs, implementing the +`message@1.0` encoding and commitment APIs. Additionally, this module +houses tests that ensure the general functioning of the `hb_message` API. + +--- + +### run_test + +A battery of test vectors for message codecs, implementing the +Test invocation function, making it easier to run a specific test. + +```erlang +run_test() -> + hb:init(), + nested_structured_fields_test( + #{ <<"device">> => <<"json@1.0">>, <<"bundle">> => true }, + test_opts(normal) + ). +``` + +### test_codecs + +Return a list of codecs to test. Disable these as necessary if you need + +```erlang +test_codecs() -> + [ + <<"structured@1.0">>, + <<"httpsig@1.0">>, + #{ <<"device">> => <<"httpsig@1.0">>, <<"bundle">> => true }, + <<"flat@1.0">>, + <<"ans104@1.0">>, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + <<"json@1.0">>, + #{ <<"device">> => <<"json@1.0">>, <<"bundle">> => true } + ]. +``` + +### suite_test_opts + +Return a set of options for testing, taking the codec name as an + +```erlang +suite_test_opts() -> + [ + #{ + name => normal, + desc => <<"Default opts">>, + opts => test_opts(normal) + } + ]. +``` + +### suite_test_opts + +```erlang +suite_test_opts(OptsName) -> + [ O || O = #{ name := OName } <- suite_test_opts(), OName == OptsName ]. +``` + +### test_opts + +```erlang +test_opts(normal) -> + #{ + store => hb_test_utils:test_store(), + priv_wallet => hb:wallet() + }. +``` + +### test_suite + +```erlang +test_suite() -> + [ + % Basic operations + {<<"Binary to binary">>, + fun binary_to_binary_test/2}, + {<<"Match">>, + fun match_test/2}, + {<<"Basic message encoding and decoding">>, + fun basic_message_codec_test/2}, + {<<"Priv survives conversion">>, + fun priv_survives_conversion_test/2}, + {<<"Message with body">>, + fun set_body_codec_test/2}, + {<<"Message with large keys">>, + fun message_with_large_keys_test/2}, + {<<"Structured field atom parsing">>, + fun structured_field_atom_parsing_test/2}, + {<<"Structured field decimal parsing">>, + fun structured_field_decimal_parsing_test/2}, + {<<"Unsigned id">>, + fun unsigned_id_test/2}, + % Nested structures + {<<"Simple nested message">>, + fun simple_nested_message_test/2}, + {<<"Message with simple embedded list">>, + fun message_with_simple_embedded_list_test/2}, + {<<"Nested empty map">>, + fun nested_empty_map_test/2}, + {<<"Empty body">>, + fun empty_body_test/2}, + {<<"Nested structured fields">>, + fun nested_structured_fields_test/2}, + {<<"Single layer message to encoding">>, + fun single_layer_message_to_encoding_test/2}, + {<<"Nested body list">>, + fun nested_body_list_test/2}, + {<<"Empty string in nested tag">>, + fun empty_string_in_nested_tag_test/2}, + {<<"Deep typed message ID">>, + fun deep_typed_message_id_test/2}, + {<<"Encode small balance table">>, + fun encode_small_balance_table_test/2}, + {<<"Encode large balance table">>, + fun encode_large_balance_table_test/2}, + {<<"Normalize commitments">>, + fun normalize_commitments_test/2}, + % Signed messages + {<<"Signed message to message and back">>, + fun signed_message_encode_decode_verify_test/2}, + {<<"Specific order signed message">>, + fun specific_order_signed_message_test/2}, + {<<"Specific order deeply nested signed message">>, + fun specific_order_deeply_nested_signed_message_test/2}, + {<<"Signed only committed data field">>, + fun signed_only_committed_data_field_test/2}, + {<<"Signed simple nested message">>, + fun simple_signed_nested_message_test/2}, + {<<"Signed nested message">>, + fun signed_nested_message_with_child_test/2}, + {<<"Committed keys">>, + fun committed_keys_test/2}, + {<<"Committed empty keys">>, + fun committed_empty_keys_test/2}, + {<<"Signed list HTTP response">>, + fun signed_list_test/2}, + {<<"Sign node message">>, + fun sign_node_message_test/2}, + {<<"Complex signed message">>, + fun complex_signed_message_test/2}, + {<<"Nested message with large keys">>, + fun nested_message_with_large_keys_test/2}, + {<<"Signed nested complex signed message">>, + fun verify_nested_complex_signed_test/2}, + % Complex structures + {<<"Nested message with large keys and content">>, + fun nested_message_with_large_keys_and_content_test/2}, + {<<"Nested message with large content">>, + fun nested_message_with_large_content_test/2}, + {<<"Deeply nested message with content">>, + fun deeply_nested_message_with_content_test/2}, + {<<"Deeply nested message with only content">>, + fun deeply_nested_message_with_only_content/2}, + {<<"Signed deep serialize and deserialize">>, + fun signed_deep_message_test/2}, + {<<"Signed nested data key">>, + fun signed_nested_data_key_test/2}, + {<<"Signed message with hashpath">>, + fun hashpath_sign_verify_test/2}, + {<<"Message with derived components">>, + fun signed_message_with_derived_components_test/2}, + {<<"Large body committed keys">>, + fun large_body_committed_keys_test/2}, + {<<"Signed with inner signed">>, + fun signed_with_inner_signed_message_test/2}, + {<<"Recursive nested list">>, + fun recursive_nested_list_test/2}, + {<<"Sign links">>, + fun sign_links_test/2}, + {<<"ID of linked message">>, + fun id_of_linked_message_test/2}, + {<<"Sign deep message from lazy cache read">>, + fun sign_deep_message_from_lazy_cache_read_test/2}, + {<<"ID of deep message and link message match">>, + fun id_of_deep_message_and_link_message_match_test/2}, + {<<"Signed non-bundle is bundlable">>, + fun signed_non_bundle_is_bundlable_test/2}, + {<<"Bundled ordering">>, + fun bundled_ordering_test/2}, + {<<"Codec round-trip conversion is idempotent">>, + fun codec_roundtrip_conversion_is_idempotent_test/2}, + {<<"Bundled and unbundled IDs differ">>, + fun bundled_and_unbundled_ids_differ_test/2}, + {<<"Tabm conversion is idempotent">>, + fun tabm_conversion_is_idempotent_test/2} + ]. +``` + +### suite_test_ + +Organizes a test battery for the `hb_message` module and its codecs. + +```erlang +suite_test_() -> + hb_test_utils:suite_with_opts( + codec_test_suite( + test_codecs(), + normal + ), + suite_test_opts(normal) + ). +``` + +### codec_test_suite + +Run the test suite for a set of codecs, using the given options type. + +```erlang +codec_test_suite(Codecs, OptsType) -> + lists:flatmap( + fun(CodecName) -> + lists:map(fun({Desc, Test}) -> + TestName = + binary_to_list( + << (suite_name(CodecName))/binary, ": ", Desc/binary >> + ), + TestSpecificOpts = test_opts(OptsType), + { + Desc, + TestName, + fun(_SuiteOpts) -> Test(CodecName, TestSpecificOpts) end + } + end, test_suite()) + end, + Codecs + ). +``` + +### suite_name + +Create a name for a suite from a codec spec. + +```erlang +suite_name(CodecSpec) when is_binary(CodecSpec) -> CodecSpec; +``` + +### suite_name + +Create a name for a suite from a codec spec. + +```erlang +suite_name(CodecSpec) when is_map(CodecSpec) -> + CodecName = maps:get(<<"device">>, CodecSpec, <<"[! NO CODEC !]">>), + case maps:get(<<"bundle">>, CodecSpec, false) of + false -> CodecName; + true -> << CodecName/binary, " (bundle)">> + end. +``` + +### is_idempotent + +Tests a message transforming function to ensure that it is idempotent. + +```erlang +is_idempotent(Func, Msg, Opts) -> + Run = fun(M) -> case Func(M) of {ok, Res} -> Res; Res -> Res end end, + After1 = Run(Msg), + After2 = Run(After1), + After3 = Run(After2), + MatchRes1 = hb_message:match(After1, After2, strict, Opts), + MatchRes2 = hb_message:match(After2, After3, strict, Opts), + ?event({is_idempotent, {match_res1, MatchRes1}, {match_res2, MatchRes2}}), + MatchRes1 andalso MatchRes2. +``` + +### tabm_conversion_is_idempotent_test + +Ensure that converting a message to/from TABM multiple times repeatedly + +```erlang +tabm_conversion_is_idempotent_test(_Codec, Opts) -> + From = fun(M) -> hb_message:convert(M, <<"structured@1.0">>, tabm, Opts) end, + To = fun(M) -> hb_message:convert(M, tabm, <<"structured@1.0">>, Opts) end, + SimpleMsg = #{ <<"a">> => <<"x">>, <<"b">> => <<"y">>, <<"c">> => <<"z">> }, + ComplexMsg = + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => + Signed = hb_message:commit( + #{ + <<"type">> => <<"Message">>, + <<"function">> => <<"fac">>, + <<"parameters">> => #{ + <<"a">> => 1 + }, + <<"content-type">> => <<"application/html">>, + <<"body">> => + << + """ + +
+ Msg1.HashPath = Msg1.ID
+ Msg3.HashPath = Msg1.Hash(Msg1.HashPath, Msg2.ID)
+ Msg3.{...} = AO-Core.apply(Msg1, Msg2)
+ ...
+
+A message's ID itself includes its HashPath, leading to the mixing of
+a Msg2's merkle list into the resulting Msg3's HashPath. This allows a single
+message to represent a history _tree_ of all of the messages that were
+applied to generate it -- rather than just a linear history.
+A message may also specify its own algorithm for generating its HashPath,
+which allows for custom logic to be used for representing the history of a
+message. When Msg2's are applied to a Msg1, the resulting Msg3's HashPath
+will be generated according to Msg1's algorithm choice.
+
+---
+
+## Exported Functions
+
+- `from_message/3`
+- `hashpath_alg/2`
+- `hashpath/2`
+- `hashpath/3`
+- `hashpath/4`
+- `hd/2`
+- `matches/2`
+- `normalize/1`
+- `pop_request/2`
+- `priv_remaining/2`
+- `priv_store_remaining/2`
+- `priv_store_remaining/3`
+- `push_request/2`
+- `push_request/3`
+- `queue_request/2`
+- `queue_request/3`
+- `regex_matches/2`
+- `term_to_path_parts/1`
+- `term_to_path_parts/2`
+- `tl/2`
+- `to_binary/1`
+- `verify_hashpath/2`
+
+---
+
+### hd
+
+This module provides utilities for manipulating the paths of a
+Extract the first key from a `Message2`'s `Path` field.
+
+```erlang
+hd(Msg2, Opts) ->
+ %?event({key_from_path, Msg2, Opts}),
+ case pop_request(Msg2, Opts) of
+ undefined -> undefined;
+ {Head, _} ->
+ % `term_to_path' returns the full path, so we need to take the
+ % `hd' of our `Head'.
+```
+
+### tl
+
+Return the message without its first path element. Note that this
+
+```erlang
+tl(Msg2, Opts) when is_map(Msg2) ->
+ case pop_request(Msg2, Opts) of
+ undefined -> undefined;
+ {_, Rest} -> Rest
+ end;
+```
+
+### tl
+
+Return the message without its first path element. Note that this
+
+```erlang
+tl(Path, Opts) when is_list(Path) ->
+ case tl(#{ <<"path">> => Path }, Opts) of
+ [] -> undefined;
+ undefined -> undefined;
+ #{ <<"path">> := Rest } -> Rest
+ end.
+```
+
+### priv_remaining
+
+Return the `Remaining-Path` of a message, from its hidden `AO-Core`
+Store the remaining path of a message in its hidden `AO-Core` key.
+
+```erlang
+priv_remaining(Msg, Opts) ->
+ Priv = hb_private:from_message(Msg),
+ AOCore = hb_maps:get(<<"ao-core">>, Priv, #{}, Opts),
+ hb_maps:get(<<"remaining">>, AOCore, undefined, Opts).
+```
+
+### priv_store_remaining
+
+Return the `Remaining-Path` of a message, from its hidden `AO-Core`
+Store the remaining path of a message in its hidden `AO-Core` key.
+
+```erlang
+priv_store_remaining(Msg, RemainingPath) ->
+ priv_store_remaining(Msg, RemainingPath, #{}).
+```
+
+### priv_store_remaining
+
+```erlang
+priv_store_remaining(Msg, RemainingPath, Opts) ->
+ Priv = hb_private:from_message(Msg),
+ AOCore = hb_maps:get(<<"ao-core">>, Priv, #{}, Opts),
+ Msg#{
+ <<"priv">> =>
+ Priv#{
+ <<"ao-core">> =>
+ AOCore#{
+ <<"remaining">> => RemainingPath
+ }
+ }
+ }.
+```
+
+### hashpath
+
+Add an ID of a Msg2 to the HashPath of another message.
+
+```erlang
+hashpath(Bin, _Opts) when is_binary(Bin) ->
+ % Default hashpath for a binary message is its SHA2-256 hash.
+```
+
+### hashpath
+
+```erlang
+hashpath(RawMsg1, Opts) ->
+ Msg1 = hb_ao:normalize_keys(RawMsg1, Opts),
+ case hb_private:from_message(Msg1) of
+ #{ <<"hashpath">> := HP } -> HP;
+ _ ->
+ % Note: We do not use `hb_message:id' here because it will call
+ % hb_ao:resolve, which will call `hashpath' recursively.
+```
+
+### hashpath
+
+```erlang
+hashpath(Msg1, Msg2, Opts) when is_map(Msg1) ->
+ Msg1Hashpath = hashpath(Msg1, Opts),
+ HashpathAlg = hashpath_alg(Msg1, Opts),
+ hashpath(Msg1Hashpath, Msg2, HashpathAlg, Opts);
+```
+
+### hashpath
+
+```erlang
+hashpath(Msg1, Msg2, Opts) ->
+ throw({hashpath_not_viable, Msg1, Msg2, Opts}).
+```
+
+### hashpath
+
+```erlang
+hashpath(Msg1, Msg2, HashpathAlg, Opts) when is_map(Msg2) ->
+ Msg2WithoutMeta = hb_maps:without(?AO_CORE_KEYS, Msg2, Opts),
+ ReqPath = from_message(request, Msg2, Opts),
+ case {map_size(Msg2WithoutMeta), ReqPath} of
+ {0, _} when ReqPath =/= undefined ->
+ hashpath(Msg1, to_binary(hd(ReqPath)), HashpathAlg, Opts);
+ _ ->
+ {ok, Msg2ID} =
+ dev_message:id(
+ Msg2,
+ #{ <<"commitments">> => <<"all">> },
+ Opts
+ ),
+ hashpath(Msg1, hb_util:human_id(Msg2ID), HashpathAlg, Opts)
+ end;
+```
+
+### hashpath
+
+```erlang
+hashpath(Msg1Hashpath, HumanMsg2ID, HashpathAlg, Opts) ->
+ ?event({hashpath, {msg1hp, {explicit, Msg1Hashpath}}, {msg2id, {explicit, HumanMsg2ID}}}),
+ HP =
+ case term_to_path_parts(Msg1Hashpath, Opts) of
+ [_] ->
+ << Msg1Hashpath/binary, "/", HumanMsg2ID/binary >>;
+ [Prev1, Prev2] ->
+ % Calculate the new base of the hashpath. We check whether the key is
+ % a human-readable binary ID, or a path part, and convert or pass
+ % through accordingly.
+```
+
+### hashpath_alg
+
+Get the hashpath function for a message from its HashPath-Alg.
+
+```erlang
+hashpath_alg(Msg, Opts) ->
+ case dev_message:get(<<"hashpath-alg">>, Msg, Opts) of
+ {ok, <<"sha-256-chain">>} ->
+ fun hb_crypto:sha256_chain/2;
+ {ok, <<"accumulate-256">>} ->
+ fun hb_crypto:accumulate/2;
+ {error, not_found} ->
+ fun hb_crypto:sha256_chain/2
+ end.
+```
+
+### push_request
+
+Add a message to the head (next to execute) of a request path.
+
+```erlang
+push_request(Msg, Path) ->
+ push_request(Msg, Path, #{}).
+```
+
+### push_request
+
+Pop the next element from a request path or path list.
+
+```erlang
+push_request(Msg, Path, Opts) ->
+ hb_maps:put(<<"path">>, term_to_path_parts(Path, Opts) ++ from_message(request, Msg, Opts), Msg, Opts).
+```
+
+### pop_request
+
+Pop the next element from a request path or path list.
+
+```erlang
+pop_request(undefined, _Opts) -> undefined;
+```
+
+### pop_request
+
+Pop the next element from a request path or path list.
+
+```erlang
+pop_request(Msg, Opts) when is_map(Msg) ->
+ %?event({popping_request, {msg, Msg}, {opts, Opts}}),
+ case pop_request(from_message(request, Msg, Opts), Opts) of
+ undefined -> undefined;
+ {undefined, _} -> undefined;
+ {Head, []} -> {Head, undefined};
+ {Head, Rest} ->
+ ?event({popped_request, Head, Rest}),
+ {Head, hb_maps:put(<<"path">>, Rest, Msg, Opts)}
+ end;
+```
+
+### pop_request
+
+Pop the next element from a request path or path list.
+
+```erlang
+pop_request([], _Opts) -> undefined;
+```
+
+### pop_request
+
+Pop the next element from a request path or path list.
+
+```erlang
+pop_request([Head|Rest], _Opts) ->
+ {Head, Rest}.
+```
+
+### queue_request
+
+Queue a message at the back of a request path. `path` is the only
+
+```erlang
+queue_request(Msg, Path) ->
+ queue_request(Msg, Path, #{}).
+```
+
+### queue_request
+
+Verify the HashPath of a message, given a list of messages that
+
+```erlang
+queue_request(Msg, Path, Opts) ->
+ hb_maps:put(<<"path">>, from_message(request, Msg, Opts) ++ term_to_path_parts(Path), Msg, Opts).
+```
+
+### verify_hashpath
+
+Verify the HashPath of a message, given a list of messages that
+
+```erlang
+verify_hashpath([Msg1, Msg2, Msg3|Rest], Opts) ->
+ CorrectHashpath = hashpath(Msg1, Msg2, Opts),
+ FromMsg3 = from_message(hashpath, Msg3, Opts),
+ CorrectHashpath == FromMsg3 andalso
+ case Rest of
+ [] -> true;
+ _ -> verify_hashpath([Msg2, Msg3|Rest], Opts)
+ end.
+```
+
+### from_message
+
+Extract the request path or hashpath from a message. We do not use
+
+```erlang
+from_message(Type, Link, Opts) when ?IS_LINK(Link) ->
+ from_message(Type, hb_cache:ensure_loaded(Link, Opts), Opts);
+```
+
+### from_message
+
+Extract the request path or hashpath from a message. We do not use
+
+```erlang
+from_message(hashpath, Msg, Opts) -> hashpath(Msg, Opts);
+```
+
+### from_message
+
+Extract the request path or hashpath from a message. We do not use
+
+```erlang
+from_message(request, #{ path := Path }, Opts) -> term_to_path_parts(Path, Opts);
+```
+
+### from_message
+
+Extract the request path or hashpath from a message. We do not use
+
+```erlang
+from_message(request, #{ <<"path">> := Path }, Opts) -> term_to_path_parts(Path, Opts);
+```
+
+### from_message
+
+Extract the request path or hashpath from a message. We do not use
+
+```erlang
+from_message(request, #{ <<"Path">> := Path }, Opts) -> term_to_path_parts(Path, Opts);
+```
+
+### from_message
+
+Extract the request path or hashpath from a message. We do not use
+Convert a term into an executable path. Supports binaries, lists, and
+
+```erlang
+from_message(request, _, _Opts) -> undefined.
+```
+
+### term_to_path_parts
+
+Extract the request path or hashpath from a message. We do not use
+Convert a term into an executable path. Supports binaries, lists, and
+
+```erlang
+term_to_path_parts(Path) ->
+ term_to_path_parts(Path, #{ error_strategy => throw }).
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts(Link, Opts) when ?IS_LINK(Link) ->
+ term_to_path_parts(hb_cache:ensure_loaded(Link, Opts), Opts);
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts([], _Opts) -> undefined;
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts(<<>>, _Opts) -> undefined;
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts(<<"/">>, _Opts) -> [];
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts(Binary, Opts) when is_binary(Binary) ->
+ case binary:match(Binary, <<"/">>) of
+ nomatch -> [Binary];
+ _ ->
+ term_to_path_parts(
+ binary:split(Binary, <<"/">>, [global, trim_all]),
+ Opts
+ )
+ end;
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts(Path = [ASCII | _], _Opts) when is_integer(ASCII) ->
+ [hb_ao:normalize_key(Path)];
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts(List, Opts) when is_list(List) ->
+ lists:flatten(lists:map(
+ fun(Part) ->
+ term_to_path_parts(Part, Opts)
+ end,
+ List
+ ));
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts(Atom, _Opts) when is_atom(Atom) -> [Atom];
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts(Integer, _Opts) when is_integer(Integer) ->
+ [hb_ao:normalize_key(Integer)];
+```
+
+### term_to_path_parts
+
+```erlang
+term_to_path_parts({as, DevName, Msgs}, _Opts) ->
+ [{as, hb_ao:normalize_key(DevName), Msgs}].
+```
+
+### to_binary
+
+Convert a path of any form to a binary.
+
+```erlang
+to_binary(Path) ->
+ Parts = binary:split(do_to_binary(Path), <<"/">>, [global, trim_all]),
+ iolist_to_binary(lists:join(<<"/">>, Parts)).
+```
+
+### do_to_binary
+
+Convert a path of any form to a binary.
+
+```erlang
+do_to_binary(Path) when is_list(Path) ->
+ case hb_util:is_string_list(Path) of
+ false ->
+ iolist_to_binary(
+ lists:join(
+ "/",
+ lists:filtermap(
+ fun(Part) ->
+ case do_to_binary(Part) of
+ <<>> -> false;
+ BinPart -> {true, BinPart}
+ end
+ end,
+ Path
+ )
+ )
+ );
+ true ->
+ to_binary(list_to_binary(Path))
+ end;
+```
+
+### do_to_binary
+
+Convert a path of any form to a binary.
+
+```erlang
+do_to_binary(Path) when is_binary(Path) ->
+ Path;
+```
+
+### do_to_binary
+
+Convert a path of any form to a binary.
+
+```erlang
+do_to_binary(Other) ->
+ hb_ao:normalize_key(Other).
+```
+
+### matches
+
+Check if two keys match.
+
+```erlang
+matches(Key1, Key2) ->
+ hb_util:to_lower(hb_ao:normalize_key(Key1)) ==
+ hb_util:to_lower(hb_ao:normalize_key(Key2)).
+```
+
+### regex_matches
+
+Check if two keys match using regex.
+
+```erlang
+regex_matches(Path1, Path2) ->
+ NormP1 = normalize(hb_ao:normalize_key(Path1)),
+ NormP2 =
+ case hb_ao:normalize_key(Path2) of
+ Normalized = <<"^", _/binary>> -> Normalized;
+ Normalized -> normalize(Normalized)
+ end,
+ try re:run(NormP1, NormP2) =/= nomatch
+ catch _A:_B:_C -> false
+ end.
+```
+
+### normalize
+
+Normalize a path to a binary, removing the leading slash if present.
+
+```erlang
+normalize(Path) ->
+ case iolist_to_binary([Path]) of
+ BinPath = <<"/", _/binary>> -> BinPath;
+ Binary -> <<"/", Binary/binary>>
+ end.
+```
+
+### hashpath_test
+
+```erlang
+hashpath_test() ->
+ Msg1 = #{ priv => #{<<"empty">> => <<"message">>} },
+ Msg2 = #{ priv => #{<<"exciting">> => <<"message2">>} },
+ Hashpath = hashpath(Msg1, Msg2, #{}),
+ ?assert(is_binary(Hashpath) andalso byte_size(Hashpath) == 87).
+```
+
+### hashpath_direct_msg2_test
+
+```erlang
+hashpath_direct_msg2_test() ->
+ Msg1 = #{ <<"base">> => <<"message">> },
+ Msg2 = #{ <<"path">> => <<"base">> },
+ Hashpath = hashpath(Msg1, Msg2, #{}),
+ [_, KeyName] = term_to_path_parts(Hashpath),
+ ?assert(matches(KeyName, <<"base">>)).
+```
+
+### multiple_hashpaths_test
+
+```erlang
+multiple_hashpaths_test() ->
+ Msg1 = #{ <<"empty">> => <<"message">> },
+ Msg2 = #{ <<"exciting">> => <<"message2">> },
+ Msg3 = #{ priv => #{<<"hashpath">> => hashpath(Msg1, Msg2, #{}) } },
+ Msg4 = #{ <<"exciting">> => <<"message4">> },
+ Msg5 = hashpath(Msg3, Msg4, #{}),
+ ?assert(is_binary(Msg5)).
+```
+
+### verify_hashpath_test
+
+```erlang
+verify_hashpath_test() ->
+ Msg1 = #{ <<"test">> => <<"initial">> },
+ Msg2 = #{ <<"firstapplied">> => <<"msg2">> },
+ Msg3 = #{ priv => #{<<"hashpath">> => hashpath(Msg1, Msg2, #{})} },
+ Msg4 = #{ priv => #{<<"hashpath">> => hashpath(Msg2, Msg3, #{})} },
+ Msg3Fake = #{ priv => #{<<"hashpath">> => hashpath(Msg4, Msg2, #{})} },
+ ?assert(verify_hashpath([Msg1, Msg2, Msg3, Msg4], #{})),
+ ?assertNot(verify_hashpath([Msg1, Msg2, Msg3Fake, Msg4], #{})).
+```
+
+### validate_path_transitions
+
+```erlang
+validate_path_transitions(X, Opts) ->
+ {Head, X2} = pop_request(X, Opts),
+ ?assertEqual(<<"a">>, Head),
+ {H2, X3} = pop_request(X2, Opts),
+ ?assertEqual(<<"b">>, H2),
+ {H3, X4} = pop_request(X3, Opts),
+ ?assertEqual(<<"c">>, H3),
+ ?assertEqual(undefined, pop_request(X4, Opts)).
+```
+
+### pop_from_message_test
+
+```erlang
+pop_from_message_test() ->
+ validate_path_transitions(#{ <<"path">> => [<<"a">>, <<"b">>, <<"c">>] }, #{}).
+```
+
+### pop_from_path_list_test
+
+```erlang
+pop_from_path_list_test() ->
+ validate_path_transitions([<<"a">>, <<"b">>, <<"c">>], #{}).
+```
+
+### hd_test
+
+```erlang
+hd_test() ->
+ ?assertEqual(<<"a">>, hd(#{ <<"path">> => [<<"a">>, <<"b">>, <<"c">>] }, #{})),
+ ?assertEqual(undefined, hd(#{ <<"path">> => undefined }, #{})).
+```
+
+### tl_test
+
+```erlang
+tl_test() ->
+ ?assertMatch([<<"b">>, <<"c">>], hb_maps:get(<<"path">>, tl(#{ <<"path">> => [<<"a">>, <<"b">>, <<"c">>] }, #{}))),
+ ?assertEqual(undefined, tl(#{ <<"path">> => [] }, #{})),
+ ?assertEqual(undefined, tl(#{ <<"path">> => <<"a">> }, #{})),
+ ?assertEqual(undefined, tl(#{ <<"path">> => undefined }, #{})),
+ ?assertEqual([<<"b">>, <<"c">>], tl([<<"a">>, <<"b">>, <<"c">>], #{ })),
+ ?assertEqual(undefined, tl([<<"c">>], #{ })).
+```
+
+### to_binary_test
+
+```erlang
+to_binary_test() ->
+ ?assertEqual(<<"a/b/c">>, to_binary([<<"a">>, <<"b">>, <<"c">>])),
+ ?assertEqual(<<"a/b/c">>, to_binary(<<"a/b/c">>)),
+ ?assertEqual(<<"a/b/c">>, to_binary([<<"a">>, <<"b">>, <<"c">>])),
+ ?assertEqual(<<"a/b/c">>, to_binary(["a", <<"b">>, <<"c">>])),
+ ?assertEqual(<<"a/b/b/c">>, to_binary([<<"a">>, [<<"b">>, <<"//b">>], <<"c">>])).
+```
+
+### term_to_path_parts_test
+
+```erlang
+term_to_path_parts_test() ->
+ ?assert(matches([<<"a">>, <<"b">>, <<"c">>],
+ term_to_path_parts(<<"a/b/c">>))),
+ ?assert(matches([<<"a">>, <<"b">>, <<"c">>],
+ term_to_path_parts([<<"a">>, <<"b">>, <<"c">>]))),
+ ?assert(matches([<<"a">>, <<"b">>, <<"c">>],
+ term_to_path_parts(["a", <<"b">>, <<"c">>]))),
+ ?assert(matches([<<"a">>, <<"b">>, <<"b">>, <<"c">>],
+ term_to_path_parts([[<<"/a">>, [<<"b">>, <<"//b">>], <<"c">>]]))),
+ ?assertEqual([], term_to_path_parts(<<"/">>)).
+% calculate_multistage_hashpath_test() ->
+% Msg1 = #{ <<"base">> => <<"message">> },
+% Msg2 = #{ <<"path">> => <<"2">> },
+% Msg3 = #{ <<"path">> => <<"3">> },
+% Msg4 = #{ <<"path">> => <<"4">> },
+% Msg5 = hashpath(Msg1, [Msg2, Msg3, Msg4], #{}),
+% ?assert(is_binary(Msg5)),
+% Msg3Path = <<"3">>,
+% Msg5b = hashpath(Msg1, [Msg2, Msg3Path, Msg4]),
+% ?assertEqual(Msg5, Msg5b).
+```
+
+### regex_matches_test
+
+```erlang
+regex_matches_test() ->
+ ?assert(regex_matches(<<"a/b/c">>, <<"a/.*/c">>)),
+ ?assertNot(regex_matches(<<"a/b/c">>, <<"a/.*/d">>)),
+ ?assert(regex_matches(<<"a/abcd/c">>, <<"a/abc.*/c">>)),
+ ?assertNot(regex_matches(<<"a/bcd/c">>, <<"a/abc.*/c">>)),
+ ?assert(regex_matches(<<"a/bcd/ignored/c">>, <<"a/.*/c">>)),
+```
+
+---
+
+*Generated from [hb_path.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_path.erl)*
diff --git a/docs/book/src/hb_persistent.erl.md b/docs/book/src/hb_persistent.erl.md
new file mode 100644
index 000000000..16e4bcf83
--- /dev/null
+++ b/docs/book/src/hb_persistent.erl.md
@@ -0,0 +1,585 @@
+# hb_persistent
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_persistent.erl)
+
+Creates and manages long-lived AO-Core resolution processes.
+These can be useful for situations where a message is large and expensive
+to serialize and deserialize, or when executions should be deliberately
+serialized to avoid parallel executions of the same computation. This
+module is called during the core `hb_ao` execution process, so care
+must be taken to avoid recursive spawns/loops.
+Built using the `pg` module, which is a distributed Erlang process group
+manager.
+
+---
+
+## Exported Functions
+
+- `await/4`
+- `default_await/5`
+- `default_grouper/3`
+- `default_worker/3`
+- `find_or_register/3`
+- `forward_work/2`
+- `group/3`
+- `notify/4`
+- `start_monitor/0`
+- `start_monitor/1`
+- `start_worker/2`
+- `start_worker/3`
+- `stop_monitor/1`
+- `unregister_notify/4`
+
+---
+
+### start
+
+Creates and manages long-lived AO-Core resolution processes.
+Ensure that the `pg` module is started.
+Start a monitor that prints the current members of the group every
+
+```erlang
+start() -> hb_name:start().
+```
+
+### start_monitor
+
+Creates and manages long-lived AO-Core resolution processes.
+Ensure that the `pg` module is started.
+Start a monitor that prints the current members of the group every
+
+```erlang
+start_monitor() ->
+ start_monitor(global).
+```
+
+### start_monitor
+
+```erlang
+start_monitor(Group) ->
+ start_monitor(Group, #{}).
+```
+
+### start_monitor
+
+```erlang
+start_monitor(Group, Opts) ->
+ start(),
+ ?event({worker_monitor, {start_monitor, Group, hb_name:all()}}),
+ spawn(fun() -> do_monitor(Group, #{}, Opts) end).
+```
+
+### stop_monitor
+
+```erlang
+stop_monitor(PID) ->
+ PID ! stop.
+```
+
+### do_monitor
+
+```erlang
+do_monitor(Group, Last, Opts) ->
+ Groups = lists:map(fun({Name, _}) -> Name end, hb_name:all()),
+ New =
+ hb_maps:from_list(
+ lists:map(
+ fun(G) ->
+ Pid = hb_name:lookup(G),
+ {
+ G,
+ #{
+ pid => Pid,
+ messages =>
+ case Pid of
+ undefined -> 0;
+ _ ->
+ length(
+ element(2,
+ erlang:process_info(Pid, messages)
+ )
+ )
+ end
+ }
+ }
+ end,
+ case Group of
+ global -> Groups;
+ TargetGroup ->
+ case lists:member(TargetGroup, Groups) of
+ true -> [TargetGroup];
+ false -> []
+ end
+ end
+ )
+ ),
+ Delta =
+ hb_maps:filter(
+ fun(G, NewState) ->
+ case hb_maps:get(G, Last, []) of
+ NewState -> false;
+ _ -> true
+ end
+ end,
+ New,
+ Opts
+ ),
+ case hb_maps:size(Delta, Opts) of
+ 0 -> ok;
+ Deltas ->
+ io:format(standard_error, "== Sitrep ==> ~p named processes. ~p changes. ~n",
+ [hb_maps:size(New, Opts), Deltas]),
+ hb_maps:map(
+ fun(G, #{pid := P, messages := Msgs}) ->
+ io:format(standard_error, "[~p: ~p] #M: ~p~n", [G, P, Msgs])
+ end,
+ Delta,
+ Opts
+ ),
+ io:format(standard_error, "~n", [])
+ end,
+ timer:sleep(1000),
+ receive stop -> stopped
+ after 0 -> do_monitor(Group, New, Opts)
+ end.
+```
+
+### find_or_register
+
+Register the process to lead an execution if none is found, otherwise
+
+```erlang
+find_or_register(Msg1, Msg2, Opts) ->
+ GroupName = group(Msg1, Msg2, Opts),
+ find_or_register(GroupName, Msg1, Msg2, Opts).
+```
+
+### find_or_register
+
+```erlang
+find_or_register(ungrouped_exec, _Msg1, _Msg2, _Opts) ->
+ {leader, ungrouped_exec};
+```
+
+### find_or_register
+
+```erlang
+find_or_register(GroupName, _Msg1, _Msg2, Opts) ->
+ case hb_opts:get(await_inprogress, false, Opts) of
+ false -> {leader, GroupName};
+ _ ->
+ Self = self(),
+ case find_execution(GroupName, Opts) of
+ {ok, Leader} when Leader =/= Self ->
+ ?event({found_leader, GroupName, {leader, Leader}}),
+ {wait, Leader};
+ {ok, Leader} when Leader =:= Self ->
+ {infinite_recursion, GroupName};
+ _ ->
+ ?event({register_resolver, {group, GroupName}}),
+ register_groupname(GroupName, Opts),
+ {leader, GroupName}
+ end
+ end.
+```
+
+### unregister_notify
+
+Unregister as the leader for an execution and notify waiting processes.
+
+```erlang
+unregister_notify(ungrouped_exec, _Msg2, _Msg3, _Opts) -> ok;
+```
+
+### unregister_notify
+
+Unregister as the leader for an execution and notify waiting processes.
+
+```erlang
+unregister_notify(GroupName, Msg2, Msg3, Opts) ->
+ unregister_groupname(GroupName, Opts),
+ notify(GroupName, Msg2, Msg3, Opts).
+```
+
+### find_execution
+
+Find a group with the given name.
+
+```erlang
+find_execution(Groupname, _Opts) ->
+ start(),
+ case hb_name:lookup(Groupname) of
+ undefined -> not_found;
+ Pid -> {ok, Pid}
+ end.
+```
+
+### group
+
+Calculate the group name for a Msg1 and Msg2 pair. Uses the Msg1's
+
+```erlang
+group(Msg1, Msg2, Opts) ->
+ Grouper =
+ hb_maps:get(grouper, hb_ao:info(Msg1, Opts), fun default_grouper/3, Opts),
+ apply(
+ Grouper,
+ hb_ao:truncate_args(Grouper, [Msg1, Msg2, Opts])
+ ).
+```
+
+### register_groupname
+
+Register for performing an AO-Core resolution.
+
+```erlang
+register_groupname(Groupname, _Opts) ->
+ ?event({registering_as, Groupname}),
+ hb_name:register(Groupname).
+```
+
+### unregister
+
+Unregister for being the leader on an AO-Core resolution.
+
+```erlang
+unregister(Msg1, Msg2, Opts) ->
+ start(),
+ unregister_groupname(group(Msg1, Msg2, Opts), Opts).
+```
+
+### unregister_groupname
+
+```erlang
+unregister_groupname(Groupname, _Opts) ->
+ ?event({unregister_resolver, {explicit, Groupname}}),
+ hb_name:unregister(Groupname).
+```
+
+### await
+
+If there was already an Erlang process handling this execution,
+
+```erlang
+await(Worker, Msg1, Msg2, Opts) ->
+ % Get the device's await function, if it exists.
+```
+
+### default_await
+
+Default await function that waits for a resolution from a worker.
+
+```erlang
+default_await(Worker, GroupName, Msg1, Msg2, Opts) ->
+ % Wait for the result.
+```
+
+### notify
+
+Check our inbox for processes that are waiting for the resolution
+
+```erlang
+notify(GroupName, Msg2, Msg3, Opts) ->
+ case is_binary(GroupName) of
+ true ->
+ ?event({notifying_all, {group, GroupName}});
+ false ->
+ ok
+ end,
+ receive
+ {resolve, Listener, GroupName, Msg2, _ListenerOpts} ->
+ ?event({notifying_listener, {listener, Listener}, {group, GroupName}}),
+ send_response(Listener, GroupName, Msg2, Msg3),
+ notify(GroupName, Msg2, Msg3, Opts)
+ after 0 ->
+ ?event(finished_notify),
+ ok
+ end.
+```
+
+### forward_work
+
+Forward requests to a newly delegated execution process.
+
+```erlang
+forward_work(NewPID, Opts) ->
+ Gather =
+ fun Gather() ->
+ receive
+ Req = {resolve, _, _, _, _} -> [Req | Gather()]
+ after 0 -> []
+ end
+ end,
+ ToForward = Gather(),
+ lists:foreach(
+ fun(Req) ->
+ NewPID ! Req
+ end,
+ ToForward
+ ),
+ case length(ToForward) > 0 of
+ true ->
+ ?event({fwded, {reqs, length(ToForward)}, {pid, NewPID}}, Opts);
+ false -> ok
+ end,
+ ok.
+```
+
+### send_response
+
+Helper function that wraps responding with a new Msg3.
+
+```erlang
+send_response(Listener, GroupName, Msg2, Msg3) ->
+ ?event(worker,
+ {send_response,
+ {listener, Listener},
+ {group, GroupName}
+ }
+ ),
+ Listener ! {resolved, self(), GroupName, Msg2, Msg3}.
+```
+
+### start_worker
+
+Start a worker process that will hold a message in memory for
+
+```erlang
+start_worker(Msg, Opts) ->
+ start_worker(group(Msg, undefined, Opts), Msg, Opts).
+```
+
+### start_worker
+
+```erlang
+start_worker(_, NotMsg, _) when not is_map(NotMsg) -> not_started;
+```
+
+### start_worker
+
+```erlang
+start_worker(GroupName, Msg, Opts) ->
+ start(),
+ ?event(worker_spawns,
+ {starting_worker, {group, GroupName}, {msg, Msg}, {opts, Opts}}
+ ),
+ WorkerPID = spawn(
+ fun() ->
+ % If the device's info contains a `worker' function we
+ % use that instead of the default implementation.
+```
+
+### default_worker
+
+A server function for handling persistent executions.
+
+```erlang
+default_worker(GroupName, Msg1, Opts) ->
+ Timeout = hb_opts:get(worker_timeout, 10000, Opts),
+ worker_event(GroupName, default_worker_waiting_for_req, Msg1, undefined, Opts),
+ receive
+ {resolve, Listener, GroupName, Msg2, ListenerOpts} ->
+ ?event(worker,
+ {work_received,
+ {listener, Listener},
+ {group, GroupName}
+ }
+ ),
+ Res =
+ hb_ao:resolve(
+ Msg1,
+ Msg2,
+ hb_maps:merge(ListenerOpts, Opts, Opts)
+ ),
+ send_response(Listener, GroupName, Msg2, Res),
+ notify(GroupName, Msg2, Res, Opts),
+ case hb_opts:get(static_worker, false, Opts) of
+ true ->
+ % Reregister for the existing group name.
+```
+
+### default_grouper
+
+Create a group name from a Msg1 and Msg2 pair as a tuple.
+
+```erlang
+default_grouper(Msg1, Msg2, Opts) ->
+ %?event({calculating_default_group_name, {msg1, Msg1}, {msg2, Msg2}}),
+ % Use Erlang's `phash2' to hash the result of the Grouper function.
+```
+
+### worker_event
+
+Log an event with the worker process. If we used the default grouper
+
+```erlang
+worker_event(Group, Data, Msg1, Msg2, Opts) when is_integer(Group) ->
+ ?event(worker, {worker_event, Group, Data, {msg1, Msg1}, {msg2, Msg2}}, Opts);
+```
+
+### worker_event
+
+Log an event with the worker process. If we used the default grouper
+
+```erlang
+worker_event(Group, Data, _, _, Opts) ->
+ ?event(worker, {worker_event, Group, Data}, Opts).
+```
+
+### test_device
+
+```erlang
+test_device() -> test_device(#{}).
+```
+
+### test_device
+
+```erlang
+test_device(Base) ->
+ #{
+ info =>
+ fun() ->
+ hb_maps:merge(
+ #{
+ grouper =>
+ fun(M1, _M2, _Opts) ->
+ erlang:phash2(M1)
+ end
+ },
+ Base
+ )
+ end,
+ slow_key =>
+ fun(_, #{ <<"wait">> := Wait }) ->
+ ?event({slow_key_wait_started, Wait}),
+ receive after Wait ->
+ {ok,
+ #{
+ waited => Wait,
+ pid => self(),
+ random_bytes =>
+ hb_util:encode(crypto:strong_rand_bytes(4))
+ }
+ }
+ end
+ end,
+ self =>
+ fun(M1, #{ <<"wait">> := Wait }) ->
+ ?event({self_waiting, {wait, Wait}}),
+ receive after Wait ->
+ ?event({self_returning, M1, {wait, Wait}}),
+ {ok, M1}
+ end
+ end
+ }.
+```
+
+### spawn_test_client
+
+```erlang
+spawn_test_client(Msg1, Msg2) ->
+ spawn_test_client(Msg1, Msg2, #{}).
+```
+
+### spawn_test_client
+
+```erlang
+spawn_test_client(Msg1, Msg2, Opts) ->
+ Ref = make_ref(),
+ TestParent = self(),
+ spawn_link(fun() ->
+ ?event({new_concurrent_test_resolver, Ref, {executing, Msg2}}),
+ Res = hb_ao:resolve(Msg1, Msg2, Opts),
+ ?event({test_worker_got_result, Ref, {result, Res}}),
+ TestParent ! {result, Ref, Res}
+ end),
+ Ref.
+```
+
+### wait_for_test_result
+
+```erlang
+wait_for_test_result(Ref) ->
+ receive {result, Ref, Res} -> Res end.
+```
+
+### deduplicated_execution_test
+
+Test merging and returning a value with a persistent worker.
+
+```erlang
+deduplicated_execution_test() ->
+ TestTime = 200,
+ Msg1 = #{ <<"device">> => test_device() },
+ Msg2 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => TestTime },
+ T0 = hb:now(),
+ Ref1 = spawn_test_client(Msg1, Msg2),
+ receive after 100 -> ok end,
+ Ref2 = spawn_test_client(Msg1, Msg2),
+ Res1 = wait_for_test_result(Ref1),
+ Res2 = wait_for_test_result(Ref2),
+ T1 = hb:now(),
+ % Check the result is the same.
+```
+
+### persistent_worker_test
+
+Test spawning a default persistent worker.
+
+```erlang
+persistent_worker_test() ->
+ TestTime = 200,
+ Msg1 = #{ <<"device">> => test_device() },
+ link(start_worker(Msg1, #{ static_worker => true })),
+ receive after 10 -> ok end,
+ Msg2 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => TestTime },
+ Msg3 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.1) },
+ Msg4 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.2) },
+ T0 = hb:now(),
+ Ref1 = spawn_test_client(Msg1, Msg2),
+ Ref2 = spawn_test_client(Msg1, Msg3),
+ Ref3 = spawn_test_client(Msg1, Msg4),
+ Res1 = wait_for_test_result(Ref1),
+ Res2 = wait_for_test_result(Ref2),
+ Res3 = wait_for_test_result(Ref3),
+ T1 = hb:now(),
+ ?assertNotEqual(Res1, Res2),
+ ?assertNotEqual(Res2, Res3),
+ ?assert(T1 - T0 >= (3*TestTime)).
+```
+
+### spawn_after_execution_test
+
+```erlang
+spawn_after_execution_test() ->
+ ?event(<<"">>),
+ TestTime = 500,
+ Msg1 = #{ <<"device">> => test_device() },
+ Msg2 = #{ <<"path">> => <<"self">>, <<"wait">> => TestTime },
+ Msg3 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.1) },
+ Msg4 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.2) },
+ T0 = hb:now(),
+ Ref1 =
+ spawn_test_client(
+ Msg1,
+ Msg2,
+ #{
+ spawn_worker => true,
+ static_worker => true,
+ hashpath => ignore
+ }
+ ),
+ receive after 10 -> ok end,
+ Ref2 = spawn_test_client(Msg1, Msg3),
+ Ref3 = spawn_test_client(Msg1, Msg4),
+ Res1 = wait_for_test_result(Ref1),
+ Res2 = wait_for_test_result(Ref2),
+ Res3 = wait_for_test_result(Ref3),
+ T1 = hb:now(),
+ ?assertNotEqual(Res1, Res2),
+ ?assertNotEqual(Res2, Res3),
+```
+
+---
+
+*Generated from [hb_persistent.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_persistent.erl)*
diff --git a/docs/book/src/hb_private.erl.md b/docs/book/src/hb_private.erl.md
new file mode 100644
index 000000000..353a46deb
--- /dev/null
+++ b/docs/book/src/hb_private.erl.md
@@ -0,0 +1,279 @@
+# hb_private
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_private.erl)
+
+This module provides basic helper utilities for managing the
+private element of a message, which can be used to store state that is
+not included in serialized messages, or those granted to users via the
+APIs. Private elements of a message can be useful for storing state that
+is only relevant temporarily. For example, a device might use the private
+element to store a cache of values that are expensive to recompute. They
+should _not_ be used for encoding state that makes the execution of a
+device non-deterministic (unless you are sure you know what you are doing).
+The `set` and `get` functions of this module allow you to run those keys
+as AO-Core paths if you would like to have private `devices` in the
+messages non-public zone.
+See `hb_ao` for more information about the AO-Core protocol
+and private elements of messages.
+
+---
+
+## Exported Functions
+
+- `from_message/1`
+- `get/3`
+- `get/4`
+- `is_private/1`
+- `merge/3`
+- `opts/1`
+- `reset/1`
+- `set_priv/2`
+- `set/3`
+- `set/4`
+
+---
+
+### from_message
+
+This module provides basic helper utilities for managing the
+Return the `private` key from a message. If the key does not exist, an
+
+```erlang
+from_message(Msg) when is_map(Msg) ->
+ case maps:is_key(<<"priv">>, Msg) of
+ true -> maps:get(<<"priv">>, Msg, #{});
+ false -> maps:get(priv, Msg, #{})
+ end;
+```
+
+### from_message
+
+This module provides basic helper utilities for managing the
+Return the `private` key from a message. If the key does not exist, an
+Helper for getting a value from the private element of a message. Uses
+
+```erlang
+from_message(_NonMapMessage) -> #{}.
+```
+
+### get
+
+This module provides basic helper utilities for managing the
+Return the `private` key from a message. If the key does not exist, an
+Helper for getting a value from the private element of a message. Uses
+
+```erlang
+get(Key, Msg, Opts) ->
+ get(Key, Msg, not_found, Opts).
+```
+
+### get
+
+```erlang
+get(InputPath, Msg, Default, Opts) ->
+ % Resolve the path against the private element of the message.
+```
+
+### set
+
+Helper function for setting a key in the private element of a message.
+
+```erlang
+set(Msg, InputPath, Value, Opts) ->
+ Path = remove_private_specifier(InputPath, Opts),
+ Priv = from_message(Msg),
+ ?event({set_private, {in, Path}, {out, Path}, {value, Value}, {opts, Opts}}),
+ NewPriv = hb_util:deep_set(Path, Value, Priv, opts(Opts)),
+ ?event({set_private_res, {out, NewPriv}}),
+ set_priv(Msg, NewPriv).
+```
+
+### set
+
+```erlang
+set(Msg, PrivMap, Opts) ->
+ CurrentPriv = from_message(Msg),
+ ?event({set_private, {in, PrivMap}, {opts, Opts}}),
+ NewPriv = hb_util:deep_merge(CurrentPriv, PrivMap, opts(Opts)),
+ ?event({set_private_res, {out, NewPriv}}),
+ set_priv(Msg, NewPriv).
+```
+
+### merge
+
+Merge the private elements of two messages into one. The keys in the
+
+```erlang
+merge(Msg1, Msg2, Opts) ->
+ % Merge the private elements of the two messages.
+```
+
+### set_priv
+
+Helper function for setting the complete private element of a message.
+
+```erlang
+set_priv(Msg, PrivMap)
+ when map_size(PrivMap) =:= 0 andalso not is_map_key(<<"priv">>, Msg) ->
+ Msg;
+```
+
+### set_priv
+
+Helper function for setting the complete private element of a message.
+Check if a key is private.
+
+```erlang
+set_priv(Msg, PrivMap) ->
+ Msg#{ <<"priv">> => PrivMap }.
+```
+
+### is_private
+
+Helper function for setting the complete private element of a message.
+Check if a key is private.
+
+```erlang
+is_private(Key) ->
+ try hb_util:bin(Key) of
+ <<"priv", _/binary>> -> true;
+ _ -> false
+ catch _:_ -> false
+ end.
+```
+
+### remove_private_specifier
+
+Remove the first key from the path if it is a private specifier.
+
+```erlang
+remove_private_specifier(InputPath, Opts) ->
+ case is_private(hd(Path = hb_path:term_to_path_parts(InputPath, Opts))) of
+ true -> tl(Path);
+ false -> Path
+ end.
+```
+
+### opts
+
+The opts map that should be used when resolving paths against the
+
+```erlang
+opts(Opts) ->
+ PrivStore =
+ case hb_opts:get(priv_store, undefined, Opts) of
+ undefined -> [];
+ PrivateStores when is_list(PrivateStores) -> PrivateStores;
+ PrivateStore -> [PrivateStore]
+ end,
+ BaseStore =
+ case hb_opts:get(store, [], Opts) of
+ SingleStore when is_map(SingleStore) -> [SingleStore];
+ Stores when is_list(Stores) -> Stores
+ end,
+ NormStore = PrivStore ++ BaseStore,
+ Opts#{
+ hashpath => ignore,
+ cache_control => [<<"no-cache">>, <<"no-store">>],
+ store => NormStore
+ }.
+```
+
+### reset
+
+Unset all of the private keys in a message or deep Erlang term.
+
+```erlang
+reset(Msg) when is_map(Msg) ->
+ maps:map(
+ fun(_Key, Val) -> reset(Val) end,
+ maps:without(
+ lists:filter(fun is_private/1, maps:keys(Msg)),
+ Msg
+ )
+ );
+```
+
+### reset
+
+Unset all of the private keys in a message or deep Erlang term.
+
+```erlang
+reset(List) when is_list(List) ->
+ % Check if any of the terms in the list are private specifiers, return an
+ % empty list if so.
+```
+
+### reset
+
+```erlang
+reset(Tuple) when is_tuple(Tuple) ->
+ list_to_tuple(reset(tuple_to_list(Tuple)));
+```
+
+### reset
+
+```erlang
+reset(NonMapMessage) ->
+ NonMapMessage.
+```
+
+### set_private_test
+
+```erlang
+set_private_test() ->
+ ?assertEqual(
+ #{<<"a">> => 1, <<"priv">> => #{<<"b">> => 2}},
+ set(#{<<"a">> => 1}, <<"b">>, 2, #{})
+ ),
+ Res = set(#{<<"a">> => 1}, <<"a">>, 1, #{}),
+ ?assertEqual(#{<<"a">> => 1, <<"priv">> => #{<<"a">> => 1}}, Res),
+ ?assertEqual(
+ #{<<"a">> => 1, <<"priv">> => #{<<"a">> => 1}},
+ set(Res, <<"a">>, 1, #{})
+ ).
+```
+
+### get_private_key_test
+
+```erlang
+get_private_key_test() ->
+ M1 = #{<<"a">> => 1, <<"priv">> => #{<<"b">> => 2}},
+ ?assertEqual(not_found, get(<<"a">>, M1, #{})),
+ {ok, [<<"a">>]} = hb_ao:resolve(M1, <<"keys">>, #{}),
+ ?assertEqual(2, get(<<"b">>, M1, #{})),
+ {error, _} = hb_ao:resolve(M1, <<"priv/a">>, #{}),
+ {error, _} = hb_ao:resolve(M1, <<"priv">>, #{}).
+```
+
+### get_deep_key_test
+
+```erlang
+get_deep_key_test() ->
+ M1 = #{<<"a">> => 1, <<"priv">> => #{<<"b">> => #{<<"c">> => 3}}},
+ ?assertEqual(3, get(<<"b/c">>, M1, #{})).
+```
+
+### priv_opts_store_read_link_test
+
+```erlang
+priv_opts_store_read_link_test() ->
+ % Write a message to the public store.
+```
+
+### priv_opts_cache_read_message_test
+
+```erlang
+priv_opts_cache_read_message_test() ->
+ hb:init(),
+ PublicStore = [hb_test_utils:test_store(hb_store_lmdb)],
+ OnlyPrivStore = [hb_test_utils:test_store(hb_store_fs)],
+ Opts = #{ store => PublicStore, priv_store => OnlyPrivStore },
+ PrivOpts = opts(Opts),
+ % Use the `~scheduler@1.0' and `~process@1.0' infrastructure to write a
+ % complex message into the public store.
+```
+
+---
+
+*Generated from [hb_private.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_private.erl)*
diff --git a/docs/book/src/hb_process_monitor.erl.md b/docs/book/src/hb_process_monitor.erl.md
new file mode 100644
index 000000000..ea96dae16
--- /dev/null
+++ b/docs/book/src/hb_process_monitor.erl.md
@@ -0,0 +1,106 @@
+# hb_process_monitor
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_process_monitor.erl)
+
+## Exported Functions
+
+- `start/1`
+- `start/2`
+- `start/3`
+- `stop/1`
+
+---
+
+### start
+
+```erlang
+start(ProcID) ->
+ start(ProcID, hb_opts:get(default_cron_rate)).
+```
+
+### start
+
+```erlang
+start(ProcID, Rate) ->
+ start(ProcID, Rate, hb_client:cron_cursor(ProcID)).
+```
+
+### start
+
+```erlang
+start(ProcID, Rate, Cursor) ->
+ Logger = hb_logger:start(),
+ Monitor = spawn(
+ fun() ->
+ server(
+ #state{
+ proc_id = ProcID,
+ cursor = Cursor,
+ logger = Logger
+ }
+ )
+ end),
+ Ticker = spawn(fun() -> ticker(Monitor, Rate) end),
+ hb_logger:register(Monitor),
+ hb_logger:log(Monitor, {ok, started_monitor, {ProcID, Rate, Cursor}}),
+ hb_logger:register(Ticker),
+ {Monitor, Logger}.
+```
+
+### stop
+
+```erlang
+stop(PID) ->
+ PID ! stop.
+```
+
+### server
+
+```erlang
+server(State) ->
+ receive
+ stop -> ok;
+ tick ->server(handle_crons(State))
+ end.
+```
+
+### handle_crons
+
+```erlang
+handle_crons(State) ->
+ case hb_client:cron(State#state.proc_id, State#state.cursor) of
+ {ok, HasNextPage, Results, Cursor} ->
+ lists:map(
+ fun(Res) ->
+ % TODO: Validate this
+ dev_mu:push(#{ message => Res }, State)
+ end,
+ Results
+ ),
+ NS = State#state{cursor = Cursor},
+ case HasNextPage of
+ true -> NS;
+ false -> handle_crons(NS)
+ end;
+ Error ->
+ hb_logger:log(State#state.logger, Error),
+ State
+ end.
+```
+
+### ticker
+
+```erlang
+ticker(Monitor, Rate) ->
+ case erlang:is_process_alive(Monitor) of
+ true ->
+ timer:sleep(Rate),
+ Monitor ! tick,
+ ticker(Monitor, Rate);
+ false ->
+ ok
+```
+
+---
+
+*Generated from [hb_process_monitor.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_process_monitor.erl)*
diff --git a/docs/book/src/hb_router.erl.md b/docs/book/src/hb_router.erl.md
new file mode 100644
index 000000000..06c229ae3
--- /dev/null
+++ b/docs/book/src/hb_router.erl.md
@@ -0,0 +1,44 @@
+# hb_router
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_router.erl)
+
+Locate a service in the AO network. This module uses
+URLs to locate services, so it can be used to locate
+nodes using IP addresses or domain names. This also
+allows us to use different protocols later, potentially.
+
+---
+
+## Exported Functions
+
+- `find/2`
+- `find/3`
+
+---
+
+### find
+
+```erlang
+find(Type, ID) ->
+ find(Type, ID, '_').
+```
+
+### find
+
+```erlang
+find(Type, ID, Address) ->
+ find(Type, ID, Address, #{}).
+```
+
+### find
+
+```erlang
+find(Type, _ID, Address, Opts) ->
+ case hb_maps:get(Type, hb_opts:get(nodes), undefined, Opts) of
+ #{ Address := Node } -> {ok, Node};
+ undefined -> {error, service_type_not_found}
+```
+
+---
+
+*Generated from [hb_router.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_router.erl)*
diff --git a/docs/book/src/hb_singleton.erl.md b/docs/book/src/hb_singleton.erl.md
new file mode 100644
index 000000000..12a3cdbca
--- /dev/null
+++ b/docs/book/src/hb_singleton.erl.md
@@ -0,0 +1,1104 @@
+# hb_singleton
+
+[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_singleton.erl)
+
+A parser that translates AO-Core HTTP API requests in TABM format
+into an ordered list of messages to evaluate. The details of this format
+are described in `docs/ao-core-http-api.md`.
+Syntax overview:
+
+ Singleton: Message containing keys and a `path` field,
+ which may also contain a query string of key-value pairs.
+ Path:
+ - /Part1/Part2/.../PartN/ => [Part1, Part2, ..., PartN]
+ - /ID/Part2/.../PartN => [ID, Part2, ..., PartN]
+ Part: (Key + Resolution), Device?, #{ K => V}?
+ - Part => #{ path => Part }
+ - `Part&Key=Value => #{ path => Part, Key => Value }`
+ - `Part=Value&... => #{ path => Part, Part => Value, ... }`
+ - `Part&Key => #{ path => Part, Key => true }`
+ - `Part&k1=v1&k2=v2 => #{ path => Part, k1 => `<<"v1">>`, k2 => `<<"v2">>` }'
+ - `Part~Device => {as, Device, #{ path => Part }}`
+ - `Part~D&K1=V1 => {as, D, #{ path => Part, K1 => `<<"v1">>` }}'
+ - `pt&k1+int=1 => #{ path => pt, k1 => 1 }`
+ - `pt~d&k1+int=1 => {as, d, #{ path => pt, k1 => 1 }}`
+ - `(/nested/path) => Resolution of the path /nested/path`
+ - `(/nested/path&k1=v1) => (resolve /nested/path)#{k1 => v1}`
+ - `(/nested/path~D&K1=V1) => (resolve /nested/path)#{K1 => V1}`
+ - `pt&k1+res=(/a/b/c) => #{ path => pt, k1 => (resolve /a/b/c) }`
+ Key:
+ - key: `<<"value">>` => #{ key => `<<"value">>`, ... } for all messages
+ - n.key: `<<"value">>` => #{ key => `<<"value">>`, ... } for Nth message
+ - key+int: 1 => #{ key => 1, ... }
+ - key+res: /nested/path => #{ key => (resolve /nested/path), ... }
+ - N.Key+res=(/a/b/c) => #{ Key => (resolve /a/b/c), ... }
+
+
+---
+
+## Exported Functions
+
+- `from_path/1`
+- `from/2`
+- `to/1`
+
+---
+
+### append_path
+
+```erlang
+-spec to(list(ao_message())) -> tabm_message().
+to(Messages) ->
+ % Iterate through all AO-Core messages folding them into the TABM message
+ % Scopes contains the following map: #{Key => [StageIndex, StageIndex2...]}
+ % that allows to scope keys to the given stage.
+```
+
+```erlang
+append_path(PathPart, #{<<"path">> := Path} = Message) ->
+ hb_maps:put(<<"path">>, <- `generate` (optional): A key that generates a secret based on a - user's request. May return either the secret - directly, or a message with a `secret` key. If - a message is returned, it is assumed to be a - modified version of the user's request and is - used for further processing. - `finalize` (optional): A key that takes the message sequence after this - device has processed it and returns it in a - modified form. --At present, the `~cookie-secret@1.0` and `~http-auth@1.0` devices implement -the `generator` interface. For example, the following hook definition will -use the `~cookie-secret@1.0` device to generate and manage wallets for -users, with authentication details stored in cookies: -
- "on": {
- "request": {
- "device": "auth-hook@1.0",
- "secret-provider": {
- "device": "cookie-secret@1.0"
- }
- }
- }
-
-`~auth-hook@1.0` expects to receive a `secret-provider` key in the hook
-base message. It may optionally also take a `generate-path` and
-`finalize-path`, which are used to generate the secret and post-process the
-response. If either `X-path` keys are not present, the `generate` and
-`finalize` paths are used upon the `secret-provider` message. If the secret
-provider's device does not implement these keys, the operations are skipped.
-Node operators may also specify a `when` message inside their hook definition
-which is used to determine when messages should be signed. The supported keys
-are:
-- `committers`: always | uncommitted | [committer1, or committer2, or ...] - `keys`: always | [key1, or key2, or ...] --Both keys are optional and can be combined to form 'and' conditions. For -example, the following hook definition will sign all uncommitted requests -that have the `Authorization` header: -
- "on": {
- "request": {
- "device": "auth-hook@1.0",
- "when": {
- "keys": ["authorization"],
- "committers": "uncommitted"
- }
- }
- }
-
-
----
-
-## Exported Functions
-
-- `request/3`
-
----
-
-### request
-
-A device offering an on-request hook that signs incoming messages with
-Process an incoming request through a key provider. The key provider
-
-```erlang
-request(Base, HookReq, Opts) ->
- ?event({auth_hook_request, {base, Base}, {hook_req, HookReq}}),
- maybe
- % Get the key provider from options and short-circuit if none is
- % provided.
-```
-
-### is_relevant
-
-Check if the request is relevant to the hook base. Node operators may
-
-```erlang
-is_relevant(Base, Request, MessageSequence, Opts) ->
- Committers = is_relevant_from_committers(Base, Request, Opts),
- Keys =
- lists:any(
- fun(Msg) -> is_relevant_from_keys(Base, Msg, Opts) end,
- [Request | MessageSequence]
- ),
- ?event({auth_hook_is_relevant, {committers, Committers}, {keys, Keys}}),
- if Committers andalso Keys -> true;
- true -> {skip, {committers, Committers}, {keys, Keys}}
- end.
-```
-
-### is_relevant_from_committers
-
-Check if the request is relevant to the hook base based on the committers
-
-```erlang
-is_relevant_from_committers(Base, Request, Opts) ->
- Config =
- hb_util:deep_get(
- [<<"when">>, <<"committers">>],
- Base,
- <<"uncommitted">>,
- Opts
- ),
- ?event({auth_hook_is_relevant_from_committers, {config, Config}, {base, Base}}),
- case Config of
- <<"always">> -> true;
- <<"uncommitted">> -> hb_message:signers(Request, Opts) == [];
- RelevantCommitters ->
- lists:any(
- fun(Signer) ->
- lists:member(Signer, RelevantCommitters)
- end,
- hb_message:signers(Request, Opts)
- )
- end.
-```
-
-### is_relevant_from_keys
-
-Check if the request is relevant to the hook base based on the presence
-
-```erlang
-is_relevant_from_keys(_Base, ID, _Opts) when is_binary(ID) ->
- false;
-```
-
-### is_relevant_from_keys
-
-Check if the request is relevant to the hook base based on the presence
-
-```erlang
-is_relevant_from_keys(Base, {as, _, Msg}, Opts) ->
- is_relevant_from_keys(Base, Msg, Opts);
-```
-
-### is_relevant_from_keys
-
-Check if the request is relevant to the hook base based on the presence
-
-```erlang
-is_relevant_from_keys(Base, {resolve, Msg}, Opts) ->
- is_relevant_from_keys(Base, Msg, Opts);
-```
-
-### is_relevant_from_keys
-
-Check if the request is relevant to the hook base based on the presence
-
-```erlang
-is_relevant_from_keys(Base, Request, Opts) ->
- Config = hb_util:deep_get([<<"when">>, <<"keys">>], Base, <<"always">>, Opts),
- ?event(
- {
- auth_hook_is_relevant_from_keys,
- {config, Config},
- {base, Base},
- {request, Request}
- }
- ),
- case Config of
- <<"always">> -> true;
- RelevantKeys ->
- lists:any(
- fun(Key) ->
- case hb_maps:find(Key, Request, Opts) of
- {ok, _} -> true;
- error -> false
- end
- end,
- RelevantKeys
- )
- end.
-```
-
-### generate_secret
-
-Normalize authentication credentials, generating new ones if needed.
-
-```erlang
-generate_secret(Provider, Request, Opts) ->
- case call_provider(<<"generate">>, Provider, Request, Opts) of
- {error, not_found} ->
- ?event({no_generate_handler, Provider}),
- {ok, Provider, strip_sensitive(Request, Opts)};
- {error, Err} ->
- % Forward the error. The main handler will fail to match this and
- % return the error to the user.
-```
-
-### strip_sensitive
-
-Strip the `secret` field from a request.
-Generate a wallet with the key if the `wallet` field is not present in
-
-```erlang
-strip_sensitive(Request, Opts) ->
- hb_maps:without([<<"secret">>], Request, Opts).
-```
-
-### generate_wallet
-
-Strip the `secret` field from a request.
-Generate a wallet with the key if the `wallet` field is not present in
-
-```erlang
-generate_wallet(Provider, Request, Opts) ->
- {ok, #{ <<"body">> := WalletID }} =
- dev_secret:generate(Provider, Request, Opts),
- ?event({generated_wallet, WalletID}),
- {ok, Provider, refresh_opts(Opts)}.
-```
-
-### sign_request
-
-Sign a request using the configured key provider
-
-```erlang
-sign_request(Provider, Msg, Opts) ->
- case hb_maps:get(<<"skip-commit">>, Provider, true, Opts) of
- false ->
- % Skip signing and return the normalized message.
-```
-
-### maybe_sign_messages
-
-Process a sequence of messages, signing those marked for signing
-
-```erlang
-maybe_sign_messages(Provider, SignedReq, Opts) ->
- Parsed = hb_singleton:from(SignedReq, Opts),
- ?event({auth_hook_parsed_messages, {sequence_length, length(Parsed)}}),
- SignKey = hb_opts:get(auth_hook_commit_key, ?DEFAULT_COMMIT_KEY, Opts),
- Processed = maybe_sign_messages(Provider, SignKey, Parsed, Opts),
- {ok, Processed}.
-```
-
-### maybe_sign_messages
-
-```erlang
-maybe_sign_messages(_Provider, _Key, [], _Opts) -> [];
-```
-
-### maybe_sign_messages
-
-```erlang
-maybe_sign_messages(Provider, Key, [Msg | Rest], Opts) when is_map(Msg) ->
- case hb_util:atom(hb_maps:get(Key, Msg, false, Opts)) of
- true ->
- Uncommitted = hb_message:uncommitted(Msg, Opts),
- ?event({auth_hook_signing_message, {uncommitted, Msg}}),
- case sign_request(Provider, Uncommitted, Opts) of
- {ok, Signed} ->
- [
- Signed
- |
- maybe_sign_messages(Provider, Key, Rest, Opts)
- ];
- {error, Err} ->
- ?event({auth_hook_sign_error, Err}),
- [{error, Err}]
- end;
- _ ->
- [Msg | maybe_sign_messages(Provider, Key, Rest, Opts)]
- end;
-```
-
-### maybe_sign_messages
-
-```erlang
-maybe_sign_messages(Provider, Key, [Msg | Rest], Opts) ->
- [Msg | maybe_sign_messages(Provider, Key, Rest, Opts)].
-```
-
-### finalize
-
-Finalize the response by adding authentication state
-
-```erlang
-finalize(KeyProvider, SignedReq, MessageSequence, Opts) ->
- % Add the signed request and message sequence to the response, mirroring the
- % structure of a normal `~hook@1.0' on-request hook.
-```
-
-### refresh_opts
-
-Refresh the options and log an event if they have changed.
-
-```erlang
-refresh_opts(Opts) ->
- NewOpts = hb_http_server:get_opts(Opts),
- case NewOpts of
- Opts -> ?event(auth_hook_no_opts_change);
- _ ->
- ?event(
- {auth_hook_opts_changed,
- {size_diff,
- erlang:external_size(NewOpts) -
- erlang:external_size(Opts)
- }
- }
- )
- end,
- NewOpts.
-```
-
-### find_provider
-
-Get the key provider from the base message or the defaults.
-
-```erlang
-find_provider(Base, Opts) ->
- case hb_maps:get(<<"secret-provider">>, Base, no_key_provider, Opts) of
- no_key_provider ->
- case hb_opts:get(hook_secret_provider, no_key_provider, Opts) of
- no_key_provider -> {error, no_key_provider};
- SecretProvider -> SecretProvider
- end;
- SecretProvider when is_binary(SecretProvider) ->
- {ok, #{ <<"device">> => SecretProvider }};
- SecretProvider when is_map(SecretProvider) ->
- {ok, SecretProvider};
- _ ->
- {error, invalid_auth_provider}
- end.
-```
-
-### call_provider
-
-Find the appropriate handler for a key in the key provider.
-
-```erlang
-call_provider(Key, Provider, Request, Opts) ->
- ?event({call_provider, {key, Key}, {provider, Provider}, {req, Request}}),
- ExecKey = hb_maps:get(<< Key/binary, "-path">>, Provider, Key, Opts),
- ?event({call_provider, {exec_key, ExecKey}}),
- case hb_ao:resolve(Provider, Request#{ <<"path">> => ExecKey }, Opts) of
- {ok, Msg} when is_map(Msg) ->
- % The result is a message. We revert the path to its original value.
-```
-
-### ignored_keys
-
-Default keys to ignore when signing
-
-```erlang
-ignored_keys(Msg, Opts) ->
- hb_maps:get(
- <<"ignored-keys">>,
- Msg,
- hb_opts:get(
- hook_auth_ignored_keys,
- ?DEFAULT_IGNORED_KEYS,
- Opts
- )
- ).
-```
-
-### cookie_test
-
-```erlang
-cookie_test() ->
- % Start a node with a secret-provider that uses the cookie device.
-```
-
-### http_auth_test
-
-```erlang
-http_auth_test() ->
- % Start a node with the `~http-auth@1.0' device as the secret-provider.
-```
-
-### chained_preprocess_test
-
-```erlang
-chained_preprocess_test() ->
- % Start a node with the `~http-auth@1.0' device as the secret-provider, with
- % a router chained afterwards in the request hook.
-```
-
-### when_test
-
-```erlang
-when_test() ->
- % Start a node with the `~http-auth@1.0' device as the secret-provider. Only
- % request commitment with the hook if the `Authorization' header is present.
-```
-
-### signers_from_commitments_response
-
-The cookie hook test(s) call `GET /commitments`, which returns the
-
-```erlang
-signers_from_commitments_response(Response, ServerWallet) ->
- ServerAddress = ar_wallet:to_address(ServerWallet),
- hb_maps:values(hb_maps:filtermap(
- fun(Key, Value) when ?IS_ID(Key) ->
- Type = hb_maps:get(<<"type">>, Value, not_found, #{}),
- Committer = hb_maps:get(<<"committer">>, Value, not_found, #{}),
- case {Type, Committer} of
- {<<"rsa-pss-sha512">>, ServerAddress} -> false;
- {<<"rsa-pss-sha512">>, _} -> {true, Committer};
- _ -> false
- end;
- (_Key, _Value) ->
- false
- end,
- Response,
- #{}
-```
-
----
-
-*Generated from [dev_auth_hook.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_auth_hook.erl)*
diff --git a/docs/book/src/dev_cache.erl.md b/docs/book/src/dev_cache.erl.md
deleted file mode 100644
index a3eae0380..000000000
--- a/docs/book/src/dev_cache.erl.md
+++ /dev/null
@@ -1,310 +0,0 @@
-# dev_cache
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_cache.erl)
-
-A device that looks up an ID from a local store and returns it,
-honoring the `accept` key to return the correct format. The cache also
-supports writing messages to the store, if the node message has the
-writer's address in its `cache_writers` key.
-
----
-
-## Exported Functions
-
-- `link/3`
-- `read/3`
-- `write/3`
-
----
-
-### read
-
-A device that looks up an ID from a local store and returns it,
-Read data from the cache.
-
-```erlang
-read(_M1, M2, Opts) ->
- Location = hb_ao:get(<<"target">>, M2, Opts),
- ?event({read, {key_extracted, Location}}),
- ?event(debug_gateway, cache_read),
- case hb_cache:read(Location, Opts) of
- {ok, Res} ->
- ?event({read, {cache_result, ok, Res}}),
- case hb_ao:get(<<"accept">>, M2, Opts) of
- <<"application/aos-2">> ->
- ?event(dev_cache,
- {read,
- {accept_header, <<"application/aos-2">>}
- }
- ),
- JSONMsg = dev_json_iface:message_to_json_struct(Res, Opts),
- ?event(dev_cache, {read, {json_message, JSONMsg}}),
- {ok,
- #{
- <<"body">> => hb_json:encode(JSONMsg),
- <<"content-type">> => <<"application/aos-2">>
- }
- };
- _ ->
- {ok, Res}
- end;
- not_found ->
- % The cache does not have this ID,but it may still be an explicit
- % `data/' path.
-```
-
-### write
-
-Write data to the cache.
-
-```erlang
-write(_M1, M2, Opts) ->
- case is_trusted_writer(M2, Opts) of
- true ->
- ?event(dev_cache, {write, {trusted_writer, true}}),
- Type = hb_ao:get(<<"type">>, M2, <<"single">>, Opts),
- ?event(dev_cache, {write, {write_type, Type}}),
- case Type of
- <<"single">> ->
- ?event(dev_cache, {write, {write_single_called}}),
- write_single(M2, Opts);
- <<"batch">> ->
- ?event(dev_cache, {write, {write_batch_called}}),
- hb_maps:map(
- fun(_, Value) ->
- ?event(dev_cache, {write, {batch_item, Value}}),
- write_single(Value, Opts)
- end,
- hb_ao:get(<<"body">>, M2, Opts),
- Opts
- );
- _ ->
- ?event(dev_cache, {write, {invalid_write_type, Type}}),
- {error,
- #{
- <<"status">> => 400,
- <<"body">> => <<"Invalid write type.">>
- }
- }
- end;
- false ->
- ?event(dev_cache, {write, {trusted_writer, false}}),
- {error,
- #{
- <<"status">> => 403,
- <<"body">> => <<"Not authorized to write to the cache.">>
- }
- }
- end.
-```
-
-### link
-
-Link a source to a destination in the cache.
-
-```erlang
-link(_Base, Req, Opts) ->
- case is_trusted_writer(Req, Opts) of
- true ->
- Source = hb_ao:get(<<"source">>, Req, Opts),
- Destination = hb_ao:get(<<"destination">>, Req, Opts),
- write_single(#{
- <<"operation">> => <<"link">>,
- <<"source">> => Source,
- <<"destination">> => Destination
- }, Opts);
- false ->
- {error, not_authorized}
- end.
-```
-
-### write_single
-
-Helper function to write a single data item to the cache.
-
-```erlang
-write_single(Msg, Opts) ->
- Body = hb_ao:get(<<"body">>, Msg, Opts),
- ?event(dev_cache, {write_single, {body_extracted, Body}}),
- Location = hb_ao:get(<<"location">>, Msg, Opts),
- ?event(dev_cache, {write_single, {location_extracted, Location}}),
- Operation = hb_ao:get(<<"operation">>, Msg, <<"write">>, Opts),
- ?event(dev_cache, {write_single, {operation, Operation}}),
- case {Operation, Body, Location} of
- {<<"write">>, not_found, _} ->
- ?event(dev_cache, {write_single, {error, "No body to write"}}),
- {error,
- #{
- <<"status">> => 400,
- <<"body">> => <<"No body to write.">>
- }
- };
- {<<"write">>, Binary, not_found} when is_binary(Binary) ->
- % When asked to write only a binary, we do not calculate any
- % alternative IDs.
-```
-
-### is_trusted_writer
-
-Verify that the request originates from a trusted writer.
-
-```erlang
-is_trusted_writer(Req, Opts) ->
- Signers = hb_message:signers(Req, Opts),
- ?event(dev_cache, {is_trusted_writer, {signers, Signers}, {req, Req}}),
- CacheWriters = hb_opts:get(cache_writers, [], Opts),
- ?event(dev_cache, {is_trusted_writer, {cache_writers, CacheWriters}}),
- AnyTrusted = lists:any(fun(Signer) -> lists:member(Signer, CacheWriters) end, Signers),
- case AnyTrusted of
- true ->
- ?event(dev_cache, {is_trusted_writer, {trusted, true}}),
- true;
- _ ->
- ?event(dev_cache, {is_trusted_writer, {trusted, false}}),
- false
- end.
-```
-
-### setup_test_env
-
-Create a test environment with a local store and node.
-
-```erlang
-setup_test_env() ->
- Timestamp = integer_to_binary(os:system_time(millisecond)),
- StorePrefix = <<"cache-TEST/remote-", Timestamp/binary>>,
- ?event(dev_cache, {setup_test_env, {start, StorePrefix}}),
- application:ensure_all_started(hb),
- ?event(dev_cache, {setup_test_env, {hb_started}}),
- LocalStore =
- #{ <<"store-module">> => hb_store_fs, <<"name">> => StorePrefix },
- ?event(dev_cache, {setup_test_env, {local_store_configured, LocalStore}}),
- hb_store:reset(LocalStore),
- ?event(dev_cache, {setup_test_env, {store_reset}}),
- Wallet = ar_wallet:new(),
- Address = hb_util:human_id(ar_wallet:to_address(Wallet)),
- ?event(dev_cache, {setup_test_env, {address, Address}}),
- Node = hb_http_server:start_node(#{
- cache_control => [<<"no-cache">>, <<"no-store">>],
- store => LocalStore,
- cache_writers => [
- Address,
- hb_util:human_id(ar_wallet:to_address(hb:wallet()))
- ],
- store_all_signed => false
- }),
- ?event(dev_cache, {setup_test_env, {node_started, Node}}),
- TestOpts = #{
- cache_control => [<<"no-cache">>, <<"no-store">>],
- store_all_signed => false,
- store => [
- #{
- <<"store-module">> => hb_store_remote_node,
- <<"node">> => Node,
- priv_wallet => Wallet
- }
- ]
- },
- {ok, TestOpts, [LocalStore, Wallet, Address, Node]}.
-```
-
-### write_to_cache
-
-Write data to the cache via HTTP.
-
-```erlang
-write_to_cache(Node, Data, Wallet) ->
- ?event(dev_cache, {write_to_cache, {start, Node}}),
- WriteMsg = #{
- <<"path">> => <<"/~cache@1.0/write">>,
- <<"method">> => <<"POST">>,
- <<"body">> => Data
- },
- ?event(dev_cache, {write_to_cache, {message_created, WriteMsg}}),
- SignedMsg = hb_message:commit(WriteMsg, Wallet),
- ?event(dev_cache, {write_to_cache, {message_signed}}),
- WriteResult = hb_http:post(Node, SignedMsg, #{}),
- ?event(dev_cache, {write_to_cache, {http_post, WriteResult}}),
- {ok, WriteResponse} = WriteResult,
- ?event(dev_cache, {write_to_cache, {response_received, WriteResponse}}),
- Status = hb_ao:get(<<"status">>, WriteResponse, 0, #{}),
- ?assertEqual(200, Status),
- Path = hb_ao:get(<<"path">>, WriteResponse, not_found, #{}),
- ?assertNotEqual(not_found, Path),
- ?event(dev_cache, {write_to_cache, {write_success, Path}}),
- {WriteResponse, Path}.
-```
-
-### read_from_cache
-
-Read data from the cache via HTTP.
-
-```erlang
-read_from_cache(Node, Path) ->
- ?event(dev_cache, {read_from_cache, {start, Node, Path}}),
- ReadMsg = #{
- <<"path">> => <<"/~cache@1.0/read">>,
- <<"method">> => <<"GET">>,
- <<"target">> => Path
- },
- ?event(dev_cache, {read_from_cache, {request_created, ReadMsg}}),
- ?event({test_read, request, ReadMsg}),
- ReadResult = hb_http:get(Node, ReadMsg, #{}),
- ?event(dev_cache, {read_from_cache, {http_get, ReadResult}}),
- case ReadResult of
- ReadResponse when is_binary(ReadResponse) ->
- ?event(dev_cache,
- {read_from_cache,
- {response_binary, ReadResponse}
- }
- ),
- ReadResponse;
- {ok, ReadResponse} ->
- ?event(dev_cache, {read_from_cache, {response_ok, ReadResponse}}),
- ReadResponse;
- {error, Reason} ->
- ?event(dev_cache, {read_from_cache, {response_error, Reason}}),
- {error, Reason}
- end.
-```
-
-### cache_write_message_test
-
-Test that the cache can be written to and read from using the hb_cache
-
-```erlang
-cache_write_message_test() ->
- ?event(dev_cache, {cache_api_test, {start}}),
- {ok, Opts, _} = setup_test_env(),
- TestData = #{
- <<"test_key">> => <<"test_value">>
- },
- ?event(dev_cache, {cache_api_test, {opts, Opts}}),
- {ok, Path} = hb_cache:write(TestData, Opts),
- ?event(dev_cache, {cache_api_test, {data_written, Path}}),
- {ok, ReadData} = hb_cache:read(Path, Opts),
- ?event(dev_cache, {cache_api_test, {data_read, ReadData}}),
- ?assert(hb_message:match(TestData, ReadData, only_present, Opts)),
- ?event(dev_cache, {cache_api_test}),
- ok.
-```
-
-### cache_write_binary_test
-
-Ensure that we can write direct binaries to the cache.
-
-```erlang
-cache_write_binary_test() ->
- ?event(dev_cache, {cache_api_test, {start}}),
- {ok, Opts, _} = setup_test_env(),
- TestData = <<"test_binary">>,
- {ok, Path} = hb_cache:write(TestData, Opts),
- {ok, ReadData} = hb_cache:read(Path, Opts),
- ?event(dev_cache, {cache_api_test, {data_read, ReadData}}),
- ?assertEqual(TestData, ReadData),
- ?event(dev_cache, {cache_api_test}),
-```
-
----
-
-*Generated from [dev_cache.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_cache.erl)*
diff --git a/docs/book/src/dev_cacheviz.erl.md b/docs/book/src/dev_cacheviz.erl.md
deleted file mode 100644
index 331f2a566..000000000
--- a/docs/book/src/dev_cacheviz.erl.md
+++ /dev/null
@@ -1,115 +0,0 @@
-# dev_cacheviz
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_cacheviz.erl)
-
-A device that generates renders (or renderable dot output) of a node's
-cache.
-
----
-
-## Exported Functions
-
-- `dot/3`
-- `index/3`
-- `js/3`
-- `json/3`
-- `svg/3`
-
----
-
-### dot
-
-A device that generates renders (or renderable dot output) of a node's
-Output the dot representation of the cache, or a specific path within
-Output the SVG representation of the cache, or a specific path within
-
-```erlang
-dot(_, Req, Opts) ->
- Target = hb_ao:get(<<"target">>, Req, all, Opts),
- Dot =
- hb_cache_render:cache_path_to_dot(
- Target,
- #{
- render_data =>
- hb_util:atom(
- hb_ao:get(<<"render-data">>, Req, false, Opts)
- )
- },
- Opts
- ),
- {ok, #{ <<"content-type">> => <<"text/vnd.graphviz">>, <<"body">> => Dot }}.
-```
-
-### svg
-
-A device that generates renders (or renderable dot output) of a node's
-Output the dot representation of the cache, or a specific path within
-Output the SVG representation of the cache, or a specific path within
-Return a JSON representation of the cache graph, suitable for use with
-
-```erlang
-svg(Base, Req, Opts) ->
- {ok, #{ <<"body">> := Dot }} = dot(Base, Req, Opts),
- ?event(cacheviz, {dot, Dot}),
- Svg = hb_cache_render:dot_to_svg(Dot),
- {ok, #{ <<"content-type">> => <<"image/svg+xml">>, <<"body">> => Svg }}.
-```
-
-### json
-
-A device that generates renders (or renderable dot output) of a node's
-Output the dot representation of the cache, or a specific path within
-Output the SVG representation of the cache, or a specific path within
-Return a JSON representation of the cache graph, suitable for use with
-
-```erlang
-json(Base, Req, Opts) ->
- ?event({json, {base, Base}, {req, Req}}),
- Target =
- case hb_ao:get(<<"target">>, Req, Opts) of
- not_found ->
- case map_size(maps:without([<<"device">>], hb_private:reset(Base))) of
- 0 ->
- all;
- _ ->
- ?event({writing_base_for_rendering, Base}),
- {ok, Path} = hb_cache:write(Base, Opts),
- ?event({wrote_message, Path}),
- ID = hb_message:id(Base, all, Opts),
- ?event({generated_id, ID}),
- ID
- end;
- <<".">> -> all;
- ReqTarget -> ReqTarget
- end,
- MaxSize = hb_util:int(hb_ao:get(<<"max-size">>, Req, 250, Opts)),
- ?event({max_size, MaxSize}),
- ?event({generating_json_for, {target, Target}}),
- Res = hb_cache_render:get_graph_data(Target, MaxSize, Opts),
- ?event({graph_data, Res}),
- Res.
-```
-
-### index
-
-Return a renderer in HTML form for the JSON format.
-Return a JS library that can be used to render the JSON format.
-
-```erlang
-index(Base, _, _Opts) ->
- ?event({cacheviz_index, {base, Base}}),
- dev_hyperbuddy:return_file(<<"cacheviz@1.0">>, <<"graph.html">>).
-```
-
-### js
-
-Return a renderer in HTML form for the JSON format.
-Return a JS library that can be used to render the JSON format.
-
-```erlang
-js(_, _, _Opts) ->
-```
-
----
-
-*Generated from [dev_cacheviz.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_cacheviz.erl)*
diff --git a/docs/book/src/dev_codec_ans104.erl.md b/docs/book/src/dev_codec_ans104.erl.md
deleted file mode 100644
index 2eda359ed..000000000
--- a/docs/book/src/dev_codec_ans104.erl.md
+++ /dev/null
@@ -1,511 +0,0 @@
-# dev_codec_ans104
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104.erl)
-
-Codec for managing transformations from `ar_bundles`-style Arweave TX
-records to and from TABMs.
-
----
-
-## Exported Functions
-
-- `commit/3`
-- `content_type/1`
-- `deserialize/3`
-- `from/3`
-- `serialize/3`
-- `to/3`
-- `verify/3`
-
----
-
-### content_type
-
-Codec for managing transformations from `ar_bundles`-style Arweave TX
-Return the content type for the codec.
-Serialize a message or TX to a binary.
-
-```erlang
-content_type(_) -> {ok, <<"application/ans104">>}.
-```
-
-### serialize
-
-Codec for managing transformations from `ar_bundles`-style Arweave TX
-Return the content type for the codec.
-Serialize a message or TX to a binary.
-
-```erlang
-serialize(Msg, Req, Opts) when is_map(Msg) ->
- serialize(to(Msg, Req, Opts), Req, Opts);
-```
-
-### serialize
-
-Codec for managing transformations from `ar_bundles`-style Arweave TX
-Return the content type for the codec.
-Serialize a message or TX to a binary.
-
-```erlang
-serialize(TX, _Req, _Opts) when is_record(TX, tx) ->
- {ok, ar_bundles:serialize(TX)}.
-```
-
-### deserialize
-
-Deserialize a binary ans104 message to a TABM.
-
-```erlang
-deserialize(#{ <<"body">> := Binary }, Req, Opts) ->
- deserialize(Binary, Req, Opts);
-```
-
-### deserialize
-
-Deserialize a binary ans104 message to a TABM.
-
-```erlang
-deserialize(Binary, Req, Opts) when is_binary(Binary) ->
- deserialize(ar_bundles:deserialize(Binary), Req, Opts);
-```
-
-### deserialize
-
-Deserialize a binary ans104 message to a TABM.
-
-```erlang
-deserialize(TX, Req, Opts) when is_record(TX, tx) ->
- from(TX, Req, Opts).
-```
-
-### commit
-
-Sign a message using the `priv_wallet` key in the options. Supports both
-
-```erlang
-commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) ->
- commit(Msg, Req#{ <<"type">> => <<"unsigned-sha256">> }, Opts);
-```
-
-### commit
-
-Sign a message using the `priv_wallet` key in the options. Supports both
-
-```erlang
-commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) ->
- commit(Msg, Req#{ <<"type">> => <<"rsa-pss-sha256">> }, Opts);
-```
-
-### commit
-
-Sign a message using the `priv_wallet` key in the options. Supports both
-
-```erlang
-commit(Msg, Req = #{ <<"type">> := <<"rsa-pss-sha256">> }, Opts) ->
- % Convert the given message to an ANS-104 TX record, sign it, and convert
- % it back to a structured message.
-```
-
-### commit
-
-```erlang
-commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) ->
- % Remove the commitments from the message, convert it to ANS-104, then back.
-```
-
-### verify
-
-Verify an ANS-104 commitment.
-
-```erlang
-verify(Msg, Req, Opts) ->
- ?event({verify, {base, Msg}, {req, Req}}),
- OnlyWithCommitment =
- hb_private:reset(
- hb_message:with_commitments(
- Req,
- Msg,
- Opts
- )
- ),
- ?event({verify, {only_with_commitment, OnlyWithCommitment}}),
- {ok, TX} = to(OnlyWithCommitment, Req, Opts),
- ?event({verify, {encoded, TX}}),
- Res = ar_bundles:verify_item(TX),
- {ok, Res}.
-```
-
-### from
-
-Convert a #tx record into a message map recursively.
-
-```erlang
-from(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, Binary};
-```
-
-### from
-
-Convert a #tx record into a message map recursively.
-
-```erlang
-from(TX, Req, Opts) when is_record(TX, tx) ->
- case lists:keyfind(<<"ao-type">>, 1, TX#tx.tags) of
- false ->
- do_from(TX, Req, Opts);
- {<<"ao-type">>, <<"binary">>} ->
- {ok, TX#tx.data}
- end.
-```
-
-### do_from
-
-```erlang
-do_from(RawTX, Req, Opts) ->
- % Ensure the TX is fully deserialized.
-```
-
-### to
-
-Internal helper to translate a message to its #tx record representation,
-
-```erlang
-to(Binary, _Req, _Opts) when is_binary(Binary) ->
- % ar_bundles cannot serialize just a simple binary or get an ID for it, so
- % we turn it into a TX record with a special tag, tx_to_message will
- % identify this tag and extract just the binary.
-```
-
-### to
-
-```erlang
-to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX};
-```
-
-### to
-
-```erlang
-to(RawTABM, Req, Opts) when is_map(RawTABM) ->
- % Ensure that the TABM is fully loaded if the `bundle` key is set to true.
-```
-
-### to
-
-```erlang
-to(Other, _Req, _Opts) ->
- throw({invalid_tx, Other}).
-```
-
-### normal_tags_test
-
-```erlang
-normal_tags_test() ->
- Msg = #{
- <<"first-tag">> => <<"first-value">>,
- <<"second-tag">> => <<"second-value">>
- },
- {ok, Encoded} = to(Msg, #{}, #{}),
- ?event({encoded, Encoded}),
- {ok, Decoded} = from(Encoded, #{}, #{}),
- ?event({decoded, Decoded}),
- ?assert(hb_message:match(Msg, Decoded)).
-```
-
-### from_maintains_tag_name_case_test
-
-```erlang
-from_maintains_tag_name_case_test() ->
- TX = #tx {
- tags = [
- {<<"Test-Tag">>, <<"test-value">>}
- ]
- },
- SignedTX = ar_bundles:sign_item(TX, hb:wallet()),
- ?event({signed_tx, SignedTX}),
- ?assert(ar_bundles:verify_item(SignedTX)),
- TABM = hb_util:ok(from(SignedTX, #{}, #{})),
- ?event({tabm, TABM}),
- ConvertedTX = hb_util:ok(to(TABM, #{}, #{})),
- ?event({converted_tx, ConvertedTX}),
- ?assert(ar_bundles:verify_item(ConvertedTX)),
- ?assertEqual(ConvertedTX, ar_bundles:normalize(SignedTX)).
-```
-
-### restore_tag_name_case_from_cache_test
-
-```erlang
-restore_tag_name_case_from_cache_test() ->
- Opts = #{ store => hb_test_utils:test_store() },
- TX = #tx {
- tags = [
- {<<"Test-Tag">>, <<"test-value">>},
- {<<"test-tag-2">>, <<"test-value-2">>}
- ]
- },
- SignedTX = ar_bundles:sign_item(TX, ar_wallet:new()),
- SignedMsg =
- hb_message:convert(
- SignedTX,
- <<"structured@1.0">>,
- <<"ans104@1.0">>,
- Opts
- ),
- SignedID = hb_message:id(SignedMsg, all),
- ?event({signed_msg, SignedMsg}),
- OnlyCommitted = hb_message:with_only_committed(SignedMsg, Opts),
- ?event({only_committed, OnlyCommitted}),
- {ok, ID} = hb_cache:write(SignedMsg, Opts),
- ?event({id, ID}),
- {ok, ReadMsg} = hb_cache:read(SignedID, Opts),
- ?event({restored_msg, ReadMsg}),
- {ok, ReadTX} = to(ReadMsg, #{}, Opts),
- ?event({restored_tx, ReadTX}),
- ?assert(hb_message:match(ReadMsg, SignedMsg)),
- ?assert(ar_bundles:verify_item(ReadTX)).
-```
-
-### unsigned_duplicated_tag_name_test
-
-```erlang
-unsigned_duplicated_tag_name_test() ->
- TX = ar_bundles:reset_ids(ar_bundles:normalize(#tx {
- tags = [
- {<<"Test-Tag">>, <<"test-value">>},
- {<<"test-tag">>, <<"test-value-2">>}
- ]
- })),
- Msg = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
- ?event({msg, Msg}),
- TX2 = hb_message:convert(Msg, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
- ?event({tx2, TX2}),
- ?assertEqual(TX, TX2).
-```
-
-### signed_duplicated_tag_name_test
-
-```erlang
-signed_duplicated_tag_name_test() ->
- TX = ar_bundles:sign_item(#tx {
- tags = [
- {<<"Test-Tag">>, <<"test-value">>},
- {<<"test-tag">>, <<"test-value-2">>}
- ]
- }, ar_wallet:new()),
- Msg = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
- ?event({msg, Msg}),
- TX2 = hb_message:convert(Msg, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
- ?event({tx2, TX2}),
- ?assertEqual(TX, TX2),
- ?assert(ar_bundles:verify_item(TX2)).
-```
-
-### simple_to_conversion_test
-
-```erlang
-simple_to_conversion_test() ->
- Msg = #{
- <<"first-tag">> => <<"first-value">>,
- <<"second-tag">> => <<"second-value">>
- },
- {ok, Encoded} = to(Msg, #{}, #{}),
- ?event({encoded, Encoded}),
- {ok, Decoded} = from(Encoded, #{}, #{}),
- ?event({decoded, Decoded}),
- ?assert(hb_message:match(Msg, hb_message:uncommitted(Decoded, #{}))).
-```
-
-### external_item_with_target_field_test
-
-Ensure that items with an explicitly defined target field lead to:
-
-```erlang
-external_item_with_target_field_test() ->
- TX =
- ar_bundles:sign_item(
- #tx {
- target = crypto:strong_rand_bytes(32),
- tags = [
- {<<"test-tag">>, <<"test-value">>},
- {<<"test-tag-2">>, <<"test-value-2">>}
- ],
- data = <<"test-data">>
- },
- ar_wallet:new()
- ),
- EncodedTarget = hb_util:encode(TX#tx.target),
- ?event({tx, TX}),
- Decoded = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
- ?event({decoded, Decoded}),
- ?assertEqual(EncodedTarget, hb_maps:get(<<"target">>, Decoded, undefined, #{})),
- {ok, OnlyCommitted} = hb_message:with_only_committed(Decoded, #{}),
- ?event({only_committed, OnlyCommitted}),
- ?assertEqual(EncodedTarget, hb_maps:get(<<"target">>, OnlyCommitted, undefined, #{})),
- Encoded = hb_message:convert(OnlyCommitted, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
- ?assertEqual(TX#tx.target, Encoded#tx.target),
- ?event({result, {initial, TX}, {result, Encoded}}),
- ?assertEqual(TX, Encoded).
-```
-
-### generate_item_with_target_tag_test
-
-Ensure that items made inside HyperBEAM use the tags to encode `target`
-
-```erlang
-generate_item_with_target_tag_test() ->
- Msg =
- #{
- <<"target">> => Target = <<"NON-ID-TARGET">>,
- <<"other-key">> => <<"other-value">>
- },
- {ok, TX} = to(Msg, #{}, #{}),
- ?event({encoded_tx, TX}),
- % The encoded TX should have ignored the `target' field, setting a tag instead.
-```
-
-### generate_item_with_target_field_test
-
-```erlang
-generate_item_with_target_field_test() ->
- Msg =
- hb_message:commit(
- #{
- <<"target">> => Target = hb_util:encode(crypto:strong_rand_bytes(32)),
- <<"other-key">> => <<"other-value">>
- },
- #{ priv_wallet => hb:wallet() },
- <<"ans104@1.0">>
- ),
- {ok, TX} = to(Msg, #{}, #{}),
- ?event({encoded_tx, TX}),
- ?assertEqual(Target, hb_util:encode(TX#tx.target)),
- Decoded = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
- ?event({decoded, Decoded}),
- ?assertEqual(Target, hb_maps:get(<<"target">>, Decoded, undefined, #{})),
- {ok, OnlyCommitted} = hb_message:with_only_committed(Decoded, #{}),
- ?event({only_committed, OnlyCommitted}),
- ?assertEqual(Target, hb_maps:get(<<"target">>, OnlyCommitted, undefined, #{})),
- Encoded = hb_message:convert(OnlyCommitted, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
- ?event({result, {initial, TX}, {result, Encoded}}),
- ?assertEqual(TX, Encoded).
-```
-
-### type_tag_test
-
-```erlang
-type_tag_test() ->
- TX =
- ar_bundles:sign_item(
- #tx {
- tags = [{<<"type">>, <<"test-value">>}]
- },
- ar_wallet:new()
- ),
- ?event({tx, TX}),
- Structured = hb_message:convert(TX, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
- ?event({structured, Structured}),
- TX2 = hb_message:convert(Structured, <<"ans104@1.0">>, <<"structured@1.0">>, #{}),
- ?event({after_conversion, TX2}),
- ?assertEqual(TX, TX2).
-```
-
-### ao_data_key_test
-
-```erlang
-ao_data_key_test() ->
- Msg =
- hb_message:commit(
- #{
- <<"other-key">> => <<"Normal value">>,
- <<"body">> => <<"Body value">>
- },
- #{ priv_wallet => hb:wallet() },
- <<"ans104@1.0">>
- ),
- ?event({msg, Msg}),
- Enc = hb_message:convert(Msg, <<"ans104@1.0">>, #{}),
- ?event({enc, Enc}),
- ?assertEqual(<<"Body value">>, Enc#tx.data),
- Dec = hb_message:convert(Enc, <<"structured@1.0">>, <<"ans104@1.0">>, #{}),
- ?event({dec, Dec}),
- ?assert(hb_message:verify(Dec, all, #{})).
-```
-
-### simple_signed_to_httpsig_test
-
-```erlang
-simple_signed_to_httpsig_test() ->
- Structured =
- hb_message:commit(
- #{ <<"test-tag">> => <<"test-value">> },
- #{ priv_wallet => ar_wallet:new() },
- #{
- <<"commitment-device">> => <<"ans104@1.0">>
- }
- ),
- ?event(debug_test, {msg, Structured}),
- HTTPSig =
- hb_message:convert(
- Structured,
- <<"httpsig@1.0">>,
- <<"structured@1.0">>,
- #{}
- ),
- ?event(debug_test, {httpsig, HTTPSig}),
- Structured2 =
- hb_message:convert(
- HTTPSig,
- <<"structured@1.0">>,
- <<"httpsig@1.0">>,
- #{}
- ),
- ?event(debug_test, {decoded, Structured2}),
- Match = hb_message:match(Structured, Structured2, #{}),
- ?assert(Match),
- ?assert(hb_message:verify(Structured2, all, #{})),
- HTTPSig2 = hb_message:convert(Structured2, <<"httpsig@1.0">>, <<"structured@1.0">>, #{}),
- ?event(debug_test, {httpsig2, HTTPSig2}),
- ?assert(hb_message:verify(HTTPSig2, all, #{})),
- ?assert(hb_message:match(HTTPSig, HTTPSig2)).
-```
-
-### unsorted_tag_map_test
-
-```erlang
-unsorted_tag_map_test() ->
- TX =
- ar_bundles:sign_item(
- #tx{
- format = ans104,
- tags = [
- {<<"z">>, <<"position-1">>},
- {<<"a">>, <<"position-2">>}
- ],
- data = <<"data">>
- },
- ar_wallet:new()
- ),
- ?assert(ar_bundles:verify_item(TX)),
- ?event(debug_test, {tx, TX}),
- {ok, TABM} = dev_codec_ans104:from(TX, #{}, #{}),
- ?event(debug_test, {tabm, TABM}),
- {ok, Decoded} = dev_codec_ans104:to(TABM, #{}, #{}),
- ?event(debug_test, {decoded, Decoded}),
- ?assert(ar_bundles:verify_item(Decoded)).
-```
-
-### field_and_tag_ordering_test
-
-```erlang
-field_and_tag_ordering_test() ->
- UnsignedTABM = #{
- <<"a">> => <<"value1">>,
- <<"z">> => <<"value2">>,
- <<"target">> => <<"NON-ID-TARGET">>
- },
- Wallet = hb:wallet(),
- SignedTABM = hb_message:commit(
- UnsignedTABM, #{priv_wallet => Wallet}, <<"ans104@1.0">>),
-```
-
----
-
-*Generated from [dev_codec_ans104.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104.erl)*
diff --git a/docs/book/src/dev_codec_ans104_from.erl.md b/docs/book/src/dev_codec_ans104_from.erl.md
deleted file mode 100644
index d9b42e17e..000000000
--- a/docs/book/src/dev_codec_ans104_from.erl.md
+++ /dev/null
@@ -1,355 +0,0 @@
-# dev_codec_ans104_from
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104_from.erl)
-
-Library functions for decoding ANS-104-style data items to TABM form.
-
----
-
-## Exported Functions
-
-- `base/5`
-- `committed/5`
-- `data/4`
-- `fields/2`
-- `tags/2`
-- `with_commitments/5`
-
----
-
-### fields
-
-Library functions for decoding ANS-104-style data items to TABM form.
-Return a TABM message containing the fields of the given decoded
-
-```erlang
-fields(Item, _Opts) ->
- case Item#tx.target of
- ?DEFAULT_TARGET -> #{};
- Target ->
- #{
- <<"target">> => hb_util:encode(Target)
- }
- end.
-```
-
-### tags
-
-Return a TABM of the raw tags of the item, including all metadata
-
-```erlang
-tags(Item, Opts) ->
- Tags = hb_ao:normalize_keys(
- deduplicating_from_list(Item#tx.tags, Opts),
- Opts
- ),
- ao_types(Tags, Opts).
-```
-
-### ao_types
-
-Ensure the encoded keys in the `ao-types` field are lowercased and
-
-```erlang
-ao_types(#{ <<"ao-types">> := AoTypes } = Tags, Opts) ->
- AOTypes = dev_codec_structured:decode_ao_types(AoTypes, Opts),
- % Normalize all keys in the ao-types map and re-encode
- NormAOTypes =
- maps:fold(
- fun(Key, Val, Acc) ->
- NormKey = hb_util:to_lower(hb_ao:normalize_key(Key)),
- Acc#{ NormKey => Val }
- end,
- #{},
- AOTypes
- ),
- EncodedAOTypes = dev_codec_structured:encode_ao_types(NormAOTypes, Opts),
- Tags#{ <<"ao-types">> := EncodedAOTypes };
-```
-
-### ao_types
-
-Ensure the encoded keys in the `ao-types` field are lowercased and
-
-```erlang
-ao_types(Tags, _Opts) ->
- Tags.
-```
-
-### data
-
-Return a TABM of the keys and values found in the data field of the item.
-
-```erlang
-data(Item, Req, Tags, Opts) ->
- % If the data field is empty, we return an empty map. If it is a map, we
- % return it as such. Otherwise, we return a map with the data key set to
- % the raw data value. This handles unbundling nested messages, as well as
- % applying the `ao-data-key' tag if given.
-```
-
-### committed
-
-Calculate the list of committed keys for an item, based on its
-
-```erlang
-committed(Item, Fields, Tags, Data, Opts) ->
- hb_util:unique(
- data_keys(Data, Opts) ++
- tag_keys(Item, Opts) ++
- field_keys(Fields, Tags, Data, Opts)
- ).
-```
-
-### field_keys
-
-Return the list of the keys from the fields TABM.
-
-```erlang
-field_keys(BaseFields, Tags, Data, Opts) ->
- HasTarget =
- hb_maps:is_key(<<"target">>, BaseFields, Opts) orelse
- hb_maps:is_key(<<"target">>, Tags, Opts) orelse
- hb_maps:is_key(<<"target">>, Data, Opts),
- case HasTarget of
- true -> [<<"target">>];
- false -> []
- end.
-```
-
-### data_keys
-
-Return the list of the keys from the data TABM.
-
-```erlang
-data_keys(Data, Opts) ->
- hb_util:to_sorted_keys(Data, Opts).
-```
-
-### tag_keys
-
-Return the list of the keys from the tags TABM. Filter all metadata
-
-```erlang
-tag_keys(Item, _Opts) ->
- MetaTags = [
- <<"bundle-format">>,
- <<"bundle-version">>,
- <<"bundle-map">>,
- <<"ao-data-key">>
- ],
- lists:filtermap(
- fun({Tag, _}) ->
- case lists:member(Tag, MetaTags) of
- true -> false;
- false -> {true, hb_util:to_lower(hb_ao:normalize_key(Tag))}
- end
- end,
- Item#tx.tags
- ).
-```
-
-### base
-
-Return the complete message for an item, less its commitments. The
-
-```erlang
-base(CommittedKeys, Fields, Tags, Data, Opts) ->
- hb_maps:from_list(
- lists:map(
- fun(Key) ->
- case hb_maps:find(Key, Data, Opts) of
- error ->
- case hb_maps:find(Key, Fields, Opts) of
- error ->
- case hb_maps:find(Key, Tags, Opts) of
- error -> throw({missing_key, Key});
- {ok, Value} -> {Key, Value}
- end;
- {ok, Value} -> {Key, Value}
- end;
- {ok, Value} -> {Key, Value}
- end
- end,
- CommittedKeys
- )
- ).
-```
-
-### with_commitments
-
-Return a message with the appropriate commitments added to it.
-
-```erlang
-with_commitments(Item, Tags, Base, CommittedKeys, Opts) ->
- case Item#tx.signature of
- ?DEFAULT_SIG ->
- case normal_tags(Item#tx.tags) of
- true -> Base;
- false ->
- with_unsigned_commitment(Item, Tags, Base, CommittedKeys, Opts)
- end;
- _ -> with_signed_commitment(Item, Tags, Base, CommittedKeys, Opts)
- end.
-```
-
-### with_unsigned_commitment
-
-Returns a commitments message for an item, containing an unsigned
-
-```erlang
-with_unsigned_commitment(Item, Tags, UncommittedMessage, CommittedKeys, Opts) ->
- ID = hb_util:human_id(Item#tx.unsigned_id),
- UncommittedMessage#{
- <<"commitments">> => #{
- ID =>
- filter_unset(
- #{
- <<"commitment-device">> => <<"ans104@1.0">>,
- <<"committed">> => CommittedKeys,
- <<"type">> => <<"unsigned-sha256">>,
- <<"bundle">> => bundle_commitment_key(Tags, Opts),
- <<"original-tags">> => original_tags(Item, Opts),
- <<"field-target">> =>
- case Item#tx.target of
- ?DEFAULT_TARGET -> unset;
- Target -> hb_util:encode(Target)
- end,
- <<"field-anchor">> =>
- case Item#tx.anchor of
- ?DEFAULT_LAST_TX -> unset;
- LastTX -> LastTX
- end
- },
- Opts
- )
- }
- }.
-```
-
-### with_signed_commitment
-
-Returns a commitments message for an item, containing a signed
-
-```erlang
-with_signed_commitment(Item, Tags, UncommittedMessage, CommittedKeys, Opts) ->
- Address = hb_util:human_id(ar_wallet:to_address(Item#tx.owner)),
- ID = hb_util:human_id(Item#tx.id),
- Commitment =
- filter_unset(
- #{
- <<"commitment-device">> => <<"ans104@1.0">>,
- <<"committer">> => Address,
- <<"committed">> => CommittedKeys,
- <<"signature">> => hb_util:encode(Item#tx.signature),
- <<"keyid">> =>
- <<"publickey:", (hb_util:encode(Item#tx.owner))/binary>>,
- <<"type">> => <<"rsa-pss-sha256">>,
- <<"bundle">> => bundle_commitment_key(Tags, Opts),
- <<"original-tags">> => original_tags(Item, Opts),
- <<"field-anchor">> =>
- case Item#tx.anchor of
- ?DEFAULT_LAST_TX -> unset;
- LastTX -> LastTX
- end,
- <<"field-target">> =>
- case Item#tx.target of
- ?DEFAULT_TARGET -> unset;
- Target -> hb_util:encode(Target)
- end
- },
- Opts
- ),
- UncommittedMessage#{
- <<"commitments">> => #{
- ID => Commitment
- }
- }.
-```
-
-### bundle_commitment_key
-
-Return the bundle key for an item.
-Check whether a list of key-value pairs contains only normalized keys.
-
-```erlang
-bundle_commitment_key(Tags, Opts) ->
- hb_util:bin(hb_maps:is_key(<<"bundle-format">>, Tags, Opts)).
-```
-
-### normal_tags
-
-Return the bundle key for an item.
-Check whether a list of key-value pairs contains only normalized keys.
-
-```erlang
-normal_tags(Tags) ->
- lists:all(
- fun({Key, _}) ->
- hb_util:to_lower(hb_ao:normalize_key(Key)) =:= Key
- end,
- Tags
- ).
-```
-
-### original_tags
-
-Return the original tags of an item if it is applicable. Otherwise,
-
-```erlang
-original_tags(Item, _Opts) ->
- case normal_tags(Item#tx.tags) of
- true -> unset;
- false -> encoded_tags_to_map(Item#tx.tags)
- end.
-```
-
-### encoded_tags_to_map
-
-Convert an ANS-104 encoded tag list into a HyperBEAM-compatible map.
-
-```erlang
-encoded_tags_to_map(Tags) ->
- hb_util:list_to_numbered_message(
- lists:map(
- fun({Key, Value}) ->
- #{
- <<"name">> => Key,
- <<"value">> => Value
- }
- end,
- Tags
- )
- ).
-```
-
-### filter_unset
-
-Remove all undefined values from a map.
-
-```erlang
-filter_unset(Map, Opts) ->
- hb_maps:filter(
- fun(_, Value) ->
- case Value of
- unset -> false;
- _ -> true
- end
- end,
- Map,
- Opts
- ).
-```
-
-### deduplicating_from_list
-
-Deduplicate a list of key-value pairs by key, generating a list of
-
-```erlang
-deduplicating_from_list(Tags, Opts) ->
- % Aggregate any duplicated tags into an ordered list of values.
-```
-
----
-
-*Generated from [dev_codec_ans104_from.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104_from.erl)*
diff --git a/docs/book/src/dev_codec_ans104_to.erl.md b/docs/book/src/dev_codec_ans104_to.erl.md
deleted file mode 100644
index 6c045f251..000000000
--- a/docs/book/src/dev_codec_ans104_to.erl.md
+++ /dev/null
@@ -1,240 +0,0 @@
-# dev_codec_ans104_to
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104_to.erl)
-
-Library functions for encoding messages to the ANS-104 format.
-
----
-
-## Exported Functions
-
-- `data/3`
-- `maybe_load/3`
-- `siginfo/2`
-- `tags/4`
-
----
-
-### maybe_load
-
-Library functions for encoding messages to the ANS-104 format.
-Determine if the message should be loaded from the cache and re-converted
-
-```erlang
-maybe_load(RawTABM, Req, Opts) ->
- case hb_util:atom(hb_ao:get(<<"bundle">>, Req, false, Opts)) of
- false -> RawTABM;
- true ->
- % Convert back to the fully loaded structured@1.0 message, then
- % convert to TABM with bundling enabled.
-```
-
-### siginfo
-
-Calculate the fields for a message, returning an initial TX record.
-
-```erlang
-siginfo(Message, Opts) ->
- MaybeCommitment =
- hb_message:commitment(
- #{ <<"commitment-device">> => <<"ans104@1.0">> },
- Message,
- Opts
- ),
- case MaybeCommitment of
- {ok, _, Commitment} -> commitment_to_tx(Commitment, Opts);
- not_found ->
- case hb_maps:find(<<"target">>, Message, Opts) of
- {ok, EncodedTarget} ->
- case hb_util:safe_decode(EncodedTarget) of
- {ok, Target} when ?IS_ID(Target) ->
- #tx{ target = Target };
- _ -> #tx{}
- end;
- error -> #tx{}
- end;
- multiple_matches ->
- throw({multiple_ans104_commitments_unsupported, Message})
- end.
-```
-
-### commitment_to_tx
-
-Convert a commitment to a base TX record. Extracts the owner, signature,
-
-```erlang
-commitment_to_tx(Commitment, Opts) ->
- Signature =
- hb_util:decode(
- maps:get(<<"signature">>, Commitment, hb_util:encode(?DEFAULT_SIG))
- ),
- Owner =
- case hb_maps:find(<<"keyid">>, Commitment, Opts) of
- {ok, KeyID} ->
- hb_util:decode(
- dev_codec_httpsig_keyid:remove_scheme_prefix(KeyID)
- );
- error -> ?DEFAULT_OWNER
- end,
- Tags =
- case hb_maps:find(<<"original-tags">>, Commitment, Opts) of
- {ok, OriginalTags} -> original_tags_to_tags(OriginalTags);
- error -> []
- end,
- LastTX =
- case hb_maps:find(<<"field-anchor">>, Commitment, Opts) of
- {ok, EncodedLastTX} -> hb_util:decode(EncodedLastTX);
- error -> ?DEFAULT_LAST_TX
- end,
- Target =
- case hb_maps:find(<<"field-target">>, Commitment, Opts) of
- {ok, EncodedTarget} -> hb_util:decode(EncodedTarget);
- error -> ?DEFAULT_TARGET
- end,
- ?event({commitment_owner, Owner}),
- ?event({commitment_signature, Signature}),
- ?event({commitment_tags, Tags}),
- ?event({commitment_last_tx, LastTX}),
- #tx{
- owner = Owner,
- signature = Signature,
- tags = Tags,
- anchor = LastTX,
- target = Target
- }.
-```
-
-### data
-
-Calculate the data field for a message.
-
-```erlang
-data(TABM, Req, Opts) ->
- DataKey = inline_key(TABM),
- % Translate the keys into a binary map. If a key has a value that is a map,
- % we recursively turn its children into messages.
-```
-
-### data_messages
-
-Calculate the data value for a message. The rules are:
-
-```erlang
-data_messages(TABM, Opts) when is_map(TABM) ->
- UncommittedTABM =
- hb_maps:without(
- [<<"commitments">>, <<"data">>, <<"target">>],
- hb_private:reset(TABM),
- Opts
- ),
- % If there are too many keys in the TABM, throw an error.
-```
-
-### tags
-
-Calculate the tags field for a data item. If the TX already has tags
-
-```erlang
-tags(#tx{ tags = ExistingTags }, _, _, _) when ExistingTags =/= [] ->
- ExistingTags;
-```
-
-### tags
-
-Calculate the tags field for a data item. If the TX already has tags
-
-```erlang
-tags(TX, TABM, Data, Opts) ->
- DataKey = inline_key(TABM),
- MaybeCommitment =
- hb_message:commitment(
- #{ <<"commitment-device">> => <<"ans104@1.0">> },
- TABM,
- Opts
- ),
- CommittedTagKeys =
- case MaybeCommitment of
- {ok, _, Commitment} ->
- % There is already a commitment, so the tags and order are
- % pre-determined. However, if the message has been bundled,
- % any `+link`-suffixed keys in the committed list may need to
- % be resolved to their base keys (e.g., `output+link` -> `output`).
-```
-
-### include_target_tag
-
-Return whether to include the `target` tag in the tags list.
-
-```erlang
-include_target_tag(TX, TABM, Opts) ->
- case {TX#tx.target, hb_maps:get(<<"target">>, TABM, undefined, Opts)} of
- {?DEFAULT_TARGET, _} -> true;
- {FieldTarget, TagTarget} when FieldTarget =/= TagTarget -> false;
- _ -> true
- end.
-```
-
-### committed_tag_keys_to_tags
-
-Apply the `ao-data-key` to the committed keys to generate the list of
-
-```erlang
-committed_tag_keys_to_tags(TX, TABM, DataKey, Committed, Opts) ->
- DataKeysToExclude =
- case TX#tx.data of
- Data when is_map(Data)-> maps:keys(Data);
- _ -> []
- end,
- case DataKey of
- <<"data">> -> [];
- _ -> [{<<"ao-data-key">>, DataKey}]
- end ++
- lists:map(
- fun(Key) ->
- case hb_maps:find(Key, TABM, Opts) of
- error -> throw({missing_committed_key, Key});
- {ok, Value} -> {Key, Value}
- end
- end,
- hb_util:list_without(
- [DataKey | DataKeysToExclude],
- Committed
- )
- ).
-```
-
-### inline_key
-
-Determine if an `ao-data-key` should be added to the message.
-
-```erlang
-inline_key(Msg) ->
- InlineKey = maps:get(<<"ao-data-key">>, Msg, undefined),
- case {
- InlineKey,
- maps:get(<<"data">>, Msg, ?DEFAULT_DATA) == ?DEFAULT_DATA,
- maps:is_key(<<"body">>, Msg)
- andalso not ?IS_LINK(maps:get(<<"body">>, Msg, undefined))
- } of
- {Explicit, _, _} when Explicit =/= undefined ->
- % ao-data-key already exists, so we honor it.
-```
-
-### original_tags_to_tags
-
-Convert a HyperBEAM-compatible map into an ANS-104 encoded tag list,
-
-```erlang
-original_tags_to_tags(TagMap) ->
- OrderedList = hb_util:message_to_ordered_list(hb_private:reset(TagMap)),
- ?event({ordered_tagmap, {explicit, OrderedList}, {input, {explicit, TagMap}}}),
- lists:map(
- fun(#{ <<"name">> := Key, <<"value">> := Value }) ->
- {Key, Value}
- end,
- OrderedList
-```
-
----
-
-*Generated from [dev_codec_ans104_to.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_ans104_to.erl)*
diff --git a/docs/book/src/dev_codec_cookie.erl.md b/docs/book/src/dev_codec_cookie.erl.md
deleted file mode 100644
index b377ac2c8..000000000
--- a/docs/book/src/dev_codec_cookie.erl.md
+++ /dev/null
@@ -1,570 +0,0 @@
-# dev_codec_cookie
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie.erl)
-
-A utility device that manages setting and encoding/decoding the cookies
-found in requests from a caller. This device implements the `~cookie@1.0`
-codec, inline with the `~message@1.0` schema for conversion.
-Additionally, a `commit` to a message using a secret generated and stored
-in the cookies of the caller, and a `verify` key that validates said
-commitments. In addition, a `generate` key is provided to perform only the
-generation side of the commitment process. The `finalize` key may be
-employed to add a `set` operation to the end of a message sequence, which
-is used in hooks that need to ensure a caller always receives cookies
-generated outside of the normal AO-Core execution flow. In totality, these
-keys implement the `generator` interface type, and may be employed in
-various contexts. For example, `~auth-hook@1.0` may be configured to use
-this device to generate and store secrets in the cookies of the caller,
-which are then used with the `~proxy-wallet@1.0` device to sign requests.
-The `commit` and `verify` keys utilize the `~httpsig@1.0`'s HMAC `secret`
-commitment scheme, which uses a secret key to commit to a message, with the
-`committer` being listed as a hash of the secret.
-This device supports the following paths:
-`/commit`: Sets a `secret` key in the cookies of the caller. The name of
-the cookie is calculated as the hash of the secret.
-`/verify`: Verifies the caller's request by checking the committer in the
-request matches the secret in the cookies of the base message.
-`/store`: Sets the keys in the request message in the cookies of the caller.
-`/extract`: Extracts the cookies from a base message.
-`/reset`: Removes all cookie keys from the base message.
-`/to`: Converts a message containing cookie sources (`cookie`, `set-cookie`,
-or `priv/cookie`) into the format specified in the request message (e.g.
-`set-cookie`, `cookie`).
-`/from`: Converts a message containing encoded cookies into a message
-containing the cookies parsed and normalized.
-
----
-
-## Exported Functions
-
-- `commit/3`
-- `extract/3`
-- `finalize/3`
-- `from/3`
-- `generate/3`
-- `get_cookie/3`
-- `opts/1`
-- `reset/2`
-- `store/3`
-- `to/3`
-- `verify/3`
-
----
-
-### opts
-
-A utility device that manages setting and encoding/decoding the cookies
-Get the private store options to use for functions in the cookie device.
-
-```erlang
-opts(Opts) -> hb_private:opts(Opts).
-%%% ~message@1.0 Commitments API keys.
-```
-
-### commit
-
-```erlang
-commit(Base, Req, RawOpts) -> dev_codec_cookie_auth:commit(Base, Req, RawOpts).
-```
-
-### verify
-
-Preprocessor keys that utilize cookies and the `~secret@1.0` device to
-
-```erlang
-verify(Base, Req, RawOpts) -> dev_codec_cookie_auth:verify(Base, Req, RawOpts).
-```
-
-### generate
-
-Preprocessor keys that utilize cookies and the `~secret@1.0` device to
-
-```erlang
-generate(Base, Req, Opts) ->
- dev_codec_cookie_auth:generate(Base, Req, Opts).
-```
-
-### finalize
-
-Finalize an `on-request` hook by adding the `set-cookie` header to the
-
-```erlang
-finalize(Base, Request, Opts) ->
- dev_codec_cookie_auth:finalize(Base, Request, Opts).
-```
-
-### get_cookie
-
-Get the cookie with the given key from the base message. The format of
-
-```erlang
-get_cookie(Base, Req, RawOpts) ->
- Opts = opts(RawOpts),
- {ok, Cookies} = extract(Base, Req, Opts),
- Key = hb_maps:get(<<"key">>, Req, undefined, Opts),
- case hb_maps:get(Key, Cookies, undefined, Opts) of
- undefined -> {error, not_found};
- Cookie ->
- Format = hb_maps:get(<<"format">>, Req, <<"default">>, Opts),
- case Format of
- <<"default">> -> {ok, Cookie};
- <<"set-cookie">> -> {ok, normalize_cookie_value(Cookie)};
- <<"cookie">> -> {ok, value(Cookie)}
- end
- end.
-```
-
-### extract
-
-Return the parsed and normalized cookies from a message.
-
-```erlang
-extract(Msg, Req, Opts) ->
- {ok, MsgWithCookie} = from(Msg, Req, Opts),
- Cookies = hb_private:get(<<"cookie">>, MsgWithCookie, #{}, Opts),
- {ok, Cookies}.
-```
-
-### store
-
-Set the keys in the request message in the cookies of the caller. Removes
-
-```erlang
-store(Base, Req, RawOpts) ->
- Opts = opts(RawOpts),
- ?event({store, {base, Base}, {req, Req}}),
- {ok, ExistingCookies} = extract(Base, Req, Opts),
- ?event({store, {existing_cookies, ExistingCookies}}),
- {ok, ResetBase} = reset(Base, Opts),
- ?event({store, {reset_base, ResetBase}}),
- MsgToSet =
- hb_maps:without(
- [
- <<"path">>,
- <<"accept-bundle">>,
- <<"ao-peer">>,
- <<"host">>,
- <<"method">>,
- <<"body">>
- ],
- hb_private:reset(Req),
- Opts
- ),
- ?event({store, {msg_to_set, MsgToSet}}),
- NewCookies = hb_maps:merge(ExistingCookies, MsgToSet, Opts),
- NewBase = hb_private:set(ResetBase, <<"cookie">>, NewCookies, Opts),
- {ok, NewBase}.
-```
-
-### reset
-
-Remove all cookie keys from the given message (including `cookie` and
-
-```erlang
-reset(Base, RawOpts) ->
- Opts = opts(RawOpts),
- WithoutBaseCookieKeys =
- hb_maps:without(
- [<<"cookie">>, <<"set-cookie">>],
- Base,
- Opts
- ),
- WithoutPrivCookie =
- hb_private:set(
- WithoutBaseCookieKeys,
- <<"cookie">>,
- unset,
- Opts
- ),
- {ok, WithoutPrivCookie}.
-```
-
-### to
-
-Convert a message containing cookie sources (`cookie`, `set-cookie`,
-
-```erlang
-to(Msg, Req, Opts) ->
- ?event({to, {msg, Msg}, {req, Req}}),
- CookieOpts = opts(Opts),
- LoadedMsg = hb_cache:ensure_all_loaded(Msg, CookieOpts),
- ?event({to, {loaded_msg, LoadedMsg}}),
- do_to(LoadedMsg, Req, CookieOpts).
-```
-
-### do_to
-
-```erlang
-do_to(Msg, Req = #{ <<"format">> := <<"set-cookie">> }, Opts) when is_map(Msg) ->
- ?event({to_set_cookie, {msg, Msg}, {req, Req}}),
- {ok, ExtractedParsedCookies} = extract(Msg, Req, Opts),
- {ok, ResetBase} = reset(Msg, Opts),
- SetCookieLines =
- maps:values(
- maps:map(
- fun to_set_cookie_line/2,
- ExtractedParsedCookies
- )
- ),
- MsgWithSetCookie =
- ResetBase#{
- <<"set-cookie">> => SetCookieLines
- },
- {ok, MsgWithSetCookie};
-```
-
-### do_to
-
-```erlang
-do_to(Msg, Req = #{ <<"format">> := <<"cookie">> }, Opts) when is_map(Msg) ->
- ?event({to_cookie, {msg, Msg}, {req, Req}}),
- {ok, ExtractedParsedCookies} = extract(Msg, Req, Opts),
- {ok, ResetBase} = reset(Msg, Opts),
- CookieLines =
- hb_maps:values(
- hb_maps:map(
- fun to_cookie_line/2,
- ExtractedParsedCookies,
- Opts
- ),
- Opts
- ),
- ?event({to_cookie, {cookie_lines, CookieLines}}),
- CookieLine = join(CookieLines, <<"; ">>),
- {ok, ResetBase#{ <<"cookie">> => CookieLine }};
-```
-
-### do_to
-
-```erlang
-do_to(Msg, _Req, _Opts) when is_map(Msg) ->
- error({cookie_to_error, {no_format_specified, Msg}});
-```
-
-### do_to
-
-```erlang
-do_to(Msg, _Req, _Opts) ->
- error({cookie_to_error, {unexpected_message_format, Msg}}).
-```
-
-### to_set_cookie_line
-
-Convert a single cookie into a `set-cookie` header line. The cookie
-
-```erlang
-to_set_cookie_line(Key, RawCookie) ->
- Cookie = normalize_cookie_value(RawCookie),
- % Encode the cookie key-value pair as a string to use as the base.
-```
-
-### to_cookie_line
-
-Convert a single cookie into a `cookie` header component. These
-
-```erlang
-to_cookie_line(Key, Cookie) ->
- to_set_cookie_line(Key, value(Cookie)).
-```
-
-### from
-
-Normalize a message containing a `cookie`, `set-cookie`, and potentially
-
-```erlang
-from(Msg, Req, Opts) ->
- CookieOpts = opts(Opts),
- LoadedMsg = hb_cache:ensure_all_loaded(Msg, Opts),
- do_from(LoadedMsg, Req, CookieOpts).
-```
-
-### do_from
-
-```erlang
-do_from(Msg, Req, Opts) when is_map(Msg) ->
- {ok, ResetBase} = reset(Msg, Opts),
- % Get the cookies, parsed, from each available source.
-```
-
-### do_from
-
-```erlang
-do_from(CookiesMsg, _Req, _Opts) ->
- error({cookie_from_error, {unexpected_message_format, CookiesMsg}}).
-```
-
-### from_cookie
-
-Convert the `cookie` key into a parsed cookie message. `cookie` headers
-
-```erlang
-from_cookie(#{ <<"cookie">> := Cookie }, Req, Opts) ->
- from_cookie(Cookie, Req, Opts);
-```
-
-### from_cookie
-
-Convert the `cookie` key into a parsed cookie message. `cookie` headers
-
-```erlang
-from_cookie(Cookies, Req, Opts) when is_list(Cookies) ->
- MergedParsed =
- lists:foldl(
- fun(Cookie, Acc) ->
- {ok, Parsed} = from_cookie(Cookie, Req, Opts),
- hb_maps:merge(Acc, Parsed, Opts)
- end,
- #{},
- Cookies
- ),
- {ok, MergedParsed};
-```
-
-### from_cookie
-
-Convert the `cookie` key into a parsed cookie message. `cookie` headers
-
-```erlang
-from_cookie(Cookie, _Req, _Opts) when is_binary(Cookie) ->
- BinaryCookiePairs = split(semicolon, Cookie),
- KeyValList =
- lists:map(
- fun(BinaryCookiePair) ->
- {[Key, Value], _Rest} = split(pair, BinaryCookiePair),
- {Key, hb_escape:decode(Value)}
- end,
- BinaryCookiePairs
- ),
- NormalizedMessage = maps:from_list(KeyValList),
- {ok, NormalizedMessage};
-```
-
-### from_cookie
-
-Convert the `cookie` key into a parsed cookie message. `cookie` headers
-
-```erlang
-from_cookie(_MsgWithoutCookie, _Req, _Opts) ->
- % The cookie key is not present in the message, so we return an empty map.
-```
-
-### from_set_cookie
-
-Convert a `set-cookie` header line into a cookie message. The `set-cookie`
-
-```erlang
-from_set_cookie(#{ <<"set-cookie">> := Cookie }, Req, Opts) ->
- ?event({from_set_cookie, {cookie, Cookie}}),
- from_set_cookie(Cookie, Req, Opts);
-```
-
-### from_set_cookie
-
-Convert a `set-cookie` header line into a cookie message. The `set-cookie`
-
-```erlang
-from_set_cookie(MsgWithoutSet, _Req, _Opts) when is_map(MsgWithoutSet) ->
- % The set-cookie key is not present in the message, so we return an empty map.
-```
-
-### from_set_cookie
-
-```erlang
-from_set_cookie(Lines, Req, Opts) when is_list(Lines) ->
- MergedParsed =
- lists:foldl(
- fun(Line, Acc) ->
- {ok, Parsed} = from_set_cookie(Line, Req, Opts),
- hb_maps:merge(Acc, Parsed)
- end,
- #{},
- Lines
- ),
- {ok, MergedParsed};
-```
-
-### from_set_cookie
-
-```erlang
-from_set_cookie(Line, _Req, Opts) when is_binary(Line) ->
- {[Key, Value], Rest} = split(pair, Line),
- ValueDecoded = hb_escape:decode(Value),
- % If there is no remaining binary after the pair, we have a simple key-value
- % pair, returning just the binary as the value. Otherwise, we split the
- % remaining binary into attributes and flags and return a message with the
- % value and those parsed elements.
-```
-
-### to_sorted_list
-
-Takes a message or list of binaries and returns a sorted list of key-
-
-```erlang
-to_sorted_list(Msg) when is_map(Msg) ->
- lists:keysort(
- 1,
- [
- {trim_bin(hb_util:bin(K)), trim_bin(V)}
- || {K, V} <- maps:to_list(Msg)
- ]
- );
-```
-
-### to_sorted_list
-
-Takes a message or list of binaries and returns a sorted list of key-
-
-```erlang
-to_sorted_list(Binaries) when is_list(Binaries) ->
- lists:sort(
- lists:map(
- fun(Bin) -> trim_bin(hb_util:bin(Bin)) end,
- Binaries
- )
- ).
-```
-
-### value
-
-Take a single parse cookie and return only the value (ignoring attributes
-
-```erlang
-value(Msg) when is_map(Msg) ->
- maps:get(<<"value">>, Msg, Msg);
-```
-
-### value
-
-Take a single parse cookie and return only the value (ignoring attributes
-
-```erlang
-value(Bin) when is_binary(Bin) ->
- Bin.
-```
-
-### normalize_cookie_value
-
-Normalize a cookie value to a map with the following keys:
-
-```erlang
-normalize_cookie_value(Msg) when is_map(Msg) ->
- Msg#{
- <<"value">> => maps:get(<<"value">>, Msg, Msg),
- <<"attributes">> => maps:get(<<"attributes">>, Msg, #{}),
- <<"flags">> => maps:get(<<"flags">>, Msg, [])
- };
-```
-
-### normalize_cookie_value
-
-Normalize a cookie value to a map with the following keys:
-
-```erlang
-normalize_cookie_value(Bin) when is_binary(Bin) ->
- #{
- <<"value">> => Bin,
- <<"attributes">> => #{},
- <<"flags">> => []
- }.
-```
-
-### trim_bin
-
-Trim a binary of leading and trailing whitespace.
-
-```erlang
-trim_bin(Bin) when is_binary(Bin) ->
- list_to_binary(string:trim(binary_to_list(Bin))).
-```
-
-### join
-
-Join a list of binaries into a `separator`-separated string. Abstracts
-
-```erlang
-join(Binaries, Separator) ->
- hb_util:bin(
- string:join(
- lists:map(fun hb_util:list/1, Binaries),
- hb_util:list(Separator)
- )
- ).
-```
-
-### split
-
-Split a binary by a separator type (`pair`, `lines`, or `attributes`).
-
-```erlang
-split(pair, Bin) ->
- [Key, ValueRest] = binary:split(Bin, <<"=">>),
- {_, Value, Rest} = hb_util:split_depth_string_aware_single($;, ValueRest),
- {[Key, unquote(Value)], trim_leading(Rest)};
-```
-
-### split
-
-Split a binary by a separator type (`pair`, `lines`, or `attributes`).
-
-```erlang
-split(lines, Bin) ->
- lists:map(fun trim_leading/1, hb_util:split_depth_string_aware($,, Bin));
-```
-
-### split
-
-Split a binary by a separator type (`pair`, `lines`, or `attributes`).
-
-```erlang
-split(semicolon, Bin) ->
- lists:map(fun trim_leading/1, hb_util:split_depth_string_aware($;, Bin)).
-```
-
-### trim_leading
-
-Remove leading whitespace from a binary, if present.
-
-```erlang
-trim_leading(Line) when not is_binary(Line) ->
- trim_leading(hb_util:bin(Line));
-```
-
-### trim_leading
-
-Remove leading whitespace from a binary, if present.
-
-```erlang
-trim_leading(<<>>) -> <<>>;
-```
-
-### trim_leading
-
-Remove leading whitespace from a binary, if present.
-
-```erlang
-trim_leading(<<" ", Rest/binary>>) -> trim_leading(Rest);
-```
-
-### trim_leading
-
-Remove leading whitespace from a binary, if present.
-Unquote a binary if it is quoted. If it is not quoted, we return the
-
-```erlang
-trim_leading(Line) -> Line.
-```
-
-### unquote
-
-Remove leading whitespace from a binary, if present.
-Unquote a binary if it is quoted. If it is not quoted, we return the
-
-```erlang
-unquote(<< $\", Rest/binary>>) ->
- {Unquoted, _} = hb_util:split_escaped_single($\", Rest),
- Unquoted;
-```
-
----
-
-*Generated from [dev_codec_cookie.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie.erl)*
diff --git a/docs/book/src/dev_codec_cookie_auth.erl.md b/docs/book/src/dev_codec_cookie_auth.erl.md
deleted file mode 100644
index 6f50f3641..000000000
--- a/docs/book/src/dev_codec_cookie_auth.erl.md
+++ /dev/null
@@ -1,329 +0,0 @@
-# dev_codec_cookie_auth
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie_auth.erl)
-
-Implements the `message@1.0` commitment interface for the `~cookie@1.0`,
-as well as the `generator` interface type for the `~auth-hook@1.0` device.
-See the [cookie codec](dev_codec_cookie.html) documentation for more details.
-
----
-
-## Exported Functions
-
-- `commit/3`
-- `finalize/3`
-- `generate/3`
-- `verify/3`
-
----
-
-### generate
-
-Implements the `message@1.0` commitment interface for the `~cookie@1.0`,
-Generate a new secret (if no `committer` specified), and use it as the
-
-```erlang
-generate(Base, Request, Opts) ->
- {WithCookie, Secrets} =
- case find_secrets(Request, Opts) of
- [] ->
- {ok, GeneratedSecret} = generate_secret(Base, Request, Opts),
- {ok, Updated} = store_secret(GeneratedSecret, Request, Opts),
- {Updated, [GeneratedSecret]};
- FoundSecrets ->
- {Request, FoundSecrets}
- end,
- ?event({normalized_cookies_found, {secrets, Secrets}}),
- {
- ok,
- WithCookie#{
- <<"secret">> => Secrets
- }
- }.
-```
-
-### finalize
-
-Finalize an `on-request` hook by adding the cookie to the chain of
-
-```erlang
-finalize(Base, Request, Opts) ->
- ?event(debug_auth, {finalize, {base, Base}, {request, Request}}),
- maybe
- {ok, SignedMsg} ?= hb_maps:find(<<"request">>, Request, Opts),
- {ok, MessageSequence} ?= hb_maps:find(<<"body">>, Request, Opts),
- % Cookie auth adds set-cookie to response
- {ok, #{ <<"set-cookie">> := SetCookie }} =
- dev_codec_cookie:to(
- SignedMsg,
- #{ <<"format">> => <<"set-cookie">> },
- Opts
- ),
- {
- ok,
- MessageSequence ++
- [#{ <<"path">> => <<"set">>, <<"set-cookie">> => SetCookie }]
- }
- else error ->
- {error, no_request}
- end.
-```
-
-### commit
-
-Generate a new secret (if no `committer` specified), and use it as the
-
-```erlang
-commit(Base, Request, RawOpts) when ?IS_LINK(Request) ->
- Opts = dev_codec_cookie:opts(RawOpts),
- commit(Base, hb_cache:ensure_loaded(Request, Opts), Opts);
-```
-
-### commit
-
-Generate a new secret (if no `committer` specified), and use it as the
-
-```erlang
-commit(Base, Req = #{ <<"secret">> := Secret }, RawOpts) ->
- Opts = dev_codec_cookie:opts(RawOpts),
- commit(hb_cache:ensure_loaded(Secret, Opts), Base, Req, Opts);
-```
-
-### commit
-
-Generate a new secret (if no `committer` specified), and use it as the
-
-```erlang
-commit(Base, Request, RawOpts) ->
- Opts = dev_codec_cookie:opts(RawOpts),
- % Calculate the key to use for the commitment.
-```
-
-### commit
-
-Given the secret key, commit the message and set the cookie. This
-
-```erlang
-commit(Secret, Base, Request, Opts) ->
- {ok, CommittedMsg} =
- dev_codec_httpsig_proxy:commit(
- <<"cookie@1.0">>,
- Secret,
- Base,
- Request,
- Opts
- ),
- store_secret(Secret, CommittedMsg, Opts).
-```
-
-### store_secret
-
-Update the nonces for a given secret.
-
-```erlang
-store_secret(Secret, Msg, Opts) ->
- CookieAddr = dev_codec_httpsig_keyid:secret_key_to_committer(Secret),
- % Create the cookie parameters, using the name as the key and the secret as
- % the value.
-```
-
-### verify
-
-Verify the HMAC commitment with the key being the secret from the
-
-```erlang
-verify(Base, ReqLink, RawOpts) when ?IS_LINK(ReqLink) ->
- Opts = dev_codec_cookie:opts(RawOpts),
- verify(Base, hb_cache:ensure_loaded(ReqLink, Opts), Opts);
-```
-
-### verify
-
-Verify the HMAC commitment with the key being the secret from the
-
-```erlang
-verify(Base, Req = #{ <<"secret">> := Secret }, RawOpts) ->
- Opts = dev_codec_cookie:opts(RawOpts),
- ?event({verify_with_explicit_key, {base, Base}, {request, Req}}),
- dev_codec_httpsig_proxy:verify(
- hb_util:decode(Secret),
- Base,
- Req,
- Opts
- );
-```
-
-### verify
-
-Verify the HMAC commitment with the key being the secret from the
-
-```erlang
-verify(Base, Request, RawOpts) ->
- Opts = dev_codec_cookie:opts(RawOpts),
- ?event({verify_finding_key, {base, Base}, {request, Request}}),
- case find_secret(Request, Opts) of
- {ok, Secret} ->
- dev_codec_httpsig_proxy:verify(
- hb_util:decode(Secret),
- Base,
- Request,
- Opts
- );
- {error, Err} ->
- {error, Err}
- end.
-```
-
-### generate_secret
-
-Generate a new secret key for the given request. The user may specify
-
-```erlang
-generate_secret(_Base, Request, Opts) ->
- case hb_maps:get(<<"generator">>, Request, undefined, Opts) of
- undefined ->
- % If no generator is specified, use the default generator.
-```
-
-### default_generator
-
-Generate a new secret key using the default generator.
-
-```erlang
-default_generator(_Opts) ->
- {ok, hb_util:encode(crypto:strong_rand_bytes(64))}.
-```
-
-### execute_generator
-
-Execute a generator function. See `generate_secret/3` for more details.
-
-```erlang
-execute_generator(GeneratorPath, Opts) when is_binary(GeneratorPath) ->
- hb_ao:resolve(GeneratorPath, Opts);
-```
-
-### execute_generator
-
-Execute a generator function. See `generate_secret/3` for more details.
-Find all secrets in the cookie of a message.
-
-```erlang
-execute_generator(Generator, Opts) ->
- Path = hb_maps:get(<<"path">>, Generator, <<"generate">>, Opts),
- hb_ao:resolve(Generator#{ <<"path">> => Path }, Opts).
-```
-
-### find_secrets
-
-Execute a generator function. See `generate_secret/3` for more details.
-Find all secrets in the cookie of a message.
-
-```erlang
-find_secrets(Request, Opts) ->
- maybe
- {ok, Cookie} ?= dev_codec_cookie:extract(Request, #{}, Opts),
- [
- hb_maps:get(SecretRef, Cookie, secret_unavailable, Opts)
- ||
- SecretRef = <<"secret-", _/binary>> <- hb_maps:keys(Cookie)
- ]
- else error -> []
- end.
-```
-
-### find_secret
-
-Find the secret key for the given committer, if it exists in the cookie.
-
-```erlang
-find_secret(Request, Opts) ->
- maybe
- {ok, Committer} ?= hb_maps:find(<<"committer">>, Request, Opts),
- find_secret(Committer, Request, Opts)
- else error -> {error, no_secret}
- end.
-```
-
-### find_secret
-
-```erlang
-find_secret(Committer, Request, Opts) ->
- maybe
- {ok, Cookie} ?= dev_codec_cookie:extract(Request, #{}, Opts),
- {ok, _Secret} ?= hb_maps:find(<<"secret-", Committer/binary>>, Cookie, Opts)
- else error -> {error, not_found}
- end.
-```
-
-### directly_invoke_commit_verify_test
-
-Call the cookie codec's `commit` and `verify` functions directly.
-
-```erlang
-directly_invoke_commit_verify_test() ->
- Base = #{ <<"test-key">> => <<"test-value">> },
- CommittedMsg =
- hb_message:commit(
- Base,
- #{},
- #{
- <<"commitment-device">> => <<"cookie@1.0">>
- }
- ),
- ?event({committed_msg, CommittedMsg}),
- ?assertEqual(1, length(hb_message:signers(CommittedMsg, #{}))),
- VerifyReq =
- apply_cookie(
- CommittedMsg#{
- <<"committers">> => hb_message:signers(CommittedMsg, #{})
- },
- CommittedMsg,
- #{}
- ),
- VerifyReqWithoutComms = hb_maps:without([<<"commitments">>], VerifyReq, #{}),
- ?event({verify_req_without_comms, VerifyReqWithoutComms}),
- ?assert(hb_message:verify(CommittedMsg, VerifyReqWithoutComms, #{})),
- ok.
-```
-
-### http_set_get_cookies_test
-
-Set keys in a cookie and verify that they can be parsed into a message.
-
-```erlang
-http_set_get_cookies_test() ->
- Node = hb_http_server:start_node(#{}),
- {ok, SetRes} =
- hb_http:get(
- Node,
- <<"/~cookie@1.0/store?k1=v1&k2=v2">>,
- #{}
- ),
- ?event(debug_cookie, {set_cookie_test, {set_res, SetRes}}),
- ?assertMatch(#{ <<"set-cookie">> := _ }, SetRes),
- Req = apply_cookie(#{ <<"path">> => <<"/~cookie@1.0/extract">> }, SetRes, #{}),
- {ok, Res} = hb_http:get(Node, Req, #{}),
- ?assertMatch(#{ <<"k1">> := <<"v1">>, <<"k2">> := <<"v2">> }, Res),
- ok.
-```
-
-### apply_cookie
-
-Takes the cookies from the `GenerateResponse` and applies them to the
-
-```erlang
-apply_cookie(NextReq, GenerateResponse, Opts) ->
- {ok, Cookie} = dev_codec_cookie:extract(GenerateResponse, #{}, Opts),
- {ok, NextWithParsedCookie} = dev_codec_cookie:store(NextReq, Cookie, Opts),
- {ok, NextWithCookie} =
- dev_codec_cookie:to(
- NextWithParsedCookie,
- #{ <<"format">> => <<"cookie">> },
- Opts
- ),
-```
-
----
-
-*Generated from [dev_codec_cookie_auth.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie_auth.erl)*
diff --git a/docs/book/src/dev_codec_cookie_test_vectors.erl.md b/docs/book/src/dev_codec_cookie_test_vectors.erl.md
deleted file mode 100644
index 54cf6b062..000000000
--- a/docs/book/src/dev_codec_cookie_test_vectors.erl.md
+++ /dev/null
@@ -1,904 +0,0 @@
-# dev_codec_cookie_test_vectors
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie_test_vectors.erl)
-
-A battery of cookie parsing and encoding test vectors.
-
----
-
-### assert_set
-
-A battery of cookie parsing and encoding test vectors.
-Assert that when given the inputs in the test set, the outputs are
-
-```erlang
-assert_set(TestSet, Fun) ->
- {Inputs, Expected} = maps:get(TestSet, test_data()),
- ?event(match_cookie, {starting_group_match, {inputs, {explicit, Inputs}}}),
- lists:foreach(
- fun(Input) ->
- Res = Fun(Input),
- ?event(
- match_cookie,
- {matching,
- {expected, {explicit, Expected}, {output, {explicit, Res}}}
- }
- ),
- ?assertEqual(Expected, Res)
- end,
- Inputs
- ).
-```
-
-### to_string
-
-Convert a cookie message to a string.
-Convert a string to a cookie message.
-
-```erlang
-to_string(CookieMsg) ->
- {ok, BaseMsg} = dev_codec_cookie:store(#{}, CookieMsg, #{}),
- {ok, Msg} =
- dev_codec_cookie:to(
- BaseMsg,
- #{ <<"format">> => <<"set-cookie">> },
- #{}
- ),
- hb_maps:get(<<"set-cookie">>, Msg, [], #{}).
-```
-
-### from_string
-
-Convert a cookie message to a string.
-Convert a string to a cookie message.
-
-```erlang
-from_string(String) ->
- {ok, BaseMsg} =
- dev_codec_cookie:from(
- #{ <<"set-cookie">> => String },
- #{},
- #{}
- ),
- {ok, Cookie} = dev_codec_cookie:extract(BaseMsg, #{}, #{}),
- Cookie.
-```
-
-### test_data
-
-returns a map of tuples of the form `testset_name => {[before], after}`.
-
-```erlang
-test_data() ->
- #{
- from_string_raw_value =>
- {
- [<<"k1=v1">>, <<"k1=\"v1\"">>],
- #{ <<"k1">> => <<"v1">> }
- },
- from_string_attributes =>
- {
- [<<"k1=v1; k2=v2">>, <<"k1=\"v1\"; k2=\"v2\"">>],
- #{
- <<"k1">> =>
- #{
- <<"value">> => <<"v1">>,
- <<"attributes">> => #{ <<"k2">> => <<"v2">> }
- }
- }
- },
- from_string_flags =>
- {
- [<<"k1=v1; k2=v2; f1; f2">>, <<"k1=\"v1\"; k2=\"v2\"; f1; f2">>],
- #{
- <<"k1">> =>
- #{
- <<"value">> => <<"v1">>,
- <<"attributes">> => #{ <<"k2">> => <<"v2">> },
- <<"flags">> => [<<"f1">>, <<"f2">>]
- }
- }
- },
- to_string_raw_value =>
- {
- [
- #{ <<"k1">> => <<"v1">> },
- #{ <<"k1">> => #{ <<"value">> => <<"v1">> } },
- #{
- <<"k1">> =>
- #{
- <<"value">> => <<"v1">>,
- <<"attributes">> => #{},
- <<"flags">> => []
- }
- }
- ],
- [<<"k1=\"v1\"">>]
- },
- to_string_attributes =>
- {
- [
- #{
- <<"k1">> =>
- #{
- <<"value">> => <<"v1">>,
- <<"attributes">> => #{ <<"k2">> => <<"v2">> }
- }
- },
- #{
- <<"k1">> =>
- #{
- <<"value">> => <<"v1">>,
- <<"attributes">> => #{ <<"k2">> => <<"v2">> },
- <<"flags">> => []
- }
- }
- ],
- [<<"k1=\"v1\"; k2=v2">>]
- },
- to_string_flags =>
- {
- [
- #{
- <<"k1">> =>
- #{
- <<"value">> => <<"v1">>,
- <<"flags">> => [<<"f1">>, <<"f2">>]
- }
- },
- #{
- <<"k1">> =>
- #{
- <<"value">> => <<"v1">>,
- <<"attributes">> => #{},
- <<"flags">> => [<<"f1">>, <<"f2">>]
- }
- }
- ],
- [<<"k1=\"v1\"; f1; f2">>]
- },
- parse_realworld_1 =>
- {
- [
- [
- <<"cart=110045_77895_53420; SameSite=Strict">>,
- <<"affiliate=e4rt45dw; SameSite=Lax">>
- ]
- ],
- #{
- <<"cart">> =>
- #{
- <<"value">> => <<"110045_77895_53420">>,
- <<"attributes">> => #{ <<"SameSite">> => <<"Strict">> }
- },
- <<"affiliate">> =>
- #{
- <<"value">> => <<"e4rt45dw">>,
- <<"attributes">> => #{ <<"SameSite">> => <<"Lax">> }
- }
- }
- },
- parse_user_settings_and_permissions =>
- {
- [
- [
- <<"user_settings=notifications=true,privacy=strict,layout=grid; Path=/; HttpOnly; Secure">>,
- <<"user_permissions=\"read;write;delete\"; Path=/; SameSite=None; Secure">>
- ]
- ],
- #{
- <<"user_settings">> =>
- #{
- <<"value">> => <<"notifications=true,privacy=strict,layout=grid">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"HttpOnly">>, <<"Secure">>]
- },
- <<"user_permissions">> =>
- #{
- <<"value">> => <<"read;write;delete">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">>, <<"SameSite">> => <<"None">> },
- <<"flags">> => [<<"Secure">>]
- }
- }
- },
- parse_session_and_temp_data =>
- {
- [
- [
- <<"SESSION_ID=abc123xyz ; path= /dashboard ; samesite=Strict ; Secure">>,
- <<"temp_data=cleanup_me; Max-Age=-1; Path=/">>
- ]
- ],
- #{
- <<"SESSION_ID">> =>
- #{
- <<"value">> => <<"abc123xyz ">>,
- <<"attributes">> => #{ <<"path">> => <<"/dashboard">>, <<"samesite">> => <<"Strict">> },
- <<"flags">> => [<<"Secure">>]
- },
- <<"temp_data">> =>
- #{
- <<"value">> => <<"cleanup_me">>,
- <<"attributes">> => #{ <<"Max-Age">> => <<"-1">>, <<"Path">> => <<"/">> }
- }
- }
- },
- parse_empty_and_anonymous =>
- {
- [
- [
- <<"user_preference=; Path=/; HttpOnly">>,
- <<"=anonymous_session_123; Path=/guest">>
- ]
- ],
- #{
- <<"user_preference">> =>
- #{
- <<"value">> => <<"">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"HttpOnly">>]
- },
- <<>> =>
- #{
- <<"value">> => <<"anonymous_session_123">>,
- <<"attributes">> => #{ <<"Path">> => <<"/guest">> }
- }
- }
- },
- parse_app_config_and_analytics =>
- {
- [
- [
- <<"$app_config$=theme@dark!%20mode; Path=/">>,
- <<"analytics_session_data_with_very_long_name_for_tracking_purposes=comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more; Path=/">>
- ]
- ],
- #{
- <<"$app_config$">> =>
- #{
- <<"value">> => <<"theme@dark! mode">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- },
- <<"analytics_session_data_with_very_long_name_for_tracking_purposes">> =>
- #{
- <<"value">> => <<"comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- }
- }
- },
- parse_debug_and_tracking =>
- {
- [
- [
- <<"debug_info=\\tIndented\\t\\nMultiline\\n; Path=/">>,
- <<"tracking_id=user_12345; CustomAttr=CustomValue; Analytics=Enabled; Path=/; HttpOnly">>
- ]
- ],
- #{
- <<"debug_info">> =>
- #{
- <<"value">> => <<"\\tIndented\\t\\nMultiline\\n">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- },
- <<"tracking_id">> =>
- #{
- <<"value">> => <<"user_12345">>,
- <<"attributes">> => #{
- <<"CustomAttr">> => <<"CustomValue">>,
- <<"Analytics">> => <<"Enabled">>,
- <<"Path">> => <<"/">>
- },
- <<"flags">> => [<<"HttpOnly">>]
- }
- }
- },
- parse_cache_and_form_token =>
- {
- [
- [
- <<"cache_bust=v1.2.3; Expires=Mon, 99 Feb 2099 25:99:99 GMT; Path=/">>,
- <<"form_token=form_abc123; SameSite=Strick; Secure">>
- ]
- ],
- #{
- <<"cache_bust">> =>
- #{
- <<"value">> => <<"v1.2.3">>,
- <<"attributes">> => #{
- <<"Expires">> => <<"Mon, 99 Feb 2099 25:99:99 GMT">>,
- <<"Path">> => <<"/">>
- }
- },
- <<"form_token">> =>
- #{
- <<"value">> => <<"form_abc123">>,
- <<"attributes">> => #{ <<"SameSite">> => <<"Strick">> },
- <<"flags">> => [<<"Secure">>]
- }
- }
- },
- parse_token_and_reactions =>
- {
- [
- [
- <<"access_token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c; Path=/; HttpOnly; Secure">>,
- <<"reaction_prefs=👍👎; Path=/; Secure">>
- ]
- ],
- #{
- <<"access_token">> =>
- #{
- <<"value">> => <<"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"HttpOnly">>, <<"Secure">>]
- },
- <<"reaction_prefs">> =>
- #{
- <<"value">> => <<"👍👎">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"Secure">>]
- }
- }
- },
- parse_error_log_and_auth_token =>
- {
- [
- [
- <<"error_log=\"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed\"; Path=/">>,
- <<"auth_token=bearer_xyz789; Secure; Path=/api; Secure; HttpOnly">>
- ]
- ],
- #{
- <<"error_log">> =>
- #{
- <<"value">> => <<"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- },
- <<"auth_token">> =>
- #{
- <<"value">> => <<"bearer_xyz789">>,
- <<"attributes">> => #{ <<"Path">> => <<"/api">> },
- <<"flags">> => [<<"HttpOnly">>,<<"Secure">>, <<"Secure">>]
- }
- }
- },
- parse_csrf_and_quick_setting =>
- {
- [
- [
- <<"csrf_token=abc123; \"HttpOnly\"; Path=/">>,
- <<"quick_setting=\"enabled\"">>
- ]
- ],
- #{
- <<"csrf_token">> =>
- #{
- <<"value">> => <<"abc123">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"HttpOnly">>]
- },
- <<"quick_setting">> => <<"enabled">>
- }
- },
- parse_admin_and_upload =>
- {
- [
- [
- <<"secret_key=confidential; Path=%2Fadmin">>,
- <<"admin_flag=true; Path=/">>
- ]
- ],
- #{
- <<"secret_key">> =>
- #{
- <<"value">> => <<"confidential">>,
- <<"attributes">> => #{ <<"Path">> => <<"%2Fadmin">> }
- },
- <<"admin_flag">> =>
- #{
- <<"value">> => <<"true">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- }
- }
- },
- parse_search_and_tags =>
- {
- [
- [
- <<"search_history=\"query,results\"; Path=/">>,
- <<"user_tags=\"work,personal\"; Path=/">>
- ]
- ],
- #{
- <<"search_history">> =>
- #{
- <<"value">> => <<"query,results">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- },
- <<"user_tags">> =>
- #{
- <<"value">> => <<"work,personal">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- }
- }
- },
- to_string_realworld_1 =>
- {
- [
- #{
- <<"cart">> =>
- #{
- <<"value">> => <<"110045_77895_53420">>,
- <<"attributes">> => #{ <<"SameSite">> => <<"Strict">> }
- },
- <<"affiliate">> =>
- #{
- <<"value">> => <<"e4rt45dw">>,
- <<"attributes">> => #{ <<"SameSite">> => <<"Lax">> }
- }
- }
- ],
- [
- <<"affiliate=\"e4rt45dw\"; SameSite=Lax">>,
- <<"cart=\"110045_77895_53420\"; SameSite=Strict">>
- ]
- },
- to_string_user_settings_and_permissions =>
- {
- [
- #{
- <<"user_settings">> =>
- #{
- <<"value">> => <<"notifications=true,privacy=strict,layout=grid">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"HttpOnly">>, <<"Secure">>]
- },
- <<"user_permissions">> =>
- #{
- <<"value">> => <<"read;write;delete">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">>, <<"SameSite">> => <<"None">> },
- <<"flags">> => [<<"Secure">>]
- }
- }
- ],
- [
- <<"user_permissions=\"read;write;delete\"; Path=/; SameSite=None; Secure">>,
- <<"user_settings=\"notifications=true,privacy=strict,layout=grid\"; Path=/; HttpOnly; Secure">>
- ]
- },
- to_string_session_and_temp_data =>
- {
- [
- #{
- <<"SESSION_ID">> =>
- #{
- <<"value">> => <<"abc123xyz ">>,
- <<"attributes">> => #{ <<"path">> => <<"/dashboard">>, <<"samesite">> => <<"Strict">> },
- <<"flags">> => [<<"Secure">>]
- },
- <<"temp_data">> =>
- #{
- <<"value">> => <<"cleanup_me">>,
- <<"attributes">> => #{ <<"Max-Age">> => <<"-1">>, <<"Path">> => <<"/">> }
- }
- }
- ],
- [
- <<"SESSION_ID=\"abc123xyz \"; path=/dashboard; samesite=Strict; Secure">>,
- <<"temp_data=\"cleanup_me\"; Max-Age=-1; Path=/">>
- ]
- },
- to_string_empty_and_anonymous =>
- {
- [
- #{
- <<"user_preference">> =>
- #{
- <<"value">> => <<"">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"HttpOnly">>]
- },
- <<>> =>
- #{
- <<"value">> => <<"anonymous_session_123">>,
- <<"attributes">> => #{ <<"Path">> => <<"/guest">> }
- }
- }
- ],
- [
- <<"=\"anonymous_session_123\"; Path=/guest">>,
- <<"user_preference=\"\"; Path=/; HttpOnly">>
- ]
- },
- to_string_app_config_and_analytics =>
- {
- [
- #{
- <<"$app_config$">> =>
- #{
- <<"value">> => <<"theme@dark!%20mode">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- },
- <<"analytics_session_data_with_very_long_name_for_tracking_purposes">> =>
- #{
- <<"value">> => <<"comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- }
- }
- ],
- [
- <<"$app_config$=\"theme@dark!%20mode\"; Path=/">>,
- <<"analytics_session_data_with_very_long_name_for_tracking_purposes=\"comprehensive_user_behavior_analytics_data_including_page_views_click_events_scroll_depth_time_spent_geographic_location_device_info_browser_details_and_more\"; Path=/">>
- ]
- },
- to_string_debug_and_tracking =>
- {
- [
- #{
- <<"debug_info">> =>
- #{
- <<"value">> => <<"\\tIndented\\t\\nMultiline\\n">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- },
- <<"tracking_id">> =>
- #{
- <<"value">> => <<"user_12345">>,
- <<"attributes">> => #{
- <<"CustomAttr">> => <<"CustomValue">>,
- <<"Analytics">> => <<"Enabled">>,
- <<"Path">> => <<"/">>
- },
- <<"flags">> => [<<"HttpOnly">>]
- }
- }
- ],
- [
- <<"debug_info=\"\\tIndented\\t\\nMultiline\\n\"; Path=/">>,
- <<"tracking_id=\"user_12345\"; Analytics=Enabled; CustomAttr=CustomValue; Path=/; HttpOnly">>
- ]
- },
- to_string_cache_and_form_token =>
- {
- [
- #{
- <<"cache_bust">> =>
- #{
- <<"value">> => <<"v1.2.3">>,
- <<"attributes">> => #{
- <<"Expires">> => <<"Mon, 99 Feb 2099 25:99:99 GMT">>,
- <<"Path">> => <<"/">>
- }
- },
- <<"form_token">> =>
- #{
- <<"value">> => <<"form_abc123">>,
- <<"attributes">> => #{ <<"SameSite">> => <<"Strick">> },
- <<"flags">> => [<<"Secure">>]
- }
- }
- ],
- [
- <<"cache_bust=\"v1.2.3\"; Expires=Mon, 99 Feb 2099 25:99:99 GMT; Path=/">>,
- <<"form_token=\"form_abc123\"; SameSite=Strick; Secure">>
- ]
- },
- to_string_token_and_reactions =>
- {
- [
- #{
- <<"access_token">> =>
- #{
- <<"value">> => <<"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"HttpOnly">>, <<"Secure">>]
- },
- <<"reaction_prefs">> =>
- #{
- <<"value">> => <<"👍👎">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"Secure">>]
- }
- }
- ],
- [
- <<"access_token=\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c\"; Path=/; HttpOnly; Secure">>,
- <<"reaction_prefs=\"👍👎\"; Path=/; Secure">>
- ]
- },
- to_string_error_log_and_auth_token =>
- {
- [
- #{
- <<"error_log">> =>
- #{
- <<"value">> => <<"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- },
- <<"auth_token">> =>
- #{
- <<"value">> => <<"bearer_xyz789">>,
- <<"attributes">> => #{ <<"Path">> => <<"/api">> },
- <<"flags">> => [<<"HttpOnly">>, <<"Secure">>, <<"Secure">>]
- }
- }
- ],
- [
- <<"auth_token=\"bearer_xyz789\"; Path=/api; HttpOnly; Secure; Secure">>,
- <<"error_log=\"timestamp=2024-01-15 10:30:00\\nlevel=ERROR\\tmessage=Database connection failed\"; Path=/">>
- ]
- },
- to_string_csrf_and_quick_setting =>
- {
- [
- #{
- <<"csrf_token">> =>
- #{
- <<"value">> => <<"abc123">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> },
- <<"flags">> => [<<"HttpOnly">>]
- },
- <<"quick_setting">> => <<"enabled">>
- }
- ],
- [
- <<"csrf_token=\"abc123\"; Path=/; HttpOnly">>,
- <<"quick_setting=\"enabled\"">>
- ]
- },
- to_string_admin_and_upload =>
- {
- [
- #{
- <<"secret_key">> =>
- #{
- <<"value">> => <<"confidential">>,
- <<"attributes">> => #{ <<"Path">> => <<"%2Fadmin">> }
- },
- <<"admin_flag">> =>
- #{
- <<"value">> => <<"true">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- }
- }
- ],
- [
- <<"admin_flag=\"true\"; Path=/">>,
- <<"secret_key=\"confidential\"; Path=%2Fadmin">>
- ]
- },
- to_string_search_and_tags =>
- {
- [
- #{
- <<"search_history">> =>
- #{
- <<"value">> => <<"query,results">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- },
- <<"user_tags">> =>
- #{
- <<"value">> => <<"work,personal">>,
- <<"attributes">> => #{ <<"Path">> => <<"/">> }
- }
- }
- ],
- [
- <<"search_history=\"query,results\"; Path=/">>,
- <<"user_tags=\"work,personal\"; Path=/">>
- ]
- }
- }.
-```
-
-### from_string_basic_test
-
-```erlang
-from_string_basic_test() ->
- assert_set(from_string_raw_value, fun from_string/1).
-```
-
-### from_string_attributes_test
-
-```erlang
-from_string_attributes_test() ->
- assert_set(from_string_attributes, fun from_string/1).
-```
-
-### from_string_flags_test
-
-```erlang
-from_string_flags_test() ->
- assert_set(from_string_flags, fun from_string/1).
-```
-
-### to_string_basic_test
-
-```erlang
-to_string_basic_test() ->
- assert_set(to_string_raw_value, fun to_string/1).
-```
-
-### to_string_attributes_test
-
-```erlang
-to_string_attributes_test() ->
- assert_set(to_string_attributes, fun to_string/1).
-```
-
-### to_string_flags_test
-
-```erlang
-to_string_flags_test() ->
- assert_set(to_string_flags, fun to_string/1).
-```
-
-### parse_realworld_test
-
-```erlang
-parse_realworld_test() ->
- assert_set(parse_realworld_1, fun from_string/1).
-```
-
-### parse_user_settings_and_permissions_test
-
-```erlang
-parse_user_settings_and_permissions_test() ->
- assert_set(parse_user_settings_and_permissions, fun from_string/1).
-```
-
-### parse_session_and_temp_data_test
-
-```erlang
-parse_session_and_temp_data_test() ->
- assert_set(parse_session_and_temp_data, fun from_string/1).
-```
-
-### parse_empty_and_anonymous_test
-
-```erlang
-parse_empty_and_anonymous_test() ->
- assert_set(parse_empty_and_anonymous, fun from_string/1).
-```
-
-### parse_app_config_and_analytics_test
-
-```erlang
-parse_app_config_and_analytics_test() ->
- assert_set(parse_app_config_and_analytics, fun from_string/1).
-```
-
-### parse_debug_and_tracking_test
-
-```erlang
-parse_debug_and_tracking_test() ->
- assert_set(parse_debug_and_tracking, fun from_string/1).
-```
-
-### parse_cache_and_form_token_test
-
-```erlang
-parse_cache_and_form_token_test() ->
- assert_set(parse_cache_and_form_token, fun from_string/1).
-```
-
-### parse_token_and_reactions_test
-
-```erlang
-parse_token_and_reactions_test() ->
- assert_set(parse_token_and_reactions, fun from_string/1).
-```
-
-### parse_error_log_and_auth_token_test
-
-```erlang
-parse_error_log_and_auth_token_test() ->
- assert_set(parse_error_log_and_auth_token, fun from_string/1).
-```
-
-### parse_csrf_and_quick_setting_test
-
-```erlang
-parse_csrf_and_quick_setting_test() ->
- assert_set(parse_csrf_and_quick_setting, fun from_string/1).
-```
-
-### parse_admin_and_upload_test
-
-```erlang
-parse_admin_and_upload_test() ->
- assert_set(parse_admin_and_upload, fun from_string/1).
-```
-
-### parse_search_and_tags_test
-
-```erlang
-parse_search_and_tags_test() ->
- assert_set(parse_search_and_tags, fun from_string/1).
-```
-
-### to_string_realworld_1_test
-
-```erlang
-to_string_realworld_1_test() ->
- assert_set(to_string_realworld_1, fun to_string/1).
-```
-
-### to_string_user_settings_and_permissions_test
-
-```erlang
-to_string_user_settings_and_permissions_test() ->
- assert_set(to_string_user_settings_and_permissions, fun to_string/1).
-```
-
-### to_string_session_and_temp_data_test
-
-```erlang
-to_string_session_and_temp_data_test() ->
- assert_set(to_string_session_and_temp_data, fun to_string/1).
-```
-
-### to_string_empty_and_anonymous_test
-
-```erlang
-to_string_empty_and_anonymous_test() ->
- assert_set(to_string_empty_and_anonymous, fun to_string/1).
-```
-
-### to_string_app_config_and_analytics_test
-
-```erlang
-to_string_app_config_and_analytics_test() ->
- assert_set(to_string_app_config_and_analytics, fun to_string/1).
-```
-
-### to_string_debug_and_tracking_test
-
-```erlang
-to_string_debug_and_tracking_test() ->
- assert_set(to_string_debug_and_tracking, fun to_string/1).
-```
-
-### to_string_cache_and_form_token_test
-
-```erlang
-to_string_cache_and_form_token_test() ->
- assert_set(to_string_cache_and_form_token, fun to_string/1).
-```
-
-### to_string_token_and_reactions_test
-
-```erlang
-to_string_token_and_reactions_test() ->
- assert_set(to_string_token_and_reactions, fun to_string/1).
-```
-
-### to_string_error_log_and_auth_token_test
-
-```erlang
-to_string_error_log_and_auth_token_test() ->
- assert_set(to_string_error_log_and_auth_token, fun to_string/1).
-```
-
-### to_string_csrf_and_quick_setting_test
-
-```erlang
-to_string_csrf_and_quick_setting_test() ->
- assert_set(to_string_csrf_and_quick_setting, fun to_string/1).
-```
-
-### to_string_admin_and_upload_test
-
-```erlang
-to_string_admin_and_upload_test() ->
- assert_set(to_string_admin_and_upload, fun to_string/1).
-```
-
-### to_string_search_and_tags_test
-
-```erlang
-to_string_search_and_tags_test() ->
-```
-
----
-
-*Generated from [dev_codec_cookie_test_vectors.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_cookie_test_vectors.erl)*
diff --git a/docs/book/src/dev_codec_flat.erl.md b/docs/book/src/dev_codec_flat.erl.md
deleted file mode 100644
index 4bb41a9a6..000000000
--- a/docs/book/src/dev_codec_flat.erl.md
+++ /dev/null
@@ -1,266 +0,0 @@
-# dev_codec_flat
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_flat.erl)
-
-A codec for turning TABMs into/from flat Erlang maps that have
-(potentially multi-layer) paths as their keys, and a normal TABM binary as
-their value.
-
----
-
-## Exported Functions
-
-- `commit/3`
-- `deserialize/1`
-- `from/3`
-- `serialize/1`
-- `serialize/2`
-- `to/3`
-- `verify/3`
-
----
-
-### commit
-
-A codec for turning TABMs into/from flat Erlang maps that have
-
-```erlang
-commit(Msg, Req, Opts) -> dev_codec_httpsig:commit(Msg, Req, Opts).
-```
-
-### verify
-
-A codec for turning TABMs into/from flat Erlang maps that have
-Convert a flat map to a TABM.
-
-```erlang
-verify(Msg, Req, Opts) -> dev_codec_httpsig:verify(Msg, Req, Opts).
-```
-
-### from
-
-A codec for turning TABMs into/from flat Erlang maps that have
-Convert a flat map to a TABM.
-
-```erlang
-from(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin};
-```
-
-### from
-
-A codec for turning TABMs into/from flat Erlang maps that have
-Convert a flat map to a TABM.
-
-```erlang
-from(Map, Req, Opts) when is_map(Map) ->
- {ok,
- maps:fold(
- fun(Path, Value, Acc) ->
- case Value of
- [] ->
- ?event(error,
- {empty_list_value,
- {path, Path},
- {value, Value},
- {map, Map}
- }
- );
- _ ->
- ok
- end,
- hb_util:deep_set(
- hb_path:term_to_path_parts(Path, Opts),
- hb_util:ok(from(Value, Req, Opts)),
- Acc,
- Opts
- )
- end,
- #{},
- Map
- )
- }.
-```
-
-### to
-
-Convert a TABM to a flat map.
-
-```erlang
-to(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin};
-```
-
-### to
-
-Convert a TABM to a flat map.
-
-```erlang
-to(Map, Req, Opts) when is_map(Map) ->
- Res =
- maps:fold(
- fun(Key, Value, Acc) ->
- case to(Value, Req, Opts) of
- {ok, SubMap} when is_map(SubMap) ->
- maps:fold(
- fun(SubKey, SubValue, InnerAcc) ->
- maps:put(
- hb_path:to_binary([Key, SubKey]),
- SubValue,
- InnerAcc
- )
- end,
- Acc,
- SubMap
- );
- {ok, SimpleValue} ->
- maps:put(hb_path:to_binary([Key]), SimpleValue, Acc)
- end
- end,
- #{},
- Map
- ),
- {ok, Res}.
-```
-
-### serialize
-
-```erlang
-serialize(Map) when is_map(Map) ->
- serialize(Map, #{}).
-```
-
-### serialize
-
-```erlang
-serialize(Map, Opts) when is_map(Map) ->
- Flattened = hb_message:convert(Map, <<"flat@1.0">>, #{}),
- {ok,
- iolist_to_binary(lists:foldl(
- fun(Key, Acc) ->
- [
- Acc,
- hb_path:to_binary(Key),
- <<": ">>,
- hb_maps:get(Key, Flattened, Opts), <<"\n">>
- ]
- end,
- <<>>,
- hb_util:to_sorted_keys(Flattened, Opts)
- )
- )
- }.
-```
-
-### deserialize
-
-```erlang
-deserialize(Bin) when is_binary(Bin) ->
- Flat = lists:foldl(
- fun(Line, Acc) ->
- case binary:split(Line, <<": ">>, [global]) of
- [Key, Value] ->
- Acc#{ Key => Value };
- _ ->
- Acc
- end
- end,
- #{},
- binary:split(Bin, <<"\n">>, [global])
- ),
- {ok, hb_message:convert(Flat, <<"structured@1.0">>, <<"flat@1.0">>, #{})}.
-%%% Tests
-```
-
-### simple_conversion_test
-
-```erlang
-simple_conversion_test() ->
- Flat = #{[<<"a">>] => <<"value">>},
- Nested = #{<<"a">> => <<"value">>},
- ?assert(hb_message:match(Nested, hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})))),
- ?assert(hb_message:match(Flat, hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})))).
-```
-
-### nested_conversion_test
-
-```erlang
-nested_conversion_test() ->
- Flat = #{<<"a/b">> => <<"value">>},
- Nested = #{<<"a">> => #{<<"b">> => <<"value">>}},
- Unflattened = hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})),
- Flattened = hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})),
- ?assert(hb_message:match(Nested, Unflattened)),
- ?assert(hb_message:match(Flat, Flattened)).
-```
-
-### multiple_paths_test
-
-```erlang
-multiple_paths_test() ->
- Flat = #{
- <<"x/y">> => <<"1">>,
- <<"x/z">> => <<"2">>,
- <<"a">> => <<"3">>
- },
- Nested = #{
- <<"x">> => #{
- <<"y">> => <<"1">>,
- <<"z">> => <<"2">>
- },
- <<"a">> => <<"3">>
- },
- ?assert(hb_message:match(Nested, hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})))),
- ?assert(hb_message:match(Flat, hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})))).
-```
-
-### path_list_test
-
-```erlang
-path_list_test() ->
- Nested = #{
- <<"x">> => #{
- [<<"y">>, <<"z">>] => #{
- <<"a">> => <<"2">>
- },
- <<"a">> => <<"2">>
- }
- },
- Flat = hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})),
- lists:foreach(
- fun(Key) ->
- ?assert(not lists:member($\n, binary_to_list(Key)))
- end,
- hb_maps:keys(Flat, #{})
- ).
-```
-
-### binary_passthrough_test
-
-```erlang
-binary_passthrough_test() ->
- Bin = <<"raw binary">>,
- ?assertEqual(Bin, hb_util:ok(dev_codec_flat:from(Bin, #{}, #{}))),
- ?assertEqual(Bin, hb_util:ok(dev_codec_flat:to(Bin, #{}, #{}))).
-```
-
-### deep_nesting_test
-
-```erlang
-deep_nesting_test() ->
- Flat = #{<<"a/b/c/d">> => <<"deep">>},
- Nested = #{<<"a">> => #{<<"b">> => #{<<"c">> => #{<<"d">> => <<"deep">>}}}},
- Unflattened = hb_util:ok(dev_codec_flat:from(Flat, #{}, #{})),
- Flattened = hb_util:ok(dev_codec_flat:to(Nested, #{}, #{})),
- ?assert(hb_message:match(Nested, Unflattened)),
- ?assert(hb_message:match(Flat, Flattened)).
-```
-
-### empty_map_test
-
-```erlang
-empty_map_test() ->
- ?assertEqual(#{}, hb_util:ok(dev_codec_flat:from(#{}, #{}, #{}))),
-```
-
----
-
-*Generated from [dev_codec_flat.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_flat.erl)*
diff --git a/docs/book/src/dev_codec_http_auth.erl.md b/docs/book/src/dev_codec_http_auth.erl.md
deleted file mode 100644
index 50b677225..000000000
--- a/docs/book/src/dev_codec_http_auth.erl.md
+++ /dev/null
@@ -1,210 +0,0 @@
-# dev_codec_http_auth
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_http_auth.erl)
-
-Implements a two-step authentication process for HTTP requests, using
-the `Basic` authentication scheme. This device is a viable implementation
-of the `generator` interface type employed by `~auth-hook@1.0`, as well as
-the `~message@1.0` commitment scheme interface.
-`http-auth@1.0``s `commit` and `verify` keys proxy to the `~httpsig@1.0`
-secret key HMAC commitment scheme, utilizing a secret key derived from the
-user's authentication information. Callers may also utilize the `generate`
-key directly to derive entropy from HTTP Authorization headers provided by
-the user. If no Authorization header is provided, the `generate` key will
-return a `401 Unauthorized` response, which triggers a recipient`s browser
-to prompt the user for authentication details and resend the request.
-The `generate` key derives secrets for it's users by calling PBKDF2 with
-the user's authentication information. The parameters for the PBKDF2
-algorithm are configurable, and can be specified in the request message:
-
- salt: The salt to use for the PBKDF2 algorithm. Defaults to
- `sha256("constant:ao")`.
- iterations: The number of iterations to use for the PBKDF2 algorithm.
- Defaults to `1,200,000`.
- alg: The hashing algorithm to use with PBKDF2. Defaults to
- `sha256`.
- key-length: The length of the key to derive from PBKDF2. Defaults to
- `64`.
-
-The default iteration count was chosen at two times the recommendation of
-OWASP in 2023 (600,000), and executes at a run rate of ~5-10 key derivations
-per second on modern CPU hardware. Additionally, the default salt was chosen
-such that it is a public constant (needed in order for reproducibility
-between nodes), and hashed in order to provide additional entropy, in
-alignment with RFC 8018, Section 4.1.
-
----
-
-## Exported Functions
-
-- `commit/3`
-- `generate/3`
-- `verify/3`
-
----
-
-### commit
-
-Implements a two-step authentication process for HTTP requests, using
-The default salt to use for the PBKDF2 algorithm. This value must be
-Generate or extract a new secret and commit to the message with the
-
-```erlang
-commit(Base, Req, Opts) ->
- case generate(Base, Req, Opts) of
- {ok, Key} ->
- {ok, CommitRes} =
- dev_codec_httpsig_proxy:commit(
- <<"http-auth@1.0">>,
- Key,
- Base,
- Req,
- Opts
- ),
- ?event({commit_result, CommitRes}),
- {ok, CommitRes};
- {error, Err} ->
- {error, Err}
- end.
-```
-
-### verify
-
-Verify a given `Base` message with a derived `Key` using the
-
-```erlang
-verify(Base, RawReq, Opts) ->
- ?event({verify_invoked, {base, Base}, {req, RawReq}}),
- {ok, Key} = generate(Base, RawReq, Opts),
- ?event({verify_found_key, {key, Key}, {base, Base}, {req, RawReq}}),
- {ok, VerifyRes} =
- dev_codec_httpsig_proxy:verify(
- Key,
- Base,
- RawReq,
- Opts
- ),
- ?event({verify_result, VerifyRes}),
- {ok, VerifyRes}.
-```
-
-### generate
-
-Collect authentication information from the client. If the `raw` flag
-
-```erlang
-generate(_Msg, ReqLink, Opts) when ?IS_LINK(ReqLink) ->
- generate(_Msg, hb_cache:ensure_loaded(ReqLink, Opts), Opts);
-```
-
-### generate
-
-Collect authentication information from the client. If the `raw` flag
-
-```erlang
-generate(_Msg, #{ <<"secret">> := Secret }, _Opts) ->
- {ok, Secret};
-```
-
-### generate
-
-Collect authentication information from the client. If the `raw` flag
-
-```erlang
-generate(_Msg, Req, Opts) ->
- case hb_maps:get(<<"authorization">>, Req, undefined, Opts) of
- <<"Basic ", Auth/binary>> ->
- Decoded = base64:decode(Auth),
- ?event(key_gen, {generated_key, {auth, Auth}, {decoded, Decoded}}),
- case hb_maps:get(<<"raw">>, Req, false, Opts) of
- true -> {ok, Decoded};
- false -> derive_key(Decoded, Req, Opts)
- end;
- undefined ->
- {error,
- #{
- <<"status">> => 401,
- <<"www-authenticate">> => <<"Basic">>,
- <<"details">> => <<"No authorization header provided.">>
- }
- };
- Unrecognized ->
- {error,
- #{
- <<"status">> => 400,
- <<"details">> =>
- <<"Unrecognized authorization header: ", Unrecognized/binary>>
- }
- }
- end.
-```
-
-### derive_key
-
-Derive a key from the authentication information using the PBKDF2
-
-```erlang
-derive_key(Decoded, Req, Opts) ->
- Alg = hb_util:atom(hb_maps:get(<<"alg">>, Req, <<"sha256">>, Opts)),
- Salt =
- hb_maps:get(
- <<"salt">>,
- Req,
- hb_crypto:sha256(?DEFAULT_SALT),
- Opts
- ),
- Iterations = hb_maps:get(<<"iterations">>, Req, 2 * 600_000, Opts),
- KeyLength = hb_maps:get(<<"key-length">>, Req, 64, Opts),
- ?event(key_gen,
- {derive_key,
- {alg, Alg},
- {salt, Salt},
- {iterations, Iterations},
- {key_length, KeyLength}
- }
- ),
- case hb_crypto:pbkdf2(Alg, Decoded, Salt, Iterations, KeyLength) of
- {ok, Key} ->
- EncodedKey = hb_util:encode(Key),
- {ok, EncodedKey};
- {error, Err} ->
- ?event(key_gen,
- {pbkdf2_error,
- {alg, Alg},
- {salt, Salt},
- {iterations, Iterations},
- {key_length, KeyLength},
- {error, Err}
- }
- ),
- {error,
- #{
- <<"status">> => 500,
- <<"details">> => <<"Failed to derive key.">>
- }
- }
- end.
-```
-
-### benchmark_pbkdf2_test
-
-```erlang
-benchmark_pbkdf2_test() ->
- Key = crypto:strong_rand_bytes(32),
- Iterations = 2 * 600_000,
- KeyLength = 32,
- Derivations =
- hb_test_utils:benchmark(
- fun() ->
- hb_crypto:pbkdf2(sha256, Key, <<"salt">>, Iterations, KeyLength)
- end
- ),
- hb_test_utils:benchmark_print(
- <<"Derived">>,
- <<"keys (1.2m iterations each)">>,
- Derivations
-```
-
----
-
-*Generated from [dev_codec_http_auth.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_http_auth.erl)*
diff --git a/docs/book/src/dev_codec_httpsig.erl.md b/docs/book/src/dev_codec_httpsig.erl.md
deleted file mode 100644
index ae5c48ab9..000000000
--- a/docs/book/src/dev_codec_httpsig.erl.md
+++ /dev/null
@@ -1,447 +0,0 @@
-# dev_codec_httpsig
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_codec_httpsig.erl)
-
-This module implements HTTP Message Signatures as described in RFC-9421
-(https://datatracker.ietf.org/doc/html/rfc9421), as an AO-Core device.
-It implements the codec standard (from/1, to/1), as well as the optional
-commitment functions (id/3, sign/3, verify/3). The commitment functions
-are found in this module, while the codec functions are relayed to the
-`dev_codec_httpsig_conv` module.
-
----
-
-## Exported Functions
-
-- `add_content_digest/2`
-- `commit/3`
-- `from/3`
-- `normalize_for_encoding/3`
-- `serialize/2`
-- `serialize/3`
-- `to/3`
-- `verify/3`
-
----
-
-### to
-
-This module implements HTTP Message Signatures as described in RFC-9421
-
-```erlang
-to(Msg, Req, Opts) -> dev_codec_httpsig_conv:to(Msg, Req, Opts).
-```
-
-### from
-
-This module implements HTTP Message Signatures as described in RFC-9421
-Generate the `Opts` to use during AO-Core operations in the codec.
-
-```erlang
-from(Msg, Req, Opts) -> dev_codec_httpsig_conv:from(Msg, Req, Opts).
-```
-
-### opts
-
-This module implements HTTP Message Signatures as described in RFC-9421
-Generate the `Opts` to use during AO-Core operations in the codec.
-
-```erlang
-opts(RawOpts) ->
- RawOpts#{
- hashpath => ignore,
- cache_control => [<<"no-cache">>, <<"no-store">>],
- force_message => false
- }.
-```
-
-### serialize
-
-A helper utility for creating a direct encoding of a HTTPSig message.
-
-```erlang
-serialize(Msg, Opts) -> serialize(Msg, #{}, Opts).
-```
-
-### serialize
-
-A helper utility for creating a direct encoding of a HTTPSig message.
-
-```erlang
-serialize(Msg, #{ <<"format">> := <<"components">> }, Opts) ->
- % Convert to HTTPSig via TABM through calling `hb_message:convert` rather
- % than executing `to/3` directly. This ensures that our responses are
- % normalized.
-```
-
-### serialize
-
-```erlang
-serialize(Msg, _Req, Opts) ->
- % We assume the default format of `binary` if none of the prior clauses
- % match.
-```
-
-### verify
-
-```erlang
-verify(Base, Req, RawOpts) ->
- % A rsa-pss-sha512 commitment is verified by regenerating the signature
- % base and validating against the signature.
-```
-
-### commit
-
-Commit to a message using the HTTP-Signature format. We use the `type`
-
-```erlang
-commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) ->
- commit(Msg, Req#{ <<"type">> => <<"hmac-sha256">> }, Opts);
-```
-
-### commit
-
-Commit to a message using the HTTP-Signature format. We use the `type`
-
-```erlang
-commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) ->
- commit(Msg, Req#{ <<"type">> => <<"rsa-pss-sha512">> }, Opts);
-```
-
-### commit
-
-Commit to a message using the HTTP-Signature format. We use the `type`
-
-```erlang
-commit(MsgToSign, Req = #{ <<"type">> := <<"rsa-pss-sha512">> }, RawOpts) ->
- ?event(
- {generating_rsa_pss_sha512_commitment, {msg, MsgToSign}, {req, Req}}
- ),
- Opts = opts(RawOpts),
- Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts),
- if Wallet =:= no_viable_wallet ->
- throw({cannot_commit, no_viable_wallet, MsgToSign});
- true ->
- ok
- end,
- % Utilize the hashpath, if present, as the tag for the commitment.
-```
-
-### commit
-
-```erlang
-commit(BaseMsg, Req = #{ <<"type">> := <<"hmac-sha256">> }, RawOpts) ->
- % Extract the key material from the request.
-```
-
-### maybe_bundle_tag_commitment
-
-Annotate the commitment with the `bundle` key if the request contains
-
-```erlang
-maybe_bundle_tag_commitment(Commitment, Req, _Opts) ->
- case hb_util:atom(maps:get(<<"bundle">>, Req, false)) of
- true -> Commitment#{ <<"bundle">> => <<"true">> };
- false -> Commitment
- end.
-```
-
-### keys_to_commit
-
-Derive the set of keys to commit to from a `commit` request and a
-
-```erlang
-keys_to_commit(_Base, #{ <<"committed">> := Explicit}, _Opts) ->
- % Case 1: Explicitly provided keys to commit.
-```
-
-### keys_to_commit
-
-```erlang
-keys_to_commit(Base, _Req, Opts) ->
- % Extract the set of committed keys from the message.
-```
-
-### add_content_digest
-
-If the `body` key is present and a binary, replace it with a
-
-```erlang
-add_content_digest(Msg, _Opts) ->
- case maps:get(<<"body">>, Msg, not_found) of
- Body when is_binary(Body) ->
- % Remove the body from the message and add the content-digest,
- % encoded as a structured field.
-```
-
-### normalize_for_encoding
-
-Given a base message and a commitment, derive the message and commitment
-
-```erlang
-normalize_for_encoding(Msg, Commitment, Opts) ->
- % Extract the requested keys to include in the signature base.
-```
-
-### key_present
-
-Calculate if a key or its `+link` TABM variant is present in a message.
-create the signature base that will be signed in order to create the
-
-```erlang
-key_present(Key, Msg) ->
- NormalizedKey = hb_ao:normalize_key(Key),
- maps:is_key(NormalizedKey, Msg)
- orelse maps:is_key(<- M1/Computed when /Pass == 1 -> - Assumes: - M1/priv/wasm/instance - M1/Process - M2/Message - M2/Assignment/Block-Height - Generates: - /wasm/handler - /wasm/params - Side-effects: - Writes the process and message as JSON representations into the - WASM environment. - M1/Computed when M2/Pass == 2 -> - Assumes: - M1/priv/wasm/instance - M2/Results - M2/Process - Generates: - /Results/Outbox - /Results/Data- ---- - -## Exported Functions - -- `compute/3` -- `generate_aos_msg/2` -- `generate_stack/1` -- `generate_stack/2` -- `generate_stack/3` -- `init/3` -- `json_to_message/2` -- `message_to_json_struct/2` - ---- - -### init - -A device that provides a way for WASM execution to interact with -Initialize the device. -On first pass prepare the call, on second pass get the results. - -```erlang -init(M1, _M2, Opts) -> - {ok, hb_ao:set(M1, #{<<"function">> => <<"handle">>}, Opts)}. -``` - -### compute - -A device that provides a way for WASM execution to interact with -Initialize the device. -On first pass prepare the call, on second pass get the results. - -```erlang -compute(M1, M2, Opts) -> - case hb_ao:get(<<"pass">>, M1, Opts) of - 1 -> prep_call(M1, M2, Opts); - 2 -> results(M1, M2, Opts); - _ -> {ok, M1} - end. -``` - -### prep_call - -Prepare the WASM environment for execution by writing the process string and - -```erlang -prep_call(RawM1, RawM2, Opts) -> - M1 = hb_cache:ensure_all_loaded(RawM1, Opts), - M2 = hb_cache:ensure_all_loaded(RawM2, Opts), - ?event({prep_call, M1, M2, Opts}), - Process = hb_ao:get(<<"process">>, M1, Opts#{ hashpath => ignore }), - Message = hb_ao:get(<<"body">>, M2, Opts#{ hashpath => ignore }), - Image = hb_ao:get(<<"process/image">>, M1, Opts), - BlockHeight = hb_ao:get(<<"block-height">>, M2, Opts), - Props = message_to_json_struct(denormalize_message(Message, Opts), Opts), - MsgProps = - Props#{ - <<"Module">> => Image, - <<"Block-Height">> => BlockHeight - }, - MsgJson = hb_json:encode(MsgProps), - ProcessProps = - #{ - <<"Process">> => message_to_json_struct(Process, Opts) - }, - ProcessJson = hb_json:encode(ProcessProps), - env_write(ProcessJson, MsgJson, M1, M2, Opts). -``` - -### denormalize_message - -Normalize a message for AOS-compatibility. - -```erlang -denormalize_message(Message, Opts) -> - NormOwnerMsg = - case hb_message:signers(Message, Opts) of - [] -> Message; - [PrimarySigner|_] -> - {ok, _, Commitment} = hb_message:commitment(PrimarySigner, Message, Opts), - Message#{ - <<"owner">> => hb_util:human_id(PrimarySigner), - <<"signature">> => - hb_ao:get(<<"signature">>, Commitment, <<>>, Opts) - } - end, - NormOwnerMsg#{ - <<"id">> => hb_message:id(Message, all, Opts) - }. -``` - -### message_to_json_struct - -```erlang -message_to_json_struct(RawMsg, Opts) -> - message_to_json_struct(RawMsg, [owner_as_address], Opts). -``` - -### message_to_json_struct - -```erlang -message_to_json_struct(RawMsg, Features, Opts) -> - TABM = - hb_message:convert( - hb_private:reset(RawMsg), - tabm, - Opts - ), - MsgWithoutCommitments = hb_maps:without([<<"commitments">>], TABM, Opts), - ID = hb_message:id(RawMsg, all), - ?event({encoding, {id, ID}, {msg, RawMsg}}), - {Owner, Signature} = - case hb_message:signers(RawMsg, Opts) of - [] -> {<<>>, <<>>}; - [Signer|_] -> - {ok, _, Commitment} = - hb_message:commitment(Signer, RawMsg, Opts), - CommitmentSignature = - hb_ao:get(<<"signature">>, Commitment, <<>>, Opts), - case lists:member(owner_as_address, Features) of - true -> - { - hb_util:native_id(Signer), - CommitmentSignature - }; - false -> - CommitmentOwner = - hb_ao:get_first( - [ - {Commitment, <<"key">>}, - {Commitment, <<"owner">>} - ], - no_signing_public_key_found_in_commitment, - Opts - ), - {CommitmentOwner, CommitmentSignature} - end - end, - Last = - hb_ao:get( - <<"anchor">>, - {as, <<"message@1.0">>, MsgWithoutCommitments}, - <<>>, - Opts - ), - Data = - hb_ao:get( - <<"data">>, - {as, <<"message@1.0">>, MsgWithoutCommitments}, - <<>>, - Opts - ), - Target = - hb_ao:get( - <<"target">>, - {as, <<"message@1.0">>, MsgWithoutCommitments}, - <<>>, - Opts - ), - % Set "From" if From-Process is Tag or set with "Owner" address - From = - hb_ao:get( - <<"from-process">>, - {as, <<"message@1.0">>, MsgWithoutCommitments}, - hb_util:encode(Owner), - Opts - ), - #{ - <<"Id">> => safe_to_id(ID), - % NOTE: In Arweave TXs, these are called "last_tx" - <<"Anchor">> => Last, - % NOTE: When sent to ao "Owner" is the wallet address - <<"Owner">> => hb_util:encode(Owner), - <<"From">> => case ?IS_ID(From) of true -> safe_to_id(From); false -> From end, - <<"Tags">> => prepare_tags(TABM, Opts), - <<"Target">> => safe_to_id(Target), - <<"Data">> => Data, - <<"Signature">> => - case byte_size(Signature) of - 0 -> <<>>; - 512 -> hb_util:encode(Signature); - _ -> Signature - end - }. -``` - -### prepare_tags - -Prepare the tags of a message as a key-value list, for use in the - -```erlang -prepare_tags(Msg, Opts) -> - % Prepare an ANS-104 message for JSON-Struct construction. -``` - -### prepare_header_case_tags - -Convert a message without an `original-tags` field into a list of - -```erlang -prepare_header_case_tags(TABM, Opts) -> - % Prepare a non-ANS-104 message for JSON-Struct construction. -``` - -### json_to_message - -Translates a compute result -- either from a WASM execution using the - -```erlang -json_to_message(JSON, Opts) when is_binary(JSON) -> - json_to_message(hb_json:decode(JSON), Opts); -``` - -### json_to_message - -Translates a compute result -- either from a WASM execution using the - -```erlang -json_to_message(Resp, Opts) when is_map(Resp) -> - {ok, Data, Messages, Patches} = normalize_results(Resp), - Output = - #{ - <<"outbox">> => - hb_maps:from_list( - [ - {MessageNum, preprocess_results(Msg, Opts)} - || - {MessageNum, Msg} <- - lists:zip( - lists:seq(1, length(Messages)), - Messages - ) - ] - ), - <<"patches">> => lists:map(fun(Patch) -> tags_to_map(Patch, Opts) end, Patches), - <<"data">> => Data - }, - {ok, Output}; -``` - -### json_to_message - -Translates a compute result -- either from a WASM execution using the - -```erlang -json_to_message(#{ <<"ok">> := false, <<"error">> := Error }, _Opts) -> - {error, Error}; -``` - -### json_to_message - -Translates a compute result -- either from a WASM execution using the - -```erlang -json_to_message(Other, _Opts) -> - {error, - #{ - <<"error">> => <<"Invalid JSON message input.">>, - <<"received">> => Other - } - }. -``` - -### safe_to_id - -```erlang -safe_to_id(<<>>) -> <<>>; -``` - -### safe_to_id - -```erlang -safe_to_id(ID) -> hb_util:human_id(ID). -``` - -### maybe_list_to_binary - -```erlang -maybe_list_to_binary(List) when is_list(List) -> - list_to_binary(List); -``` - -### maybe_list_to_binary - -```erlang -maybe_list_to_binary(Bin) -> - Bin. -``` - -### header_case_string - -```erlang -header_case_string(Key) -> - NormKey = hb_ao:normalize_key(Key), - Words = string:lexemes(NormKey, "-"), - TitleCaseWords = - lists:map( - fun binary_to_list/1, - lists:map( - fun string:titlecase/1, - Words - ) - ), - TitleCaseKey = list_to_binary(string:join(TitleCaseWords, "-")), - TitleCaseKey. -``` - -### results - -Read the computed results out of the WASM environment, assuming that - -```erlang -results(M1, M2, Opts) -> - Prefix = dev_stack:prefix(M1, M2, Opts), - Type = hb_ao:get(<<"results/", Prefix/binary, "/type">>, M1, Opts), - Proc = hb_ao:get(<<"process">>, M1, Opts), - case hb_ao:normalize_key(Type) of - <<"error">> -> - {error, - hb_ao:set( - M1, - #{ - <<"outbox">> => undefined, - <<"results">> => - #{ - <<"body">> => <<"WASM execution error.">> - } - }, - Opts - ) - }; - <<"ok">> -> - {ok, Str} = env_read(M1, M2, Opts), - try hb_json:decode(Str) of - #{<<"ok">> := true, <<"response">> := Resp} -> - {ok, ProcessedResults} = json_to_message(Resp, Opts), - PostProcessed = postprocess_outbox(ProcessedResults, Proc, Opts), - Out = hb_ao:set( - M1, - <<"results">>, - PostProcessed, - Opts - ), - ?event(debug_iface, {results, {processed, ProcessedResults}, {out, Out}}), - {ok, Out} - catch - _:_ -> - ?event(error, {json_error, Str}), - {error, - hb_ao:set( - M1, - #{ - <<"results/outbox">> => undefined, - <<"results/body">> => - <<"JSON error parsing result output.">> - }, - Opts - ) - } - end - end. -``` - -### env_read - -Read the results out of the execution environment. - -```erlang -env_read(M1, M2, Opts) -> - Prefix = dev_stack:prefix(M1, M2, Opts), - Output = hb_ao:get(<<"results/", Prefix/binary, "/output">>, M1, Opts), - case hb_private:get(<
- `GET /estimate?type=pre|post&body=[...]&request=RequestMessage` - `GET /price?type=pre|post&body=[...]&request=RequestMessage` --The `body` key is used to pass either the request or response messages to the -device. The `type` key is used to specify whether the inquiry is for a request -(pre) or a response (post) object. Requests carry lists of messages that will -be executed, while responses carry the results of the execution. The `price` -key may return `infinity` if the node will not serve a user under any -circumstances. Else, the value returned by the `price` key will be passed to -the ledger device as the `amount` key. -A ledger device should implement the following keys: -
- `POST /credit?message=PaymentMessage&request=RequestMessage` - `POST /charge?amount=PriceMessage&request=RequestMessage` - `GET /balance?request=RequestMessage` --The `type` key is optional and defaults to `pre`. If `type` is set to `post`, -the charge must be applied to the ledger, whereas the `pre` type is used to -check whether the charge would succeed before execution. - ---- - -## Exported Functions - -- `balance/3` -- `request/3` -- `response/3` - ---- - -### request - -The HyperBEAM core payment ledger. This module allows the operator to -Estimate the cost of a transaction and decide whether to proceed with - -```erlang -request(State, Raw, NodeMsg) -> - PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), - LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), - Messages = hb_ao:get(<<"body">>, Raw, NodeMsg#{ hashpath => ignore }), - Request = hb_ao:get(<<"request">>, Raw, NodeMsg), - IsChargable = is_chargable_req(Request, NodeMsg), - ?event(payment, - {preprocess_with_devices, - PricingDevice, - LedgerDevice, - {chargable, IsChargable} - } - ), - case {IsChargable, (PricingDevice =/= false) and (LedgerDevice =/= false)} of - {false, _} -> - ?event(payment, non_chargable_route), - {ok, #{ <<"body">> => Messages }}; - {true, false} -> - ?event(payment, {p4_pre_pricing_response, {error, <<"infinity">>}}), - {ok, #{ <<"body">> => Messages }}; - {true, true} -> - PricingMsg = State#{ <<"device">> => PricingDevice }, - LedgerMsg = State#{ <<"device">> => LedgerDevice }, - PricingReq = #{ - <<"path">> => <<"estimate">>, - <<"request">> => Request, - <<"body">> => Messages - }, - ?event({p4_pricing_request, {devmsg, PricingMsg}, {req, PricingReq}}), - case hb_ao:resolve(PricingMsg, PricingReq, NodeMsg) of - {ok, <<"infinity">>} -> - % The device states that under no circumstances should we - % proceed with the request. -``` - -### response - -Postprocess the request after it has been fulfilled. - -```erlang -response(State, RawResponse, NodeMsg) -> - PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), - LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), - Response = - hb_ao:get( - <<"body">>, - RawResponse, - NodeMsg#{ hashpath => ignore } - ), - Request = hb_ao:get(<<"request">>, RawResponse, NodeMsg), - ?event(payment, {post_processing_with_devices, PricingDevice, LedgerDevice}), - ?event({response_hook, {request, Request}, {response, Response}}), - case ((PricingDevice =/= false) and (LedgerDevice =/= false)) andalso - is_chargable_req(Request, NodeMsg) of - false -> - {ok, #{ <<"body">> => Response }}; - true -> - PricingMsg = State#{ <<"device">> => PricingDevice }, - LedgerMsg = State#{ <<"device">> => LedgerDevice }, - PricingReq = #{ - <<"path">> => <<"price">>, - <<"request">> => Request, - <<"body">> => Response - }, - ?event({post_pricing_request, PricingReq}), - PricingRes = - case hb_ao:resolve(PricingMsg, PricingReq, NodeMsg) of - {error, _Error} -> - % The pricing device is unable to give us a cost for - % the request, so we try to estimate it instead. -``` - -### balance - -Get the balance of a user in the ledger. - -```erlang -balance(_, Req, NodeMsg) -> - case dev_hook:find(<<"request">>, NodeMsg) of - [] -> - {error, <<"No request hook found.">>}; - [Handler] -> - LedgerDevice = - hb_ao:get(<<"ledger-device">>, Handler, false, NodeMsg), - LedgerMsg = Handler#{ <<"device">> => LedgerDevice }, - LedgerReq = #{ - <<"path">> => <<"balance">>, - <<"request">> => Req - }, - ?event({ledger_message, {ledger_msg, LedgerMsg}}), - case hb_ao:resolve(LedgerMsg, LedgerReq, NodeMsg) of - {ok, Balance} -> - {ok, Balance}; - {error, Error} -> - {error, Error} - end - end. -``` - -### is_chargable_req - -The node operator may elect to make certain routes non-chargable, using - -```erlang -is_chargable_req(Req, NodeMsg) -> - NonChargableRoutes = - hb_opts:get( - p4_non_chargable_routes, - ?DEFAULT_NON_CHARGABLE_ROUTES, - NodeMsg - ), - Matches = - dev_router:match( - #{ <<"routes">> => NonChargableRoutes }, - Req, - NodeMsg - ), - ?event( - { - is_chargable, - {non_chargable_routes, NonChargableRoutes}, - {req, Req}, - {matches, Matches} - } - ), - case Matches of - {error, no_matching_route} -> true; - _ -> false - end. -``` - -### test_opts - -```erlang -test_opts(Opts) -> - test_opts(Opts, <<"faff@1.0">>). -``` - -### test_opts - -```erlang -test_opts(Opts, PricingDev) -> - test_opts(Opts, PricingDev, <<"faff@1.0">>). -``` - -### test_opts - -```erlang -test_opts(Opts, PricingDev, LedgerDev) -> - ProcessorMsg = - #{ - <<"device">> => <<"p4@1.0">>, - <<"pricing-device">> => PricingDev, - <<"ledger-device">> => LedgerDev - }, - Opts#{ - on => #{ - <<"request">> => ProcessorMsg, - <<"response">> => ProcessorMsg - } - }. -``` - -### faff_test - -Simple test of p4's capabilities with the `faff@1.0` device. - -```erlang -faff_test() -> - GoodWallet = ar_wallet:new(), - BadWallet = ar_wallet:new(), - Node = hb_http_server:start_node( - test_opts( - #{ - faff_allow_list => - [hb_util:human_id(ar_wallet:to_address(GoodWallet))] - } - ) - ), - Req = #{ - <<"path">> => <<"/greeting">>, - <<"greeting">> => <<"Hello, world!">> - }, - GoodSignedReq = hb_message:commit(Req, GoodWallet), - ?event({req, GoodSignedReq}), - BadSignedReq = hb_message:commit(Req, BadWallet), - ?event({req, BadSignedReq}), - {ok, Res} = hb_http:get(Node, GoodSignedReq, #{}), - ?event(payment, {res, Res}), - ?assertEqual(<<"Hello, world!">>, Res), - ?assertMatch({error, _}, hb_http:get(Node, BadSignedReq, #{})). -``` - -### non_chargable_route_test - -Test that a non-chargable route is not charged for. - -```erlang -non_chargable_route_test() -> - Wallet = ar_wallet:new(), - Processor = - #{ - <<"device">> => <<"p4@1.0">>, - <<"ledger-device">> => <<"simple-pay@1.0">>, - <<"pricing-device">> => <<"simple-pay@1.0">> - }, - Node = hb_http_server:start_node( - #{ - p4_non_chargable_routes => - [ - #{ <<"template">> => <<"/~p4@1.0/balance">> }, - #{ <<"template">> => <<"/~meta@1.0/*/*">> } - ], - on => #{ - <<"request">> => Processor, - <<"response">> => Processor - }, - operator => hb:address() - } - ), - Req = #{ - <<"path">> => <<"/~p4@1.0/balance">> - }, - GoodSignedReq = hb_message:commit(Req, Wallet), - Res = hb_http:get(Node, GoodSignedReq, #{}), - ?event({res1, Res}), - ?assertMatch({ok, 0}, Res), - Req2 = #{ <<"path">> => <<"/~meta@1.0/info/operator">> }, - GoodSignedReq2 = hb_message:commit(Req2, Wallet), - Res2 = hb_http:get(Node, GoodSignedReq2, #{}), - ?event({res2, Res2}), - OperatorAddress = hb_util:human_id(hb:address()), - ?assertEqual({ok, OperatorAddress}, Res2), - Req3 = #{ <<"path">> => <<"/~scheduler@1.0">> }, - BadSignedReq3 = hb_message:commit(Req3, Wallet), - Res3 = hb_http:get(Node, BadSignedReq3, #{}), - ?event({res3, Res3}), - ?assertMatch({error, _}, Res3). -``` - -### hyper_token_ledger_test_ - -Ensure that Lua scripts can be used as pricing and ledger devices. Our - -```erlang -hyper_token_ledger_test_() -> - {timeout, 60, fun hyper_token_ledger/0}. -``` - -### hyper_token_ledger - -```erlang -hyper_token_ledger() -> - % Create the wallets necessary and read the files containing the scripts. -``` - ---- - -*Generated from [dev_p4.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_p4.erl)* diff --git a/docs/book/src/dev_patch.erl.md b/docs/book/src/dev_patch.erl.md deleted file mode 100644 index 7fc36cc2e..000000000 --- a/docs/book/src/dev_patch.erl.md +++ /dev/null @@ -1,288 +0,0 @@ -# dev_patch - -[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_patch.erl) - -A device that can be used to reorganize a message: Moving data from -one path inside it to another. This device's function runs in two modes: -1. When using `all` to move all data at the path given in `from` to the - path given in `to`. -2. When using `patches` to move all submessages in the source to the target, - _if_ they have a `method` key of `PATCH` or a `device` key of `patch@1.0`. -Source and destination paths may be prepended by `base:` or `req:` keys to -indicate that they are relative to either of the message's that the -computation is being performed on. -The search order for finding the source and destination keys is as follows, -where `X` is either `from` or `to`: -1. The `patch-X` key of the execution message. -2. The `X` key of the execution message. -3. The `patch-X` key of the request message. -4. The `X` key of the request message. -Additionally, this device implements the standard computation device keys, -allowing it to be used as an element of an execution stack pipeline, etc. - ---- - -## Exported Functions - -- `all/3` -- `compute/3` -- `init/3` -- `normalize/3` -- `patches/3` -- `snapshot/3` - ---- - -### init - -A device that can be used to reorganize a message: Moving data from -Necessary hooks for compliance with the `execution-device` standard. - -```erlang -init(Msg1, _Msg2, _Opts) -> {ok, Msg1}. -``` - -### normalize - -A device that can be used to reorganize a message: Moving data from -Necessary hooks for compliance with the `execution-device` standard. - -```erlang -normalize(Msg1, _Msg2, _Opts) -> {ok, Msg1}. -``` - -### snapshot - -A device that can be used to reorganize a message: Moving data from -Necessary hooks for compliance with the `execution-device` standard. - -```erlang -snapshot(Msg1, _Msg2, _Opts) -> {ok, Msg1}. -``` - -### compute - -A device that can be used to reorganize a message: Moving data from -Necessary hooks for compliance with the `execution-device` standard. -Get the value found at the `patch-from` key of the message, or the - -```erlang -compute(Msg1, Msg2, Opts) -> patches(Msg1, Msg2, Opts). -``` - -### all - -A device that can be used to reorganize a message: Moving data from -Necessary hooks for compliance with the `execution-device` standard. -Get the value found at the `patch-from` key of the message, or the - -```erlang -all(Msg1, Msg2, Opts) -> - move(all, Msg1, Msg2, Opts). -``` - -### patches - -Find relevant `PATCH` messages in the given source key of the execution - -```erlang -patches(Msg1, Msg2, Opts) -> - move(patches, Msg1, Msg2, Opts). -``` - -### move - -Unified executor for the `all` and `patches` modes. - -```erlang -move(Mode, Msg1, Msg2, Opts) -> - maybe - % Find the input paths. -``` - -### uninitialized_patch_test - -```erlang -uninitialized_patch_test() -> - InitState = #{ - <<"device">> => <<"patch@1.0">>, - <<"results">> => #{ - <<"outbox">> => #{ - <<"1">> => #{ - <<"method">> => <<"PATCH">>, - <<"prices">> => #{ - <<"apple">> => 100, - <<"banana">> => 200 - } - }, - <<"2">> => #{ - <<"method">> => <<"GET">>, - <<"prices">> => #{ - <<"apple">> => 1000 - } - } - } - }, - <<"other-message">> => <<"other-value">>, - <<"patch-to">> => <<"/">>, - <<"patch-from">> => <<"/results/outbox">> - }, - {ok, ResolvedState} = - hb_ao:resolve( - InitState, - <<"compute">>, - #{} - ), - ?event({resolved_state, ResolvedState}), - ?assertEqual( - 100, - hb_ao:get(<<"prices/apple">>, ResolvedState, #{}) - ), - ?assertMatch( - not_found, - hb_ao:get(<<"results/outbox/1">>, ResolvedState, #{}) - ). -``` - -### patch_to_submessage_test - -```erlang -patch_to_submessage_test() -> - InitState = #{ - <<"device">> => <<"patch@1.0">>, - <<"results">> => #{ - <<"outbox">> => #{ - <<"1">> => - hb_message:commit(#{ - <<"method">> => <<"PATCH">>, - <<"prices">> => #{ - <<"apple">> => 100, - <<"banana">> => 200 - } - }, - hb:wallet() - ) - } - }, - <<"state">> => #{ - <<"prices">> => #{ - <<"apple">> => 1000 - } - }, - <<"other-message">> => <<"other-value">>, - <<"patch-to">> => <<"/state">>, - <<"patch-from">> => <<"/results/outbox">> - }, - {ok, ResolvedState} = - hb_ao:resolve( - InitState, - <<"compute">>, - #{} - ), - ?event({resolved_state, ResolvedState}), - ?assertEqual( - 100, - hb_ao:get(<<"state/prices/apple">>, ResolvedState, #{}) - ). -``` - -### all_mode_test - -```erlang -all_mode_test() -> - InitState = #{ - <<"device">> => <<"patch@1.0">>, - <<"input">> => #{ - <<"zones">> => #{ - <<"1">> => #{ - <<"method">> => <<"PATCH">>, - <<"prices">> => #{ - <<"apple">> => 100, - <<"banana">> => 200 - } - }, - <<"2">> => #{ - <<"method">> => <<"GET">>, - <<"prices">> => #{ - <<"orange">> => 300 - } - } - } - }, - <<"state">> => #{ - <<"prices">> => #{ - <<"apple">> => 1000 - } - } - }, - {ok, ResolvedState} = - hb_ao:resolve( - InitState, - #{ - <<"path">> => <<"all">>, - <<"patch-to">> => <<"/state">>, - <<"patch-from">> => <<"/input/zones">> - }, - #{} - ), - ?event({resolved_state, ResolvedState}), - ?assertEqual( - 100, - hb_ao:get(<<"state/1/prices/apple">>, ResolvedState, #{}) - ), - ?assertEqual( - 300, - hb_ao:get(<<"state/2/prices/orange">>, ResolvedState, #{}) - ), - ?assertEqual( - not_found, - hb_ao:get(<<"input/zones">>, ResolvedState, #{}) - ). -``` - -### req_prefix_test - -```erlang -req_prefix_test() -> - BaseMsg = #{ - <<"device">> => <<"patch@1.0">>, - <<"state">> => #{ - <<"prices">> => #{ - <<"apple">> => 1000 - } - } - }, - ReqMsg = #{ - <<"path">> => <<"all">>, - <<"patch-from">> => <<"req:/results/outbox/1">>, - <<"patch-to">> => <<"/state">>, - <<"results">> => #{ - <<"outbox">> => #{ - <<"1">> => #{ - <<"method">> => <<"PATCH">>, - <<"prices">> => #{ - <<"apple">> => 100, - <<"banana">> => 200 - } - } - } - } - }, - {ok, ResolvedState} = hb_ao:resolve(BaseMsg, ReqMsg, #{}), - ?event({resolved_state, ResolvedState}), - ?assertEqual( - 100, - hb_ao:get(<<"state/prices/apple">>, ResolvedState, #{}) - ), - ?assertEqual( - 200, - hb_ao:get(<<"state/prices/banana">>, ResolvedState, #{}) - ), - ?assertEqual( - not_found, - hb_ao:get(<<"results/outbox/1">>, ResolvedState, #{}) -``` - ---- - -*Generated from [dev_patch.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_patch.erl)* diff --git a/docs/book/src/dev_poda.erl.md b/docs/book/src/dev_poda.erl.md deleted file mode 100644 index 27df7947a..000000000 --- a/docs/book/src/dev_poda.erl.md +++ /dev/null @@ -1,298 +0,0 @@ -# dev_poda - -[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_poda.erl) - -A simple exemplar decentralized proof of authority consensus algorithm -A simple exemplar decentralized proof of authority consensus algorithm -for AO processes. This device is split into two flows, spanning three -actions. -Execution flow: -1. Initialization. -2. Validation of incoming messages before execution. -Commitment flow: -1. Adding commitments to results, either on a CU or MU. - ---- - -## Exported Functions - -- `execute/3` -- `init/2` -- `is_user_signed/1` -- `push/3` - ---- - -### init - -A simple exemplar decentralized proof of authority consensus algorithm - -```erlang -init(S, Params) -> - {ok, S, extract_opts(Params)}. -``` - -### extract_opts - -```erlang -extract_opts(Params) -> - Authorities = - lists:filtermap( - fun({<<"authority">>, Addr}) -> {true, Addr}; - (_) -> false end, - Params - ), - {_, RawQuorum} = lists:keyfind(<<"quorum">>, 1, Params), - Quorum = binary_to_integer(RawQuorum), - ?event({poda_authorities, Authorities}), - #{ - authorities => Authorities, - quorum => Quorum - }. -``` - -### execute - -```erlang -execute(Outer = #tx { data = #{ <<"body">> := Msg } }, S = #{ <<"pass">> := 1 }, Opts) -> - case is_user_signed(Msg) of - true -> - {ok, S}; - false -> - case validate(Msg, Opts) of - true -> - ?event({poda_validated, ok}), - % Add the validations to the VFS. -``` - -### execute - -```erlang -execute(_M, S = #{ <<"pass">> := 3, <<"results">> := _Results }, _Opts) -> - {ok, S}; -``` - -### execute - -```erlang -execute(_M, S, _Opts) -> - {ok, S}. -``` - -### validate - -```erlang -validate(Msg, Opts) -> - validate_stage(1, Msg, Opts). -``` - -### validate_stage - -```erlang -validate_stage(1, Msg, Opts) when is_record(Msg, tx) -> - validate_stage(1, Msg#tx.data, Opts); -``` - -### validate_stage - -```erlang -validate_stage(1, #{ <<"commitments">> := Commitments, <<"body">> := Content }, Opts) -> - validate_stage(2, Commitments, Content, Opts); -``` - -### validate_stage - -```erlang -validate_stage(1, _M, _Opts) -> {false, <<"Required PoDA messages missing">>}. -``` - -### validate_stage - -```erlang -validate_stage(2, #tx { data = Commitments }, Content, Opts) -> - validate_stage(2, Commitments, Content, Opts); -``` - -### validate_stage - -```erlang -validate_stage(2, Commitments, Content, Opts) -> - % Ensure that all commitments are valid and signed by a - % trusted authority. -``` - -### validate_stage - -```erlang -validate_stage(3, Content, Commitments, Opts = #{ <<"quorum">> := Quorum }) -> - Validations = - lists:filter( - fun({_, Comm}) -> validate_commitment(Content, Comm, Opts) end, - hb_maps:to_list(Commitments, Opts) - ), - ?event({poda_validations, length(Validations)}), - case length(Validations) >= Quorum of - true -> - ?event({poda_quorum_reached, length(Validations)}), - true; - false -> {false, <<"Not enough validations">>} - end. -``` - -### validate_commitment - -```erlang -validate_commitment(Msg, Comm, Opts) -> - MsgID = hb_util:encode(ar_bundles:id(Msg, unsigned)), - AttSigner = hb_util:encode(ar_bundles:signer(Comm)), - ?event({poda_commitment, {signer, AttSigner, hb_maps:get(authorities, Opts, undefined, Opts)}, {msg_id, MsgID}}), - ValidSigner = lists:member(AttSigner, hb_maps:get(authorities, Opts, undefined, Opts)), - ValidSignature = ar_bundles:verify_item(Comm), - RelevantMsg = ar_bundles:id(Comm, unsigned) == MsgID orelse - (lists:keyfind(<<"commitment-for">>, 1, Comm#tx.tags) - == {<<"commitment-for">>, MsgID}) orelse - ar_bundles:member(ar_bundles:id(Msg, unsigned), Comm), - case ValidSigner and ValidSignature and RelevantMsg of - false -> - ?event({poda_commitment_invalid, - {commitment, ar_bundles:id(Comm, signed)}, - {signer, AttSigner}, - {valid_signer, ValidSigner}, - {valid_signature, ValidSignature}, - {relevant_msg, RelevantMsg}} - ), - false; - true -> true - end. -``` - -### return_error - -```erlang -return_error(S = #{ <<"wallet">> := Wallet }, Reason) -> - ?event({poda_return_error, Reason}), - ?debug_wait(10000), - {skip, S#{ - results => #{ - <<"/outbox">> => - ar_bundles:sign_item( - #tx{ - data = Reason, - tags = [{<<"error">>, <<"PoDA">>}] - }, - Wallet - ) - } - }}. -``` - -### is_user_signed - -Determines if a user committed - -```erlang -is_user_signed(#tx { data = #{ <<"body">> := Msg } }) -> - ?no_prod(use_real_commitment_detection), - lists:keyfind(<<"from-process">>, 1, Msg#tx.tags) == false; -``` - -### is_user_signed - -Determines if a user committed - -```erlang -is_user_signed(_) -> true. -%%% Commitment flow: Adding commitments to results. -``` - -### push - -Hook used by the MU pathway (currently) to add commitments to an - -```erlang -push(_Item, S = #{ <<"results">> := ResultsMsg }, Opts) -> - NewRes = commit_to_results(ResultsMsg, S, Opts), - {ok, S#{ <<"results">> => NewRes }}. -``` - -### commit_to_results - -Hook used by the MU pathway (currently) to add commitments to an - -```erlang -commit_to_results(Msg, S, Opts) -> - case is_map(Msg#tx.data) of - true -> - % Add commitments to the outbox and spawn items. -``` - -### add_commitments - -```erlang -add_commitments(NewMsg, S = #{ <<"assignment">> := Assignment, <<"store">> := _Store, <<"logger">> := _Logger, <<"wallet">> := Wallet }, Opts) -> - Process = find_process(NewMsg, S), - case is_record(Process, tx) andalso lists:member({<<"device">>, <<"PODA">>}, Process#tx.tags) of - true -> - #{ <<"authorities">> := InitAuthorities, <<"quorum">> := Quorum } = - extract_opts(Process#tx.tags), - ?event({poda_push, InitAuthorities, Quorum}), - % Aggregate validations from other nodes. -``` - -### pfiltermap - -Helper function for parallel execution of commitment - -```erlang -pfiltermap(Pred, List) -> - Parent = self(), - Pids = lists:map(fun(X) -> - spawn_monitor(fun() -> - Result = {X, Pred(X)}, - ?event({pfiltermap, sending_result, self()}), - Parent ! {self(), Result} - end) - end, List), - ?event({pfiltermap, waiting_for_results, Pids}), - [ - Res - || - {true, Res} <- - lists:map(fun({Pid, Ref}) -> - receive - {Pid, {_Item, Result}} -> - ?event({pfiltermap, received_result, Pid}), - Result; - % Handle crashes as filterable events - {'DOWN', Ref, process, Pid, _Reason} -> - ?event({pfiltermap, crashed, Pid}), - false; - Other -> - ?event({pfiltermap, unexpected_message, Other}), - false - end - end, Pids) - ]. -``` - -### find_process - -Find the process that this message is targeting, in order to - -```erlang -find_process(Item, #{ <<"logger">> := _Logger, <<"store">> := Store }) -> - case Item#tx.target of - X when X =/= <<>> -> - ?event({poda_find_process, hb_util:id(Item#tx.target)}), - {ok, Proc} = hb_cache:read(Store, hb_util:id(Item#tx.target)), - Proc; - _ -> - case lists:keyfind(<<"type">>, 1, Item#tx.tags) of - {<<"type">>, <<"process">>} -> Item; - _ -> process_not_specified - end -``` - ---- - -*Generated from [dev_poda.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_poda.erl)* diff --git a/docs/book/src/dev_process.erl.md b/docs/book/src/dev_process.erl.md deleted file mode 100644 index 76b2f0283..000000000 --- a/docs/book/src/dev_process.erl.md +++ /dev/null @@ -1,1221 +0,0 @@ -# dev_process - -[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_process.erl) - -This module contains the device implementation of AO processes -in AO-Core. The core functionality of the module is in 'routing' requests -for different functionality (scheduling, computing, and pushing messages) -to the appropriate device. This is achieved by swapping out the device -of the process message with the necessary component in order to run the -execution, then swapping it back before returning. Computation is supported -as a stack of devices, customizable by the user, while the scheduling -device is (by default) a single device. -This allows the devices to share state as needed. Additionally, after each -computation step the device caches the result at a path relative to the -process definition itself, such that the process message's ID can act as an -immutable reference to the process's growing list of interactions. See -`dev_process_cache` for details. -The external API of the device is as follows: -
-GET /ID/Schedule: Returns the messages in the schedule -POST /ID/Schedule: Adds a message to the schedule -GET /ID/Compute/[IDorSlotNum]: Returns the state of the process after - applying a message -GET /ID/Now: Returns the `/Results` key of the latest - computed message --An example process definition will look like this: -
- Device: Process/1.0 - Scheduler-Device: Scheduler/1.0 - Execution-Device: Stack/1.0 - Execution-Stack: "Scheduler/1.0", "Cron/1.0", "WASM/1.0", "PoDA/1.0" - Cron-Frequency: 10-Minutes - WASM-Image: WASMImageID - PoDA: - Device: PoDA/1.0 - Authority: A - Authority: B - Authority: C - Quorum: 2 --Runtime options: - Cache-Frequency: The number of assignments that will be computed - before the full (restorable) state should be cached. - Cache-Keys: A list of the keys that should be cached for all - assignments, in addition to `/Results`. - ---- - -## Exported Functions - -- `as_process/2` -- `as/3` -- `compute/3` -- `dev_test_process/0` -- `do_test_restore/0` -- `ensure_process_key/2` -- `info/1` -- `init/0` -- `now/3` -- `process_id/3` -- `push/3` -- `schedule_aos_call/2` -- `schedule_aos_call/3` -- `schedule/3` -- `slot/3` -- `snapshot/3` -- `test_aos_process/0` -- `test_aos_process/1` -- `test_wasm_process/1` - ---- - -### info - -This module contains the device implementation of AO processes -When the info key is called, we should return the process exports. - -```erlang -info(_Msg1) -> - #{ - worker => fun dev_process_worker:server/3, - grouper => fun dev_process_worker:group/3, - await => fun dev_process_worker:await/5, - excludes => [ - <<"test">>, - <<"init">>, - <<"ping_ping_script">>, - <<"schedule_aos_call">>, - <<"test_aos_process">>, - <<"dev_test_process">>, - <<"test_wasm_process">> - ] - }. -``` - -### as - -Return the process state with the device swapped out for the device - -```erlang -as(RawMsg1, Msg2, Opts) -> - {ok, Msg1} = ensure_loaded(RawMsg1, Msg2, Opts), - Key = - hb_ao:get_first( - [ - {{as, <<"message@1.0">>, Msg2}, <<"as">>}, - {{as, <<"message@1.0">>, Msg2}, <<"as-device">>} - ], - <<"execution">>, - Opts - ), - {ok, - hb_util:deep_merge( - ensure_process_key(Msg1, Opts), - #{ - <<"device">> => - hb_maps:get( - << Key/binary, "-device">>, - Msg1, - default_device(Msg1, Key, Opts), - Opts - ), - % Configure input prefix for proper message routing within the - % device - <<"input-prefix">> => - case hb_maps:get(<<"input-prefix">>, Msg1, not_found, Opts) of - not_found -> <<"process">>; - Prefix -> Prefix - end, - % Configure output prefixes for result organization - <<"output-prefixes">> => - hb_maps:get( - <
- curl /~relay@.1.0/call?method=GET?0.path=https://www.arweave.net/ -- ---- - -## Exported Functions - -- `call/3` -- `cast/3` -- `request/3` - ---- - -### call - -This module implements the relay device, which is responsible for -Execute a `call` request using a node's routes. - -```erlang -call(M1, RawM2, Opts) -> - ?event({relay_call, {m1, M1}, {raw_m2, RawM2}}), - {ok, BaseTarget} = hb_message:find_target(M1, RawM2, Opts), - ?event({relay_call, {message_to_relay, BaseTarget}}), - RelayPath = - hb_ao:get_first( - [ - {M1, <<"path">>}, - {{as, <<"message@1.0">>, BaseTarget}, <<"path">>}, - {RawM2, <<"relay-path">>}, - {M1, <<"relay-path">>} - ], - Opts - ), - RelayDevice = - hb_ao:get_first( - [ - {M1, <<"relay-device">>}, - {{as, <<"message@1.0">>, BaseTarget}, <<"relay-device">>}, - {RawM2, <<"relay-device">>} - ], - Opts - ), - RelayPeer = - hb_ao:get_first( - [ - {M1, <<"peer">>}, - {{as, <<"message@1.0">>, BaseTarget}, <<"peer">>}, - {RawM2, <<"peer">>} - ], - Opts - ), - RelayMethod = - hb_ao:get_first( - [ - {M1, <<"method">>}, - {{as, <<"message@1.0">>, BaseTarget}, <<"method">>}, - {RawM2, <<"relay-method">>}, - {M1, <<"relay-method">>}, - {RawM2, <<"method">>} - ], - Opts - ), - RelayBody = - hb_ao:get_first( - [ - {M1, <<"body">>}, - {{as, <<"message@1.0">>, BaseTarget}, <<"body">>}, - {RawM2, <<"relay-body">>}, - {M1, <<"relay-body">>}, - {RawM2, <<"body">>} - ], - Opts - ), - Commit = - hb_ao:get_first( - [ - {{as, <<"message@1.0">>, BaseTarget}, <<"commit-request">>}, - {RawM2, <<"relay-commit-request">>}, - {M1, <<"relay-commit-request">>}, - {RawM2, <<"commit-request">>}, - {M1, <<"commit-request">>} - ], - false, - Opts - ), - TargetMod1 = - if RelayBody == not_found -> BaseTarget; - true -> BaseTarget#{<<"body">> => RelayBody} - end, - TargetMod2 = - TargetMod1#{ - <<"method">> => RelayMethod, - <<"path">> => RelayPath - }, - TargetMod3 = - case RelayDevice of - not_found -> hb_maps:without([<<"device">>], TargetMod2); - _ -> TargetMod2#{<<"device">> => RelayDevice} - end, - TargetMod4 = - case Commit of - true -> - case hb_opts:get(relay_allow_commit_request, false, Opts) of - true -> - ?event(debug_relay, {recommitting, TargetMod3}, Opts), - Committed = hb_message:commit(TargetMod3, Opts), - ?event(debug_relay, {relay_call, {committed, Committed}}, Opts), - true = hb_message:verify(Committed, all), - Committed; - false -> - throw(relay_commit_request_not_allowed) - end; - false -> TargetMod3 - end, - ?event(debug_relay, {relay_call, {without_http_params, TargetMod3}}), - ?event(debug_relay, {relay_call, {with_http_params, TargetMod4}}), - true = hb_message:verify(TargetMod4), - ?event(debug_relay, {relay_call, {verified, true}}), - Client = - case hb_maps:get(<<"http-client">>, BaseTarget, not_found, Opts) of - not_found -> hb_opts:get(relay_http_client, Opts); - RequestedClient -> RequestedClient - end, - % Let `hb_http:request/2' handle finding the peer and dispatching the - % request, unless the peer is explicitly given. -``` - -### cast - -Execute a request in the same way as `call/3`, but asynchronously. Always -Preprocess a request to check if it should be relayed to a different node. - -```erlang -cast(M1, M2, Opts) -> - spawn(fun() -> call(M1, M2, Opts) end), - {ok, <<"OK">>}. -``` - -### request - -Execute a request in the same way as `call/3`, but asynchronously. Always -Preprocess a request to check if it should be relayed to a different node. - -```erlang -request(_Msg1, Msg2, Opts) -> - {ok, - #{ - <<"body">> => - [ - #{ <<"device">> => <<"relay@1.0">> }, - #{ - <<"path">> => <<"call">>, - <<"target">> => <<"body">>, - <<"body">> => - hb_ao:get(<<"request">>, Msg2, Opts#{ hashpath => ignore }) - } - ] - } - }. -``` - -### call_get_test - -```erlang -call_get_test() -> - application:ensure_all_started([hb]), - {ok, #{<<"body">> := Body}} = - hb_ao:resolve( - #{ - <<"device">> => <<"relay@1.0">>, - <<"method">> => <<"GET">>, - <<"path">> => <<"https://www.google.com/">> - }, - <<"call">>, - #{ protocol => http2 } - ), - ?assertEqual(true, byte_size(Body) > 10_000). -``` - -### relay_nearest_test - -```erlang -relay_nearest_test() -> - Peer1 = hb_http_server:start_node(#{ priv_wallet => W1 = ar_wallet:new() }), - Peer2 = hb_http_server:start_node(#{ priv_wallet => W2 = ar_wallet:new() }), - Address1 = hb_util:human_id(ar_wallet:to_address(W1)), - Address2 = hb_util:human_id(ar_wallet:to_address(W2)), - Peers = [Address1, Address2], - Node = - hb_http_server:start_node(Opts = #{ - store => hb_opts:get(store), - priv_wallet => ar_wallet:new(), - routes => [ - #{ - <<"template">> => <<"/.*">>, - <<"strategy">> => <<"Nearest">>, - <<"nodes">> => [ - #{ - <<"prefix">> => Peer1, - <<"wallet">> => Address1 - }, - #{ - <<"prefix">> => Peer2, - <<"wallet">> => Address2 - } - ] - } - ] - }), - {ok, RelayRes} = - hb_http:get( - Node, - <<"/~relay@1.0/call?relay-path=/~meta@1.0/info">>, - Opts#{ http_only_result => false } - ), - ?event( - {relay_res, - {response, RelayRes}, - {signer, hb_message:signers(RelayRes, Opts)}, - {peers, Peers} - } - ), - HasValidSigner = - lists:any( - fun(Peer) -> - lists:member(Peer, hb_message:signers(RelayRes, Opts)) - end, - Peers - ), - ?assert(HasValidSigner). -``` - -### commit_request_test - -Test that a `relay@1.0/call` correctly commits requests as specified. - -```erlang -commit_request_test() -> - Port = 10000 + rand:uniform(10000), - Wallet = ar_wallet:new(), - Executor = - hb_http_server:start_node( - #{ - port => Port, - force_signed_requests => true - } - ), - Node = - hb_http_server:start_node(#{ - priv_wallet => Wallet, - relay_allow_commit_request => true, - routes => - [ - #{ - <<"template">> => <<"/test-key">>, - <<"strategy">> => <<"Nearest">>, - <<"nodes">> => [ - #{ - <<"wallet">> => hb_util:human_id(Wallet), - <<"prefix">> => Executor - } - ] - } - ], - on => #{ - <<"request">> => - #{ - <<"device">> => <<"router@1.0">>, - <<"path">> => <<"preprocess">>, - <<"commit-request">> => true - } - } - }), - {ok, Res} = - hb_http:get( - Node, - #{ - <<"path">> => <<"test-key">>, - <<"test-key">> => <<"value">> - }, - #{} - ), - ?event({res, Res}), -``` - ---- - -*Generated from [dev_relay.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_relay.erl)* diff --git a/docs/book/src/dev_router.erl.md b/docs/book/src/dev_router.erl.md deleted file mode 100644 index 5249f8434..000000000 --- a/docs/book/src/dev_router.erl.md +++ /dev/null @@ -1,1450 +0,0 @@ -# dev_router - -[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/dev_router.erl) - -A device that routes outbound messages from the node to their -appropriate network recipients via HTTP. All messages are initially -routed to a single process per node, which then load-balances them -between downstream workers that perform the actual requests. -The routes for the router are defined in the `routes` key of the `Opts`, -as a precidence-ordered list of maps. The first map that matches the -message will be used to determine the route. -Multiple nodes can be specified as viable for a single route, with the -`Choose` key determining how many nodes to choose from the list (defaulting -to 1). The `Strategy` key determines the load distribution strategy, -which can be one of `Random`, `By-Base`, or `Nearest`. The route may also -define additional parallel execution parameters, which are used by the -`hb_http` module to manage control of requests. -The structure of the routes should be as follows: -
- Node?: The node to route the message to. - Nodes?: A list of nodes to route the message to. - Strategy?: The load distribution strategy to use. - Choose?: The number of nodes to choose from the list. - Template?: A message template to match the message against, either as a - map or a path regex. -- ---- - -## Exported Functions - -- `info/1` -- `info/3` -- `match/3` -- `preprocess/3` -- `register/3` -- `route/2` -- `route/3` -- `routes/3` - ---- - -### info - -A device that routes outbound messages from the node to their -Exported function for getting device info, controls which functions are - -```erlang -info(_) -> - #{ exports => [info, routes, route, match, register, preprocess] }. -``` - -### info - -HTTP info response providing information about this device - -```erlang -info(_Msg1, _Msg2, _Opts) -> - InfoBody = #{ - <<"description">> => <<"Router device for handling outbound message routing">>, - <<"version">> => <<"1.0">>, - <<"api">> => #{ - <<"info">> => #{ - <<"description">> => <<"Get device info">> - }, - <<"routes">> => #{ - <<"description">> => <<"Get or add routes">>, - <<"method">> => <<"GET or POST">> - }, - <<"route">> => #{ - <<"description">> => <<"Find a route for a message">>, - <<"required_params">> => #{ - <<"route-path">> => <<"Path to route">> - } - }, - <<"match">> => #{ - <<"description">> => <<"Match a message against available routes">> - }, - <<"register">> => #{ - <<"description">> => <<"Register a route with a remote router node">>, - <<"node-message">> => #{ - <<"routes">> => - [ - #{ - <<"registration-peer">> => <<"Location of the router peer">>, - <<"prefix">> => <<"Prefix for the route">>, - <<"price">> => <<"Price for the route">>, - <<"template">> => <<"Template to match the route">> - } - ] - } - }, - <<"preprocess">> => #{ - <<"description">> => <<"Preprocess a request to check if it should be relayed">> - } - } - }, - {ok, InfoBody}. -``` - -### register - -Register function that allows telling the current node to register -Device function that returns all known routes. - -```erlang -register(_M1, M2, Opts) -> - %% Extract all required parameters from options - %% These values will be used to construct the registration message - RouterOpts = hb_opts:get(router_opts, #{}, Opts), - RouterRegMsgs = - case hb_maps:get(<<"offered">>, RouterOpts, #{}, Opts) of - RegList when is_list(RegList) -> RegList; - RegMsg when is_map(RegMsg) -> [RegMsg] - end, - lists:foreach( - fun(RegMsg) -> - RouterNode = - hb_ao:get( - <<"registration-peer">>, - RegMsg, - not_found, - Opts - ), - {ok, SigOpts} = - case hb_ao:get(<<"as">>, M2, not_found, Opts) of - not_found -> {ok, Opts}; - AsID -> hb_opts:as(AsID, Opts) - end, - % Post registration request to the router node - % The message includes our route details and attestation - % for verification - {ok, Res} = - hb_http:post( - RouterNode, - <<"/~router@1.0/routes">>, - hb_message:commit( - #{ - <<"subject">> => <<"self">>, - <<"action">> => <<"register">>, - <<"route">> => RegMsg - }, - SigOpts - ), - Opts - ), - ?event({registered, {msg, M2}, {res, Res}}), - {ok, <<"Route registered.">>} - end, - RouterRegMsgs - ), - {ok, <<"Routes registered.">>}. -``` - -### routes - -Register function that allows telling the current node to register -Device function that returns all known routes. - -```erlang -routes(M1, M2, Opts) -> - ?event({routes_msg, M1, M2}), - Routes = load_routes(Opts), - ?event({routes, Routes}), - case hb_ao:get(<<"method">>, M2, Opts) of - <<"POST">> -> - RouterOpts = hb_opts:get(router_opts, #{}, Opts), - ?event(debug_route_reg, {router_opts, RouterOpts}), - case hb_maps:get(<<"registrar">>, RouterOpts, not_found, Opts) of - not_found -> - % There is no registrar; register if and only if the message - % is signed by an authorized operator. -``` - -### route - -Find the appropriate route for the given message. If we are able to - -```erlang -route(Msg, Opts) -> route(undefined, Msg, Opts). -``` - -### route - -Find the appropriate route for the given message. If we are able to - -```erlang -route(_, Msg, Opts) -> - Routes = load_routes(Opts), - R = match_routes(Msg, Routes, Opts), - ?event({find_route, {msg, Msg}, {routes, Routes}, {res, R}}), - case (R =/= no_matches) andalso hb_ao:get(<<"node">>, R, Opts) of - false -> {error, no_matches}; - Node when is_binary(Node) -> {ok, Node}; - Node when is_map(Node) -> apply_route(Msg, Node, Opts); - not_found -> - ModR = apply_routes(Msg, R, Opts), - case hb_ao:get(<<"strategy">>, R, Opts) of - not_found -> {ok, ModR}; - <<"All">> -> {ok, ModR}; - Strategy -> - ChooseN = hb_ao:get(<<"choose">>, R, 1, Opts), - % Get the first element of the path -- the `base' message - % of the request. -``` - -### load_routes - -Load the current routes for the node. Allows either explicit routes from - -```erlang -load_routes(Opts) -> - RouterOpts = hb_opts:get(router_opts, #{}, Opts), - case hb_maps:get(<<"provider">>, RouterOpts, not_found, Opts) of - not_found -> hb_opts:get(routes, [], Opts); - RoutesProvider -> - ProviderMsgs = hb_singleton:from(RoutesProvider, Opts), - ?event({<<"provider">>, ProviderMsgs}), - case hb_ao:resolve_many(ProviderMsgs, Opts) of - {ok, Routes} -> hb_cache:ensure_all_loaded(Routes, Opts); - {error, Error} -> throw({routes, routes_provider_failed, Error}) - end - end. -``` - -### extract_base - -Extract the base message ID from a request message. Produces a single - -```erlang -extract_base(#{ <<"path">> := Path }, Opts) -> - extract_base(Path, Opts); -``` - -### extract_base - -Extract the base message ID from a request message. Produces a single - -```erlang -extract_base(RawPath, Opts) when is_binary(RawPath) -> - BasePath = hb_path:hd(#{ <<"path">> => RawPath }, Opts), - case ?IS_ID(BasePath) of - true -> BasePath; - false -> - case binary:split(BasePath, [<<"\~">>, <<"?">>, <<"&">>], [global]) of - [BaseMsgID|_] when ?IS_ID(BaseMsgID) -> BaseMsgID; - _ -> hb_crypto:sha256(BasePath) - end - end. -``` - -### apply_routes - -Generate a `uri` key for each node in a route. -Apply a node map's rules for transforming the path of the message. - -```erlang -apply_routes(Msg, R, Opts) -> - Nodes = hb_ao:get(<<"nodes">>, R, Opts), - NodesWithRouteApplied = - lists:map( - fun(N) -> - ?event({apply_route, {msg, Msg}, {node, N}}), - case apply_route(Msg, N, Opts) of - {ok, URI} when is_binary(URI) -> N#{ <<"uri">> => URI }; - {ok, RMsg} -> hb_maps:merge(N, RMsg); - {error, _} -> N - end - end, - hb_util:message_to_ordered_list(Nodes, Opts) - ), - ?event({nodes_after_apply, NodesWithRouteApplied}), - R#{ <<"nodes">> => NodesWithRouteApplied }. -``` - -### apply_route - -Generate a `uri` key for each node in a route. -Apply a node map's rules for transforming the path of the message. - -```erlang -apply_route(Msg, Route, Opts) -> - % LoadedRoute = hb_cache:ensure_all_loaded(Route, Opts), - RouteOpts = hb_maps:get(<<"opts">>, Route, #{}), - {ok, #{ - <<"opts">> => RouteOpts, - <<"uri">> => - hb_util:ok( - do_apply_route( - Msg, - hb_maps:without([<<"opts">>], Route, Opts), - Opts - ) - ) - }}. -``` - -### do_apply_route - -```erlang -do_apply_route(#{ <<"route-path">> := Path }, R, Opts) -> - do_apply_route(#{ <<"path">> => Path }, R, Opts); -``` - -### do_apply_route - -```erlang -do_apply_route(#{ <<"path">> := RawPath }, #{ <<"prefix">> := RawPrefix }, Opts) -> - Path = hb_cache:ensure_loaded(RawPath, Opts), - Prefix = hb_cache:ensure_loaded(RawPrefix, Opts), - {ok, <
-It exposes the following keys for scheduling:
- `#{ method: GET, path: <<"/info">> }` ->
- Returns information about the scheduler.
- `#{ method: GET, path: <<"/slot">> }` -> `slot(Msg1, Msg2, Opts)`
- Returns the current slot for a process.
- `#{ method: GET, path: <<"/schedule">> }` -> `get_schedule(Msg1, Msg2, Opts)`
- Returns the schedule for a process in a cursor-traversable format.
- ` #{ method: POST, path: <<"/schedule">> }` -> `post_schedule(Msg1, Msg2, Opts)`
- Schedules a new message for a process, or starts a new scheduler
- for the given message.
-
-
----
-
-## Exported Functions
-
-- `checkpoint/1`
-- `info/0`
-- `location/3`
-- `next/3`
-- `parse_schedulers/1`
-- `router/4`
-- `schedule/3`
-- `slot/3`
-- `start/0`
-- `status/3`
-- `test_process/0`
-
----
-
-### start
-
-A simple scheduler scheme for AO.
-Helper to ensure that the environment is started.
-
-```erlang
-start() ->
- % We need the rocksdb backend to run for hb_cache module to work
- application:ensure_all_started(hb),
- <-Device -> Stack -Device-Stack/1/Name -> Add-One-Device -Device-Stack/2/Name -> Add-Two-Device --When called with the message: -
-#{ Path = "FuncName", binary => `<<"0">>` }
-
-Will produce the output:
-
-#{ Path = "FuncName", binary => `<<"3">>` }
-{ok, #{ bin => `<<"3">>` }}
-
-In map mode, the stack will run over all the devices in the stack, and
-combine their results into a single message. Each of the devices'
-output values have a key that is the device's name in the `Device-Stack`
-(its number if the stack is a list).
-You can switch between fold and map modes by setting the `Mode` key in the
-`Msg2` to either `Fold` or `Map`, or set it globally for the stack by
-setting the `Mode` key in the `Msg1` message. The key in `Msg2` takes
-precedence over the key in `Msg1`.
-The key that is called upon the device stack is the same key that is used
-upon the devices that are contained within it. For example, in the above
-scenario we resolve FuncName on the stack, leading FuncName to be called on
-Add-One-Device and Add-Two-Device.
-A device stack responds to special statuses upon responses as follows:
- `skip`: Skips the rest of the device stack for the current pass.
- `pass`: Causes the stack to increment its pass number and re-execute
- the stack from the first device, maintaining the state
- accumulated so far. Only available in fold mode.
-In all cases, the device stack will return the accumulated state to the
-caller as the result of the call to the stack.
-The dev_stack adds additional metadata to the message in order to track
-the state of its execution as it progresses through devices. These keys
-are as follows:
- `Stack-Pass`: The number of times the stack has reset and re-executed
- from the first device for the current message.
- `Input-Prefix`: The prefix that the device should use for its outputs
- and inputs.
- `Output-Prefix`: The device that was previously executed.
-All counters used by the stack are initialized to 1.
-Additionally, as implemented in HyperBEAM, the device stack will honor a
-number of options that are passed to it as keys in the message. Each of
-these options is also passed through to the devices contained within the
-stack during execution. These options include:
- `Error-Strategy`: Determines how the stack handles errors from devices.
- See `maybe_error/5` for more information.
- `Allow-Multipass`: Determines whether the stack is allowed to automatically
- re-execute from the first device when the `pass` tag is returned. See
- `maybe_pass/3` for more information.
-Under-the-hood, dev_stack uses a `default` handler to resolve all calls to
-devices, aside `set/2` which it calls itself to mutate the message's `device`
-key in order to change which device is currently being executed. This method
-allows dev_stack to ensure that the message's HashPath is always correct,
-even as it delegates calls to other devices. An example flow for a `dev_stack`
-execution is as follows:
-
- /Msg1/AlicesExcitingKey ->
- dev_stack:execute ->
- /Msg1/Set?device=/Device-Stack/1 ->
- /Msg2/AlicesExcitingKey ->
- /Msg3/Set?device=/Device-Stack/2 ->
- /Msg4/AlicesExcitingKey
- ... ->
- /MsgN/Set?device=[This-Device] ->
- returns {ok, /MsgN+1} ->
- /MsgN+1
-
-In this example, the `device` key is mutated a number of times, but the
-resulting HashPath remains correct and verifiable.
-
----
-
-## Exported Functions
-
-- `generate_append_device/1`
-- `info/2`
-- `input_prefix/3`
-- `output_prefix/3`
-- `prefix/3`
-- `router/4`
-
----
-
-### info
-
-A device that contains a stack of other devices, and manages their
-
-```erlang
-info(Msg, Opts) ->
- hb_maps:merge(
- #{
- handler => fun router/4,
- excludes => [<<"set">>, <<"keys">>]
- },
- case hb_maps:get(<<"stack-keys">>, Msg, not_found, Opts) of
- not_found -> #{};
- StackKeys -> #{ exports => StackKeys }
- end
- ).
-```
-
-### prefix
-
-Return the default prefix for the stack.
-Return the input prefix for the stack.
-
-```erlang
-prefix(Msg1, _Msg2, Opts) ->
- hb_ao:get(<<"output-prefix">>, {as, dev_message, Msg1}, <<"">>, Opts).
-```
-
-### input_prefix
-
-Return the default prefix for the stack.
-Return the input prefix for the stack.
-Return the output prefix for the stack.
-
-```erlang
-input_prefix(Msg1, _Msg2, Opts) ->
- hb_ao:get(<<"input-prefix">>, {as, dev_message, Msg1}, <<"">>, Opts).
-```
-
-### output_prefix
-
-Return the default prefix for the stack.
-Return the input prefix for the stack.
-Return the output prefix for the stack.
-The device stack key router. Sends the request to `resolve_stack`,
-
-```erlang
-output_prefix(Msg1, _Msg2, Opts) ->
- hb_ao:get(<<"output-prefix">>, {as, dev_message, Msg1}, <<"">>, Opts).
-```
-
-### router
-
-Return the default prefix for the stack.
-Return the input prefix for the stack.
-Return the output prefix for the stack.
-The device stack key router. Sends the request to `resolve_stack`,
-
-```erlang
-router(<<"keys">>, Message1, Message2, Opts) ->
- ?event({keys_called, {msg1, Message1}, {msg2, Message2}}),
- dev_message:keys(Message1, Opts);
-```
-
-### router
-
-Return the default prefix for the stack.
-Return the input prefix for the stack.
-Return the output prefix for the stack.
-The device stack key router. Sends the request to `resolve_stack`,
-
-```erlang
-router(Key, Message1, Message2, Opts) ->
- case hb_path:matches(Key, <<"transform">>) of
- true -> transformer_message(Message1, Opts);
- false -> router(Message1, Message2, Opts)
- end.
-```
-
-### router
-
-```erlang
-router(Message1, Message2, Opts) ->
- ?event({router_called, {msg1, Message1}, {msg2, Message2}}),
- Mode =
- case hb_ao:get(<<"mode">>, Message2, not_found, Opts) of
- not_found ->
- hb_ao:get(
- <<"mode">>,
- {as, dev_message, Message1},
- <<"Fold">>,
- Opts
- );
- Msg2Mode -> Msg2Mode
- end,
- case Mode of
- <<"Fold">> -> resolve_fold(Message1, Message2, Opts);
- <<"Map">> -> resolve_map(Message1, Message2, Opts)
- end.
-```
-
-### transformer_message
-
-Return a message which, when given a key, will transform the message
-
-```erlang
-transformer_message(Msg1, Opts) ->
- ?event({creating_transformer, {for, Msg1}}),
- BaseInfo = info(Msg1, Opts),
- {ok,
- Msg1#{
- <<"device">> => #{
- info =>
- fun() ->
- hb_maps:merge(
- BaseInfo,
- #{
- handler =>
- fun(Key, MsgX1) ->
- transform(MsgX1, Key, Opts)
- end
- },
- Opts
- )
- end,
- <<"type">> => <<"stack-transformer">>
- }
- }
- }.
-```
-
-### transform
-
-Return Message1, transformed such that the device named `Key` from the
-
-```erlang
-transform(Msg1, Key, Opts) ->
- % Get the device stack message from Msg1.
-```
-
-### resolve_fold
-
-The main device stack execution engine. See the moduledoc for more
-
-```erlang
-resolve_fold(Message1, Message2, Opts) ->
- {ok, InitDevMsg} = dev_message:get(<<"device">>, Message1, Opts),
- StartingPassValue =
- hb_ao:get(<<"pass">>, {as, dev_message, Message1}, unset, Opts),
- PreparedMessage = hb_ao:set(Message1, <<"pass">>, 1, Opts),
- case resolve_fold(PreparedMessage, Message2, 1, Opts) of
- {ok, Raw} when not is_map(Raw) ->
- {ok, Raw};
- {ok, Result} ->
- dev_message:set(
- Result,
- #{
- <<"device">> => InitDevMsg,
- <<"input-prefix">> =>
- hb_ao:get(
- <<"previous-input-prefix">>,
- {as, dev_message, Result},
- undefined,
- Opts
- ),
- <<"output-prefix">> =>
- hb_ao:get(
- <<"previous-output-prefix">>,
- {as, dev_message, Result},
- undefined,
- Opts
- ),
- <<"device-key">> => unset,
- <<"device-stack-previous">> => unset,
- <<"pass">> => StartingPassValue
- },
- Opts
- );
- Else ->
- Else
- end.
-```
-
-### resolve_fold
-
-```erlang
-resolve_fold(Message1, Message2, DevNum, Opts) ->
- case transform(Message1, DevNum, Opts) of
- {ok, Message3} ->
- ?event({stack_execute, DevNum, {msg1, Message3}, {msg2, Message2}}),
- case hb_ao:resolve(Message3, Message2, Opts) of
- {ok, Message4} when is_map(Message4) ->
- ?event({result, ok, DevNum, Message4}),
- resolve_fold(Message4, Message2, DevNum + 1, Opts);
- {error, not_found} ->
- ?event({skipping_device, not_found, DevNum, Message3}),
- resolve_fold(Message3, Message2, DevNum + 1, Opts);
- {ok, RawResult} ->
- ?event({returning_raw_result, RawResult}),
- {ok, RawResult};
- {skip, Message4} when is_map(Message4) ->
- ?event({result, skip, DevNum, Message4}),
- {ok, Message4};
- {pass, Message4} when is_map(Message4) ->
- ?event({result, pass, {dev, DevNum}, Message4}),
- resolve_fold(
- increment_pass(Message4, Opts),
- Message2,
- 1,
- Opts
- );
- {error, Info} ->
- ?event({result, error, {dev, DevNum}, Info}),
- maybe_error(Message1, Message2, DevNum, Info, Opts);
- Unexpected ->
- ?event({result, unexpected, {dev, DevNum}, Unexpected}),
- maybe_error(
- Message1,
- Message2,
- DevNum,
- {unexpected_result, Unexpected},
- Opts
- )
- end;
- not_found ->
- ?event({execution_complete, DevNum, Message1}),
- {ok, Message1}
- end.
-```
-
-### resolve_map
-
-Map over the devices in the stack, accumulating the output in a single
-
-```erlang
-resolve_map(Message1, Message2, Opts) ->
- ?event({resolving_map, {msg1, Message1}, {msg2, Message2}}),
- DevKeys =
- hb_ao:get(
- <<"device-stack">>,
- {as, dev_message, Message1},
- Opts
- ),
- Res = {ok,
- hb_maps:filtermap(
- fun(Key, _Dev) ->
- {ok, OrigWithDev} = transform(Message1, Key, Opts),
- case hb_ao:resolve(OrigWithDev, Message2, Opts) of
- {ok, Value} -> {true, Value};
- _ -> false
- end
- end,
- hb_maps:without(?AO_CORE_KEYS, hb_ao:normalize_keys(DevKeys, Opts), Opts),
- Opts
- )
- },
- Res.
-```
-
-### increment_pass
-
-Helper to increment the pass number.
-
-```erlang
-increment_pass(Message, Opts) ->
- hb_ao:set(
- Message,
- #{ <<"pass">> => hb_ao:get(<<"pass">>, {as, dev_message, Message}, 1, Opts) + 1 },
- Opts
- ).
-```
-
-### maybe_error
-
-```erlang
-maybe_error(Message1, Message2, DevNum, Info, Opts) ->
- case hb_opts:get(error_strategy, throw, Opts) of
- stop ->
- {error, {stack_call_failed, Message1, Message2, DevNum, Info}};
- throw ->
- erlang:raise(
- error,
- {device_failed,
- {dev_num, DevNum},
- {msg1, Message1},
- {msg2, Message2},
- {info, Info}
- },
- []
- )
- end.
-```
-
-### generate_append_device
-
-```erlang
-generate_append_device(Separator) ->
- generate_append_device(Separator, ok).
-```
-
-### generate_append_device
-
-```erlang
-generate_append_device(Separator, Status) ->
- #{
- append =>
- fun(M1 = #{ <<"pass">> := 3 }, _) ->
- % Stop after 3 passes.
-```
-
-### transform_internal_call_device_test
-
-Test that the transform function can be called correctly internally
-
-```erlang
-transform_internal_call_device_test() ->
- AppendDev = generate_append_device(<<"_">>),
- Msg1 =
- #{
- <<"device">> => <<"stack@1.0">>,
- <<"device-stack">> =>
- #{
- <<"1">> => AppendDev,
- <<"2">> => <<"message@1.0">>
- }
- },
- ?assertMatch(
- <<"message@1.0">>,
- hb_ao:get(
- <<"device">>,
- element(2, transform(Msg1, <<"2">>, #{}))
- )
- ).
-```
-
-### transform_external_call_device_test
-
-Ensure we can generate a transformer message that can be called to
-
-```erlang
-transform_external_call_device_test() ->
- Msg1 = #{
- <<"device">> => <<"stack@1.0">>,
- <<"device-stack">> =>
- #{
- <<"make-cool">> =>
- #{
- info =>
- fun() ->
- #{
- handler =>
- fun(<<"keys">>, MsgX1) ->
- ?event({test_dev_keys_called, MsgX1}),
- {ok, hb_maps:keys(MsgX1, #{})};
- (Key, MsgX1) ->
- {ok, Value} =
- dev_message:get(Key, MsgX1, #{}),
- dev_message:set(
- MsgX1,
- #{ Key =>
- << Value/binary, "-Cool">>
- },
- #{}
- )
- end
- }
- end,
- <<"suffix">> => <<"-Cool">>
- }
- },
- <<"value">> => <<"Super">>
- },
- ?assertMatch(
- {ok, #{ <<"value">> := <<"Super-Cool">> }},
- hb_ao:resolve(Msg1, #{
- <<"path">> => <<"/transform/make-cool/value">>
- }, #{})
- ).
-```
-
-### example_device_for_stack_test
-
-```erlang
-example_device_for_stack_test() ->
- % Test the example device that we use for later stack tests, such that
- % we know that an error later is actually from the stack, and not from
- % the example device.
-```
-
-### simple_stack_execute_test
-
-```erlang
-simple_stack_execute_test() ->
- Msg = #{
- <<"device">> => <<"stack@1.0">>,
- <<"device-stack">> =>
- #{
- <<"1">> => generate_append_device(<<"!D1!">>),
- <<"2">> => generate_append_device(<<"_D2_">>)
- },
- <<"result">> => <<"INIT">>
- },
- ?event({stack_executing, test, {explicit, Msg}}),
- ?assertMatch(
- {ok, #{ <<"result">> := <<"INIT!D1!2_D2_2">> }},
- hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"2">> }, #{})
- ).
-```
-
-### many_devices_test
-
-```erlang
-many_devices_test() ->
- Msg = #{
- <<"device">> => <<"stack@1.0">>,
- <<"device-stack">> =>
- #{
- <<"1">> => generate_append_device(<<"+D1">>),
- <<"2">> => generate_append_device(<<"+D2">>),
- <<"3">> => generate_append_device(<<"+D3">>),
- <<"4">> => generate_append_device(<<"+D4">>),
- <<"5">> => generate_append_device(<<"+D5">>),
- <<"6">> => generate_append_device(<<"+D6">>),
- <<"7">> => generate_append_device(<<"+D7">>),
- <<"8">> => generate_append_device(<<"+D8">>)
- },
- <<"result">> => <<"INIT">>
- },
- ?assertMatch(
- {ok,
- #{
- <<"result">> :=
- <<"INIT+D12+D22+D32+D42+D52+D62+D72+D82">>
- }
- },
- hb_ao:resolve(Msg, #{ <<"path">> => <<"append">>, <<"bin">> => <<"2">> }, #{})
- ).
-```
-
-### benchmark_test
-
-```erlang
-benchmark_test() ->
- BenchTime = 0.3,
- Msg = #{
- <<"device">> => <<"stack@1.0">>,
- <<"device-stack">> =>
- #{
- <<"1">> => generate_append_device(<<"+D1">>),
- <<"2">> => generate_append_device(<<"+D2">>),
- <<"3">> => generate_append_device(<<"+D3">>),
- <<"4">> => generate_append_device(<<"+D4">>),
- <<"5">> => generate_append_device(<<"+D5">>)
- },
- <<"result">> => <<"INIT">>
- },
- Iterations =
- hb_test_utils:benchmark(
- fun() ->
- hb_ao:resolve(Msg,
- #{
- <<"path">> => <<"append">>,
- <<"bin">> => <<"2">>
- },
- #{}
- ),
- {count, 5}
- end,
- BenchTime
- ),
- hb_test_utils:benchmark_print(
- <<"Stack:">>,
- <<"resolutions">>,
- Iterations,
- BenchTime
- ),
- ?assert(Iterations >= 10).
-```
-
-### test_prefix_msg
-
-```erlang
-test_prefix_msg() ->
- Dev = #{
- prefix_set =>
- fun(M1, M2, Opts) ->
- In = input_prefix(M1, M2, Opts),
- Out = output_prefix(M1, M2, Opts),
- Key = hb_ao:get(<<"key">>, M2, Opts),
- Value = hb_ao:get(<- M1/Init -> - Assumes: - M1/process - M1/[Prefix]/image - Generates: - /priv/[Prefix]/instance - /priv/[Prefix]/import-resolver - Side-effects: - Creates a WASM executor loaded in memory of the HyperBEAM node. - M1/Compute -> - Assumes: - M1/priv/[Prefix]/instance - M1/priv/[Prefix]/import-resolver - M1/process - M2/message - M2/message/function OR M1/function - M2/message/parameters OR M1/parameters - Generates: - /results/[Prefix]/type - /results/[Prefix]/output - Side-effects: - Calls the WASM executor with the message and process. - M1/[Prefix]/state -> - Assumes: - M1/priv/[Prefix]/instance - Generates: - Raw binary WASM state -- ---- - -## Exported Functions - -- `cache_wasm_image/1` -- `cache_wasm_image/2` -- `compute/3` -- `import/3` -- `info/2` -- `init/3` -- `instance/3` -- `normalize/3` -- `snapshot/3` -- `terminate/3` - ---- - -### info - -A device that executes a WASM image on messages using the Memory-64 -Export all functions aside the `instance/3` function. - -```erlang -info(_Msg1, _Opts) -> - #{ - excludes => [instance] - }. -``` - -### init - -Boot a WASM image on the image stated in the `process/image` field of - -```erlang -init(M1, M2, Opts) -> - ?event(running_init), - % Where we should read initial parameters from. -``` - -### default_import_resolver - -Take a BEAMR import call and resolve it using `hb_ao`. - -```erlang -default_import_resolver(Msg1, Msg2, Opts) -> - #{ - instance := WASM, - module := Module, - func := Func, - args := Args, - func_sig := Signature - } = Msg2, - Prefix = dev_stack:prefix(Msg1, Msg2, Opts), - {ok, Msg3} = - hb_ao:resolve( - hb_private:set( - Msg1, - #{ <
- DevMod:ExportedFunc : Key resolution functions. All are assumed to be - device keys (thus, present in every message that - uses it) unless specified by `DevMod:info()`. - Each function takes a set of parameters - of the form `DevMod:KeyHandler(Msg1, Msg2, Opts)`. - Each of these arguments can be ommitted if not - needed. Non-exported functions are not assumed - to be device keys. - DevMod:info : Optional. Returns a map of options for the device. All - options are optional and assumed to be the defaults if - not specified. This function can accept a `Message1` as - an argument, allowing it to specify its functionality - based on a specific message if appropriate. - info/exports : Overrides the export list of the Erlang module, such that - only the functions in this list are assumed to be device - keys. Defaults to all of the functions that DevMod - exports in the Erlang environment. - info/excludes : A list of keys that should not be resolved by the device, - despite being present in the Erlang module exports list. - info/handler : A function that should be used to handle _all_ keys for - messages using the device. - info/default : A function that should be used to handle all keys that - are not explicitly implemented by the device. Defaults to - the `dev_message` device, which contains general keys for - interacting with messages. - info/default_mod : A different device module that should be used to - handle all keys that are not explicitly implemented - by the device. Defaults to the `dev_message` device. - info/grouper : A function that returns the concurrency 'group' name for - an execution. Executions with the same group name will - be executed by sending a message to the associated process - and waiting for a response. This allows you to control - concurrency of execution and to allow executions to share - in-memory state as applicable. Default: A derivation of - Msg1+Msg2. This means that concurrent calls for the same - output will lead to only a single execution. - info/worker : A function that should be run as the 'server' loop of - the executor for interactions using the device. -The HyperBEAM resolver also takes a number of runtime options that change -the way that the environment operates: -`update_hashpath`: Whether to add the `Msg2` to `HashPath` for the `Msg3`. - Default: true. -`add_key`: Whether to add the key to the start of the arguments. - Default: `- ---- - -## Exported Functions - -- `deep_set/4` -- `find_exported_function/5` -- `force_message/2` -- `get_first/2` -- `get_first/3` -- `get/2` -- `get/3` -- `get/4` -- `info/2` -- `is_exported/4` -- `keys/1` -- `keys/2` -- `keys/3` -- `load_device/2` -- `message_to_device/2` -- `message_to_fun/3` -- `normalize_key/1` -- `normalize_key/2` -- `normalize_keys/1` -- `normalize_keys/2` -- `remove/2` -- `remove/3` -- `resolve_many/2` -- `resolve/2` -- `resolve/3` -- `set/3` -- `set/4` -- `truncate_args/2` - ---- - -### resolve - -This module is the root of the device call logic of the -Get the value of a message's key by running its associated device - -```erlang -resolve(Path, Opts) when is_binary(Path) -> - resolve(#{ <<"path">> => Path }, Opts); -``` - -### resolve - -This module is the root of the device call logic of the -Get the value of a message's key by running its associated device - -```erlang -resolve(SingletonMsg, _Opts) - when is_map(SingletonMsg), not is_map_key(<<"path">>, SingletonMsg) -> - {error, <<"Attempted to resolve a message without a path.">>}; -``` - -### resolve - -This module is the root of the device call logic of the -Get the value of a message's key by running its associated device - -```erlang -resolve(SingletonMsg, Opts) -> - resolve_many(hb_singleton:from(SingletonMsg, Opts), Opts). -``` - -### resolve - -```erlang -resolve(Msg1, Path, Opts) when not is_map(Path) -> - resolve(Msg1, #{ <<"path">> => Path }, Opts); -``` - -### resolve - -```erlang -resolve(Msg1, Msg2, Opts) -> - PathParts = hb_path:from_message(request, Msg2, Opts), - ?event(ao_core, {stage, 1, prepare_multimessage_resolution, {path_parts, PathParts}}), - MessagesToExec = [ Msg2#{ <<"path">> => Path } || Path <- PathParts ], - ?event(ao_core, {stage, 1, prepare_multimessage_resolution, {messages_to_exec, MessagesToExec}}), - resolve_many([Msg1 | MessagesToExec], Opts). -``` - -### resolve_many - -Resolve a list of messages in sequence. Take the output of the first - -```erlang -resolve_many([ID], Opts) when ?IS_ID(ID) -> - % Note: This case is necessary to place specifically here for two reasons: - % 1. It is not in `do_resolve_many' because we need to handle the case - % where a result from a prior invocation is an ID itself. We should not - % attempt to resolve such IDs further. -``` - -### resolve_many - -```erlang -resolve_many(ListMsg, Opts) when is_map(ListMsg) -> - % We have been given a message rather than a list of messages, so we should - % convert it to a list, assuming that the message is monotonically numbered. -``` - -### resolve_many - -```erlang -resolve_many({as, DevID, Msg}, Opts) -> - subresolve(#{}, DevID, Msg, Opts); -``` - -### resolve_many - -```erlang -resolve_many([{resolve, Subres}], Opts) -> - resolve_many(Subres, Opts); -``` - -### resolve_many - -```erlang -resolve_many(MsgList, Opts) -> - ?event(ao_core, {resolve_many, MsgList}, Opts), - Res = do_resolve_many(MsgList, Opts), - ?event(ao_core, {resolve_many_complete, {res, Res}, {req, MsgList}}, Opts), - Res. -``` - -### do_resolve_many - -```erlang -do_resolve_many([], _Opts) -> - {failure, <<"Attempted to resolve an empty message sequence.">>}; -``` - -### do_resolve_many - -```erlang -do_resolve_many([Msg3], Opts) -> - ?event(ao_core, {stage, 11, resolve_complete, Msg3}), - {ok, hb_cache:ensure_loaded(Msg3, Opts)}; -``` - -### do_resolve_many - -```erlang -do_resolve_many([Msg1, Msg2 | MsgList], Opts) -> - ?event(ao_core, {stage, 0, resolve_many, {msg1, Msg1}, {msg2, Msg2}}), - case resolve_stage(1, Msg1, Msg2, Opts) of - {ok, Msg3} -> - ?event(ao_core, - { - stage, - 13, - resolved_step, - {msg3, Msg3}, - {opts, Opts} - }, - Opts - ), - do_resolve_many([Msg3 | MsgList], Opts); - Res -> - % The result is not a resolvable message. Return it. -``` - -### resolve_stage - -```erlang -resolve_stage(1, Link, Msg2, Opts) when ?IS_LINK(Link) -> - % If the first message is a link, we should load the message and - % continue with the resolution. -``` - -### resolve_stage - -```erlang -resolve_stage(1, Msg1, Link, Opts) when ?IS_LINK(Link) -> - % If the second message is a link, we should load the message and - % continue with the resolution. -``` - -### resolve_stage - -```erlang -resolve_stage(1, {as, DevID, Ref}, Msg2, Opts) when ?IS_ID(Ref) orelse ?IS_LINK(Ref) -> - % Normalize `as' requests with a raw ID or link as the path. Links will be - % loaded in following stages. -``` - -### resolve_stage - -```erlang -resolve_stage(1, {as, DevID, Link}, Msg2, Opts) when ?IS_LINK(Link) -> - % If the first message is an `as' with a link, we should load the message and - % continue with the resolution. -``` - -### resolve_stage - -```erlang -resolve_stage(1, {as, DevID, Raw = #{ <<"path">> := ID }}, Msg2, Opts) when ?IS_ID(ID) -> - % If the first message is an `as' with an ID, we should load the message and - % apply the non-path elements of the sub-request to it. -``` - -### resolve_stage - -```erlang -resolve_stage(1, Raw = {as, DevID, SubReq}, Msg2, Opts) -> - % Set the device of the message to the specified one and resolve the sub-path. -``` - -### resolve_stage - -```erlang -resolve_stage(1, RawMsg1, Msg2Outer = #{ <<"path">> := {as, DevID, Msg2Inner} }, Opts) -> - % Set the device to the specified `DevID' and resolve the message. Merging - % the `Msg2Inner' into the `Msg2Outer' message first. We return the result - % of the sub-resolution directly. -``` - -### resolve_stage - -```erlang -resolve_stage(1, {resolve, Subres}, Msg2, Opts) -> - % If the first message is a `{resolve, Subres}' tuple, we should execute it - % directly, then apply the request to the result. -``` - -### resolve_stage - -```erlang -resolve_stage(1, Msg1, {resolve, Subres}, Opts) -> - % If the second message is a `{resolve, Subresolution}' tuple, we should - % execute the subresolution directly to gain the underlying `Msg2' for - % our execution. We assume that the subresolution is already in a normalized, - % executable form, so we pass it to `resolve_many' for execution. -``` - -### resolve_stage - -```erlang -resolve_stage(1, Msg1, Msg2, Opts) when is_list(Msg1) -> - % Normalize lists to numbered maps (base=1) if necessary. -``` - -### resolve_stage - -```erlang -resolve_stage(1, Msg1, NonMapMsg2, Opts) when not is_map(NonMapMsg2) -> - ?event(ao_core, {stage, 1, path_normalize}), - resolve_stage(1, Msg1, #{ <<"path">> => NonMapMsg2 }, Opts); -``` - -### resolve_stage - -```erlang -resolve_stage(1, RawMsg1, RawMsg2, Opts) -> - % Normalize the path to a private key containing the list of remaining - % keys to resolve. -``` - -### resolve_stage - -```erlang -resolve_stage(2, Msg1, Msg2, Opts) -> - ?event(ao_core, {stage, 2, cache_lookup}, Opts), - % Lookup request in the cache. If we find a result, return it. -``` - -### resolve_stage - -```erlang -resolve_stage(3, Msg1, Msg2, Opts) when not is_map(Msg1) or not is_map(Msg2) -> - % Validation check: If the messages are not maps, we cannot find a key - % in them, so return not_found. -``` - -### resolve_stage - -```erlang -resolve_stage(3, Msg1, Msg2, Opts) -> - ?event(ao_core, {stage, 3, validation_check}, Opts), - % Validation check: Check if the message is valid. -``` - -### resolve_stage - -```erlang -resolve_stage(4, Msg1, Msg2, Opts) -> - ?event(ao_core, {stage, 4, persistent_resolver_lookup}, Opts), - % Persistent-resolver lookup: Search for local (or Distributed - % Erlang cluster) processes that are already performing the execution. -``` - -### resolve_stage - -```erlang -resolve_stage(5, Msg1, Msg2, ExecName, Opts) -> - ?event(ao_core, {stage, 5, device_lookup}, Opts), - % Device lookup: Find the Erlang function that should be utilized to - % execute Msg2 on Msg1. -``` - -### resolve_stage - -```erlang -resolve_stage(6, Func, Msg1, Msg2, ExecName, Opts) -> - ?event(ao_core, {stage, 6, ExecName, execution}, Opts), - % Execution. -``` - -### resolve_stage - -```erlang -resolve_stage(7, Msg1, Msg2, {St, Res}, ExecName, Opts = #{ on := On = #{ <<"step">> := _ }}) -> - ?event(ao_core, {stage, 7, ExecName, executing_step_hook, {on, On}}, Opts), - % If the `step' hook is defined, we execute it. Note: This function clause - % matches directly on the `on' key of the `Opts' map. This is in order to - % remove the expensive lookup check that would otherwise be performed on every - % execution. -``` - -### resolve_stage - -```erlang -resolve_stage(7, Msg1, Msg2, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 7, ExecName, no_step_hook}, Opts), - resolve_stage(8, Msg1, Msg2, Res, ExecName, Opts); -``` - -### resolve_stage - -```erlang -resolve_stage(8, Msg1, Msg2, {ok, {resolve, Sublist}}, ExecName, Opts) -> - ?event(ao_core, {stage, 8, ExecName, subresolve_result}, Opts), - % If the result is a `{resolve, Sublist}' tuple, we need to execute it - % as a sub-resolution. -``` - -### resolve_stage - -```erlang -resolve_stage(8, Msg1, Msg2, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), - resolve_stage(9, Msg1, Msg2, Res, ExecName, Opts); -``` - -### resolve_stage - -```erlang -resolve_stage(9, Msg1, Msg2, {ok, Msg3}, ExecName, Opts) when is_map(Msg3) -> - ?event(ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), - % Cryptographic linking. Now that we have generated the result, we - % need to cryptographically link the output to its input via a hashpath. -``` - -### resolve_stage - -```erlang -resolve_stage(9, Msg1, Msg2, {Status, Msg3}, ExecName, Opts) when is_map(Msg3) -> - ?event(ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), - ?event(hashpath, {resetting_hashpath_msg3, {msg1, Msg1}, {msg2, Msg2}, {opts, Opts}}), - % Skip cryptographic linking and reset the hashpath if the result is abnormal. -``` - -### resolve_stage - -```erlang -resolve_stage(9, Msg1, Msg2, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), - % Skip cryptographic linking and continue if we don't have a map that can have - % a hashpath at all. -``` - -### resolve_stage - -```erlang -resolve_stage(10, Msg1, Msg2, {ok, Msg3}, ExecName, Opts) -> - ?event(ao_core, {stage, 10, ExecName, result_caching}, Opts), - % Result caching: Optionally, cache the result of the computation locally. -``` - -### resolve_stage - -```erlang -resolve_stage(10, Msg1, Msg2, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), - % Skip result caching if the result is abnormal. -``` - -### resolve_stage - -```erlang -resolve_stage(11, Msg1, Msg2, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 11, ExecName}, Opts), - % Notify processes that requested the resolution while we were executing and - % unregister ourselves from the group. -``` - -### resolve_stage - -```erlang -resolve_stage(12, _Msg1, _Msg2, {ok, Msg3} = Res, ExecName, Opts) -> - ?event(ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), - % Check if we should spawn a worker for the current execution - case {is_map(Msg3), hb_opts:get(spawn_worker, false, Opts#{ prefer => local })} of - {A, B} when (A == false) or (B == false) -> - Res; - {_, _} -> - % Spawn a worker for the current execution - WorkerPID = hb_persistent:start_worker(ExecName, Msg3, Opts), - hb_persistent:forward_work(WorkerPID, Opts), - Res - end; -``` - -### resolve_stage - -```erlang -resolve_stage(12, _Msg1, _Msg2, OtherRes, ExecName, Opts) -> - ?event(ao_core, {stage, 12, ExecName, abnormal_status_skip_spawning}, Opts), - OtherRes. -``` - -### subresolve - -Execute a sub-resolution. - -```erlang -subresolve(RawMsg1, DevID, ReqPath, Opts) when is_binary(ReqPath) -> - % If the request is a binary, we assume that it is a path. -``` - -### subresolve - -```erlang -subresolve(RawMsg1, DevID, Req, Opts) -> - % First, ensure that the message is loaded from the cache. -``` - -### maybe_profiled_apply - -If the `AO_PROFILING` macro is defined (set by building/launching with - -```erlang -maybe_profiled_apply(Func, Args, _Msg1, _Msg2, _Opts) -> - apply(Func, Args). -``` - -### maybe_profiled_apply - -```erlang -maybe_profiled_apply(Func, Args, Msg1, Msg2, Opts) -> - CallStack = erlang:get(ao_stack), - ?event(ao_trace, - {profiling_apply, - {func, Func}, - {args, Args}, - {call_stack, CallStack} - } - ), - Key = - case hb_maps:get(<<"device">>, Msg1, undefined, Opts) of - undefined -> - hb_util:bin(erlang:fun_to_list(Func)); - Device -> - case hb_maps:get(<<"path">>, Msg2, undefined, Opts) of - undefined -> - hb_util:bin(erlang:fun_to_list(Func)); - Path -> - MethodStr = - case hb_maps:get(<<"method">>, Msg2, undefined, Opts) of - undefined -> <<"">>; - <<"GET">> -> <<"">>; - Method -> <<"<", Method/binary, ">">> - end, - << - (hb_util:bin(Device))/binary, - "/", - MethodStr/binary, - (hb_util:bin(Path))/binary - >> - end - end, - put( - ao_stack, - case CallStack of - undefined -> [Key]; - Stack -> [Key | Stack] - end - ), - {ExecMicroSecs, Res} = timer:tc(fun() -> apply(Func, Args) end), - put(ao_stack, CallStack), - hb_event:increment(<<"ao-call-counts">>, Key, Opts), - hb_event:increment(<<"ao-total-durations">>, Key, Opts, ExecMicroSecs), - case CallStack of - undefined -> ok; - [Caller|_] -> - hb_event:increment( - <<"ao-callers:", Key/binary>>, - hb_util:bin( - [ - <<"duration:">>, - Caller - ] - ), - Opts, - ExecMicroSecs - ), - hb_event:increment( - <<"ao-callers:", Key/binary>>, - hb_util:bin( - [ - <<"calls:">>, - Caller - ]), - Opts - ) - end, - Res. -``` - -### ensure_message_loaded - -Ensure that a message is loaded from the cache if it is an ID, or - -```erlang -ensure_message_loaded(MsgID, Opts) when ?IS_ID(MsgID) -> - case hb_cache:read(MsgID, Opts) of - {ok, LoadedMsg} -> - LoadedMsg; - not_found -> - throw({necessary_message_not_found, <<"/">>, MsgID}) - end; -``` - -### ensure_message_loaded - -Ensure that a message is loaded from the cache if it is an ID, or - -```erlang -ensure_message_loaded(MsgLink, Opts) when ?IS_LINK(MsgLink) -> - hb_cache:ensure_loaded(MsgLink, Opts); -``` - -### ensure_message_loaded - -Ensure that a message is loaded from the cache if it is an ID, or - -```erlang -ensure_message_loaded(Msg, _Opts) -> - Msg. -``` - -### error_invalid_message - -Catch all return if the message is invalid. - -```erlang -error_invalid_message(Msg1, Msg2, Opts) -> - ?event( - ao_core, - {error, {type, invalid_message}, - {msg1, Msg1}, - {msg2, Msg2}, - {opts, Opts} - }, - Opts - ), - { - error, - #{ - <<"status">> => 400, - <<"body">> => <<"Request contains non-verifiable message.">> - } - }. -``` - -### error_infinite - -Catch all return if we are in an infinite loop. - -```erlang -error_infinite(Msg1, Msg2, Opts) -> - ?event( - ao_core, - {error, {type, infinite_recursion}, - {msg1, Msg1}, - {msg2, Msg2}, - {opts, Opts} - }, - Opts - ), - ?trace(), - { - error, - #{ - <<"status">> => 508, - <<"body">> => <<"Request creates infinite recursion.">> - } - }. -``` - -### error_invalid_intermediate_status - -```erlang -error_invalid_intermediate_status(Msg1, Msg2, Msg3, RemainingPath, Opts) -> - ?event( - ao_core, - {error, {type, invalid_intermediate_status}, - {msg2, Msg2}, - {msg3, Msg3}, - {remaining_path, RemainingPath}, - {opts, Opts} - }, - Opts - ), - ?event(ao_result, - {intermediate_failure, {msg1, Msg1}, - {msg2, Msg2}, {msg3, Msg3}, - {remaining_path, RemainingPath}, {opts, Opts}}), - { - error, - #{ - <<"status">> => 422, - <<"body">> => Msg3, - <<"key">> => hb_maps:get(<<"path">>, Msg2, <<"Key unknown.">>, Opts), - <<"remaining-path">> => RemainingPath - } - }. -``` - -### error_execution - -Handle an error in a device call. - -```erlang -error_execution(ExecGroup, Msg2, Whence, {Class, Exception, Stacktrace}, Opts) -> - Error = {error, Whence, {Class, Exception, Stacktrace}}, - hb_persistent:unregister_notify(ExecGroup, Msg2, Error, Opts), - ?event(ao_core, {handle_error, Error, {opts, Opts}}, Opts), - case hb_opts:get(error_strategy, throw, Opts) of - throw -> erlang:raise(Class, Exception, Stacktrace); - _ -> Error - end. -``` - -### maybe_force_message - -Force the result of a device call into a message if the result is not - -```erlang -maybe_force_message({Status, Res}, Opts) -> - case hb_opts:get(force_message, false, Opts) of - true -> force_message({Status, Res}, Opts); - false -> {Status, Res} - end; -``` - -### maybe_force_message - -Force the result of a device call into a message if the result is not - -```erlang -maybe_force_message(Res, Opts) -> - maybe_force_message({ok, Res}, Opts). -``` - -### force_message - -```erlang -force_message({Status, Res}, Opts) when is_list(Res) -> - force_message({Status, normalize_keys(Res, Opts)}, Opts); -``` - -### force_message - -```erlang -force_message({Status, Subres = {resolve, _}}, _Opts) -> - {Status, Subres}; -``` - -### force_message - -```erlang -force_message({Status, Literal}, _Opts) when not is_map(Literal) -> - ?event({force_message_from_literal, Literal}), - {Status, #{ <<"ao-result">> => <<"body">>, <<"body">> => Literal }}; -``` - -### force_message - -```erlang -force_message({Status, M = #{ <<"status">> := Status, <<"body">> := Body }}, _Opts) - when map_size(M) == 2 -> - ?event({force_message_from_literal_with_status, M}), - {Status, #{ - <<"status">> => Status, - <<"ao-result">> => <<"body">>, - <<"body">> => Body - }}; -``` - -### force_message - -```erlang -force_message({Status, Map}, _Opts) -> - ?event({force_message_from_map, Map}), - {Status, Map}. -``` - -### get - -Shortcut for resolving a key in a message without its status if it is - -```erlang -get(Path, Msg) -> - get(Path, Msg, #{}). -``` - -### get - -```erlang -get(Path, Msg, Opts) -> - get(Path, Msg, not_found, Opts). -``` - -### get - -```erlang -get(Path, {as, Device, Msg}, Default, Opts) -> - get( - Path, - set( - Msg, - #{ <<"device">> => Device }, - internal_opts(Opts) - ), - Default, - Opts - ); -``` - -### get - -```erlang -get(Path, Msg, Default, Opts) -> - case resolve(Msg, #{ <<"path">> => Path }, Opts#{ spawn_worker => false }) of - {ok, Value} -> Value; - {error, _} -> Default - end. -``` - -### get_first - -take a sequence of base messages and paths, then return the value of the - -```erlang -get_first(Paths, Opts) -> get_first(Paths, not_found, Opts). -``` - -### get_first - -take a sequence of base messages and paths, then return the value of the - -```erlang -get_first([], Default, _Opts) -> Default; -``` - -### get_first - -take a sequence of base messages and paths, then return the value of the - -```erlang -get_first([{Base, Path}|Msgs], Default, Opts) -> - case get(Path, Base, Opts) of - not_found -> get_first(Msgs, Default, Opts); - Value -> Value - end. -``` - -### keys - -Shortcut to get the list of keys from a message. - -```erlang -keys(Msg) -> keys(Msg, #{}). -``` - -### keys - -Shortcut to get the list of keys from a message. - -```erlang -keys(Msg, Opts) -> keys(Msg, Opts, keep). -``` - -### keys - -Shortcut to get the list of keys from a message. - -```erlang -keys(Msg, Opts, keep) -> - % There is quite a lot of AO-Core-specific machinery here. We: - % 1. `get' the keys from the message, via AO-Core in order to trigger the - % `keys' function on its device. -``` - -### keys - -```erlang -keys(Msg, Opts, remove) -> - lists:filter( - fun(Key) -> not lists:member(Key, ?AO_CORE_KEYS) end, - keys(Msg, Opts, keep) - ). -``` - -### set - -Shortcut for setting a key in the message using its underlying device. - -```erlang -set(RawMsg1, RawMsg2, Opts) when is_map(RawMsg2) -> - Msg1 = normalize_keys(RawMsg1, Opts), - Msg2 = hb_maps:without([<<"hashpath">>, <<"priv">>], normalize_keys(RawMsg2, Opts), Opts), - ?event(ao_internal, {set_called, {msg1, Msg1}, {msg2, Msg2}}, Opts), - % Get the next key to set. -``` - -### set - -```erlang -set(Msg1, Key, Value, Opts) -> - % For an individual key, we run deep_set with the key as the path. -``` - -### deep_set - -Recursively search a map, resolving keys, and set the value of the key - -```erlang -deep_set(Msg, [], Value, Opts) when is_map(Msg) or is_list(Msg) -> - device_set(Msg, <<"/">>, Value, Opts); -``` - -### deep_set - -Recursively search a map, resolving keys, and set the value of the key - -```erlang -deep_set(_Msg, [], Value, _Opts) -> - Value; -``` - -### deep_set - -Recursively search a map, resolving keys, and set the value of the key - -```erlang -deep_set(Msg, [Key], Value, Opts) -> - device_set(Msg, Key, Value, Opts); -``` - -### deep_set - -Recursively search a map, resolving keys, and set the value of the key - -```erlang -deep_set(Msg, [Key|Rest], Value, Opts) -> - case resolve(Msg, Key, Opts) of - {ok, SubMsg} -> - ?event( - {traversing_deeper_to_set, - {current_key, Key}, - {current_value, SubMsg}, - {rest, Rest} - } - ), - Res = device_set(Msg, Key, deep_set(SubMsg, Rest, Value, Opts), <<"explicit">>, Opts), - ?event({deep_set_result, {msg, Msg}, {key, Key}, {res, Res}}), - Res; - _ -> - ?event( - {creating_new_map, - {current_key, Key}, - {rest, Rest} - } - ), - Msg#{ Key => deep_set(#{}, Rest, Value, Opts) } - end. -``` - -### device_set - -Call the device's `set` function. - -```erlang -device_set(Msg, Key, Value, Opts) -> - device_set(Msg, Key, Value, <<"deep">>, Opts). -``` - -### device_set - -Call the device's `set` function. - -```erlang -device_set(Msg, Key, Value, Mode, Opts) -> - ReqWithoutMode = - case Key of - <<"path">> -> - #{ <<"path">> => <<"set_path">>, <<"value">> => Value }; - <<"/">> when is_map(Value) -> - % The value is a map and it is to be `set' at the root of the - % message. Subsequently, we call the device's `set' function - % with all of the keys found in the message, leading it to be - % merged into the message. -``` - -### remove - -Remove a key from a message, using its underlying device. - -```erlang -remove(Msg, Key) -> remove(Msg, Key, #{}). -``` - -### remove - -Remove a key from a message, using its underlying device. - -```erlang -remove(Msg, Key, Opts) -> - hb_util:ok( - resolve( - Msg, - #{ <<"path">> => <<"remove">>, <<"item">> => Key }, - internal_opts(Opts) - ), - Opts - ). -``` - -### truncate_args - -Truncate the arguments of a function to the number of arguments it - -```erlang -truncate_args(Fun, Args) -> - {arity, Arity} = erlang:fun_info(Fun, arity), - lists:sublist(Args, Arity). -``` - -### message_to_fun - -Calculate the Erlang function that should be called to get a value for - -```erlang -message_to_fun(Msg, Key, Opts) -> - % Get the device module from the message. -``` - -### message_to_device - -Extract the device module from a message. - -```erlang -message_to_device(Msg, Opts) -> - case dev_message:get(<<"device">>, Msg, Opts) of - {error, not_found} -> - % The message does not specify a device, so we use the default device. -``` - -### info_handler_to_fun - -Parse a handler key given by a device's `info`. - -```erlang -info_handler_to_fun(Handler, _Msg, _Key, _Opts) when is_function(Handler) -> - {add_key, Handler}; -``` - -### info_handler_to_fun - -Parse a handler key given by a device's `info`. - -```erlang -info_handler_to_fun(HandlerMap, Msg, Key, Opts) -> - case hb_maps:find(excludes, HandlerMap, Opts) of - {ok, Exclude} -> - case lists:member(Key, Exclude) of - true -> - {ok, MsgWithoutDevice} = - dev_message:remove(Msg, #{ item => device }, Opts), - message_to_fun( - MsgWithoutDevice#{ <<"device">> => default_module() }, - Key, - Opts - ); - false -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} - end; - error -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} - end. -``` - -### find_exported_function - -Find the function with the highest arity that has the given name, if it - -```erlang -find_exported_function(Msg, Dev, Key, MaxArity, Opts) when is_map(Dev) -> - case hb_maps:get(normalize_key(Key), normalize_keys(Dev, Opts), not_found, Opts) of - not_found -> not_found; - Fun when is_function(Fun) -> - case erlang:fun_info(Fun, arity) of - {arity, Arity} when Arity =< MaxArity -> - case is_exported(Msg, Dev, Key, Opts) of - true -> {ok, Fun}; - false -> not_found - end; - _ -> not_found - end - end; -``` - -### find_exported_function - -Find the function with the highest arity that has the given name, if it - -```erlang -find_exported_function(_Msg, _Mod, _Key, Arity, _Opts) when Arity < 0 -> - not_found; -``` - -### find_exported_function - -Find the function with the highest arity that has the given name, if it - -```erlang -find_exported_function(Msg, Mod, Key, Arity, Opts) when not is_atom(Key) -> - try hb_util:key_to_atom(Key, false) of - KeyAtom -> find_exported_function(Msg, Mod, KeyAtom, Arity, Opts) - catch _:_ -> not_found - end; -``` - -### find_exported_function - -Find the function with the highest arity that has the given name, if it - -```erlang -find_exported_function(Msg, Mod, Key, Arity, Opts) -> - case erlang:function_exported(Mod, Key, Arity) of - true -> - case is_exported(Msg, Mod, Key, Opts) of - true -> {ok, fun Mod:Key/Arity}; - false -> not_found - end; - false -> - find_exported_function(Msg, Mod, Key, Arity - 1, Opts) - end. -``` - -### is_exported - -Check if a device is guarding a key via its `exports` list. Defaults to - -```erlang -is_exported(_Msg, _Dev, info, _Opts) -> true; -``` - -### is_exported - -Check if a device is guarding a key via its `exports` list. Defaults to - -```erlang -is_exported(Msg, Dev, Key, Opts) -> - is_exported(info(Dev, Msg, Opts), Key, Opts). -``` - -### is_exported - -```erlang -is_exported(_, info, _Opts) -> true; -``` - -### is_exported - -```erlang -is_exported(Info = #{ excludes := Excludes }, Key, Opts) -> - case lists:member(normalize_key(Key), lists:map(fun normalize_key/1, Excludes)) of - true -> false; - false -> is_exported(hb_maps:remove(excludes, Info, Opts), Key, Opts) - end; -``` - -### is_exported - -```erlang -is_exported(#{ exports := Exports }, Key, _Opts) -> - lists:member(normalize_key(Key), lists:map(fun normalize_key/1, Exports)); -``` - -### is_exported - -Convert a key to a binary in normalized form. - -```erlang -is_exported(_Info, _Key, _Opts) -> true. -``` - -### normalize_key - -Convert a key to a binary in normalized form. - -```erlang -normalize_key(Key) -> normalize_key(Key, #{}). -``` - -### normalize_key - -Convert a key to a binary in normalized form. - -```erlang -normalize_key(Key, _Opts) when is_binary(Key) -> Key; -``` - -### normalize_key - -Convert a key to a binary in normalized form. - -```erlang -normalize_key(Key, _Opts) when is_atom(Key) -> atom_to_binary(Key); -``` - -### normalize_key - -Convert a key to a binary in normalized form. - -```erlang -normalize_key(Key, _Opts) when is_integer(Key) -> integer_to_binary(Key); -``` - -### normalize_key - -Convert a key to a binary in normalized form. - -```erlang -normalize_key(Key, _Opts) when is_list(Key) -> - case hb_util:is_string_list(Key) of - true -> normalize_key(list_to_binary(Key)); - false -> - iolist_to_binary( - lists:join( - <<"/">>, - lists:map(fun normalize_key/1, Key) - ) - ) - end. -``` - -### normalize_keys - -Ensure that a message is processable by the AO-Core resolver: No lists. - -```erlang -normalize_keys(Msg) -> normalize_keys(Msg, #{}). -``` - -### normalize_keys - -Ensure that a message is processable by the AO-Core resolver: No lists. - -```erlang -normalize_keys(Msg1, Opts) when is_list(Msg1) -> - normalize_keys( - hb_maps:from_list( - lists:zip( - lists:seq(1, length(Msg1)), - Msg1 - ) - ), - Opts - ); -``` - -### normalize_keys - -Ensure that a message is processable by the AO-Core resolver: No lists. - -```erlang -normalize_keys(Map, Opts) when is_map(Map) -> - hb_maps:from_list( - lists:map( - fun({Key, Value}) when is_map(Value) -> - {hb_ao:normalize_key(Key), Value}; - ({Key, Value}) -> - {hb_ao:normalize_key(Key), Value} - end, - hb_maps:to_list(Map, Opts) - ) - ); -``` - -### normalize_keys - -Ensure that a message is processable by the AO-Core resolver: No lists. -Load a device module from its name or a message ID. - -```erlang -normalize_keys(Other, _Opts) -> Other. -``` - -### load_device - -Ensure that a message is processable by the AO-Core resolver: No lists. -Load a device module from its name or a message ID. - -```erlang -load_device(Map, _Opts) when is_map(Map) -> {ok, Map}; -``` - -### load_device - -Ensure that a message is processable by the AO-Core resolver: No lists. -Load a device module from its name or a message ID. - -```erlang -load_device(ID, _Opts) when is_atom(ID) -> - try ID:module_info(), {ok, ID} - catch _:_ -> {error, not_loadable} - end; -``` - -### load_device - -Ensure that a message is processable by the AO-Core resolver: No lists. -Load a device module from its name or a message ID. - -```erlang -load_device(ID, Opts) when ?IS_ID(ID) -> - ?event(device_load, {requested_load, {id, ID}}, Opts), - case hb_opts:get(load_remote_devices, false, Opts) of - false -> - {error, remote_devices_disabled}; - true -> - ?event(device_load, {loading_from_cache, {id, ID}}, Opts), - {ok, Msg} = hb_cache:read(ID, Opts), - ?event(device_load, {received_device, {id, ID}, {msg, Msg}}, Opts), - TrustedSigners = hb_opts:get(trusted_device_signers, [], Opts), - Trusted = - lists:any( - fun(Signer) -> - lists:member(Signer, TrustedSigners) - end, - hb_message:signers(Msg, Opts) - ), - ?event(device_load, - {verifying_device_trust, - {id, ID}, - {trusted, Trusted}, - {signers, hb_message:signers(Msg, Opts)} - }, - Opts - ), - case Trusted of - false -> {error, device_signer_not_trusted}; - true -> - ?event(device_load, {loading_device, {id, ID}}, Opts), - case hb_maps:get(<<"content-type">>, Msg, undefined, Opts) of - <<"application/beam">> -> - case verify_device_compatibility(Msg, Opts) of - ok -> - ModName = - hb_util:key_to_atom( - hb_maps:get( - <<"module-name">>, - Msg, - undefined, - Opts - ), - new_atoms - ), - LoadRes = - erlang:load_module( - ModName, - hb_maps:get( - <<"body">>, - Msg, - undefined, - Opts - ) - ), - case LoadRes of - {module, _} -> - {ok, ModName}; - {error, Reason} -> - {error, {device_load_failed, Reason}} - end; - {error, Reason} -> - {error, {device_load_failed, Reason}} - end; - Other -> - {error, - {device_load_failed, - {incompatible_content_type, Other}, - {expected, <<"application/beam">>}, - {found, Other} - } - } - end - end - end; -``` - -### load_device - -Ensure that a message is processable by the AO-Core resolver: No lists. -Load a device module from its name or a message ID. - -```erlang -load_device(ID, Opts) -> - NormKey = - case is_atom(ID) of - true -> ID; - false -> normalize_key(ID) - end, - case lists:search( - fun (#{ <<"name">> := Name }) -> Name =:= NormKey end, - Preloaded = hb_opts:get(preloaded_devices, [], Opts) - ) of - false -> {error, {module_not_admissable, NormKey, Preloaded}}; - {value, #{ <<"module">> := Mod }} -> load_device(Mod, Opts) - end. -``` - -### verify_device_compatibility - -Verify that a device is compatible with the current machine. - -```erlang -verify_device_compatibility(Msg, Opts) -> - ?event(device_load, {verifying_device_compatibility, {msg, Msg}}, Opts), - Required = - lists:filtermap( - fun({<<"requires-", Key/binary>>, Value}) -> - {true, - { - hb_util:key_to_atom( - hb_ao:normalize_key(Key), - new_atoms - ), - hb_cache:ensure_loaded(Value, Opts) - } - }; - (_) -> false - end, - hb_maps:to_list(Msg, Opts) - ), - ?event(device_load, - {discerned_requirements, - {required, Required}, - {msg, Msg} - }, - Opts - ), - FailedToMatch = - lists:filtermap( - fun({Property, Value}) -> - % The values of these properties are _not_ 'keys', but we normalize - % them as such in order to make them comparable. -``` - -### info - -Get the info map for a device, optionally giving it a message if the - -```erlang -info(Msg, Opts) -> - info(message_to_device(Msg, Opts), Msg, Opts). -``` - -### info - -```erlang -info(DevMod, Msg, Opts) -> - %?event({calculating_info, {dev, DevMod}, {msg, Msg}}), - case find_exported_function(Msg, DevMod, info, 2, Opts) of - {ok, Fun} -> - Res = apply(Fun, truncate_args(Fun, [Msg, Opts])), - % ?event({ - % info_result, - % {dev, DevMod}, - % {args, truncate_args(Fun, [Msg])}, - % {result, Res} - % }), - Res; - not_found -> #{} - end. -``` - -### default_module - -The default device is the identity device, which simply returns the -The execution options that are used internally by this module - -```erlang -default_module() -> dev_message. -``` - -### internal_opts - -The default device is the identity device, which simply returns the -The execution options that are used internally by this module - -```erlang -internal_opts(Opts) -> - hb_maps:merge(Opts, #{ - topic => hb_opts:get(topic, ao_internal, Opts), - hashpath => ignore, - cache_control => [<<"no-cache">>, <<"no-store">>], - spawn_worker => false, - await_inprogress => false -``` - ---- - -*Generated from [hb_ao.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_ao.erl)* diff --git a/docs/book/src/hb_ao_test_vectors.erl.md b/docs/book/src/hb_ao_test_vectors.erl.md deleted file mode 100644 index 9a27b1619..000000000 --- a/docs/book/src/hb_ao_test_vectors.erl.md +++ /dev/null @@ -1,892 +0,0 @@ -# hb_ao_test_vectors - -[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_ao_test_vectors.erl) - -Uses a series of different `Opts` values to test the resolution engine's -execution under different circumstances. - ---- - -### run_test - -Uses a series of different `Opts` values to test the resolution engine's -Easy hook to make a test executable via the command line: - -```erlang -run_test() -> - multiple_as_subresolutions_test(#{}). -``` - -### suite_test_ - -Run each test in the file with each set of options. Start and reset - -```erlang -suite_test_() -> - hb_test_utils:suite_with_opts(test_suite(), test_opts()). -``` - -### benchmark_test_ - -```erlang -benchmark_test_() -> - hb_test_utils:suite_with_opts(benchmark_suite(), test_opts()). -``` - -### test_suite - -```erlang -test_suite() -> - [ - {resolve_simple, "resolve simple", - fun resolve_simple_test/1}, - {resolve_id, "resolve id", - fun resolve_id_test/1}, - {start_as, "start as", - fun start_as_test/1}, - {start_as_with_parameters, "start as with parameters", - fun start_as_with_parameters_test/1}, - {load_as, "load as", - fun load_as_test/1}, - {as_path, "as path", - fun as_path_test/1}, - {continue_as, "continue as", - fun continue_as_test/1}, - {multiple_as_subresolutions, "multiple as subresolutions", - fun multiple_as_subresolutions_test/1}, - {resolve_key_twice, "resolve key twice", - fun resolve_key_twice_test/1}, - {resolve_from_multiple_keys, "resolve from multiple keys", - fun resolve_from_multiple_keys_test/1}, - {resolve_path_element, "resolve path element", - fun resolve_path_element_test/1}, - {resolve_binary_key, "resolve binary key", - fun resolve_binary_key_test/1}, - {key_to_binary, "key to binary", - fun key_to_binary_test/1}, - {key_from_id_device_with_args, "key from id device with args", - fun key_from_id_device_with_args_test/1}, - {device_with_handler_function, "device with handler function", - fun device_with_handler_function_test/1}, - {device_with_default_handler_function, - "device with default handler function", - fun device_with_default_handler_function_test/1}, - {basic_get, "basic get", - fun basic_get_test/1}, - {recursive_get, "recursive get", - fun recursive_get_test/1}, - {deep_recursive_get, "deep recursive get", - fun deep_recursive_get_test/1}, - {basic_set, "basic set", - fun basic_set_test/1}, - {get_with_device, "get with device", - fun get_with_device_test/1}, - {get_as_with_device, "get as with device", - fun get_as_with_device_test/1}, - {set_with_device, "set with device", - fun set_with_device_test/1}, - {deep_set, "deep set", - fun deep_set_test/1}, - {deep_set_with_device, "deep set with device", - fun deep_set_with_device_test/1}, - {device_exports, "device exports", - fun device_exports_test/1}, - {device_excludes, "device excludes", - fun device_excludes_test/1}, - {denormalized_device_key, "denormalized device key", - fun denormalized_device_key_test/1}, - {list_transform, "list transform", - fun list_transform_test/1}, - {step_hook, "step hook", - fun step_hook_test/1} - ]. -``` - -### benchmark_suite - -```erlang -benchmark_suite() -> - [ - {benchmark_simple, "simple resolution benchmark", - fun benchmark_simple_test/1}, - {benchmark_multistep, "multistep resolution benchmark", - fun benchmark_multistep_test/1}, - {benchmark_get, "get benchmark", - fun benchmark_get_test/1}, - {benchmark_set, "single value set benchmark", - fun benchmark_set_test/1}, - {benchmark_set_multiple, "set two keys benchmark", - fun benchmark_set_multiple_test/1}, - {benchmark_set_multiple_deep, "set two keys deep benchmark", - fun benchmark_set_multiple_deep_test/1} - ]. -``` - -### test_opts - -```erlang -test_opts() -> - [ - #{ - name => normal, - desc => "Default opts", - opts => #{}, - skip => [] - }, - #{ - name => without_hashpath, - desc => "Default without hashpath", - opts => #{ - hashpath => ignore - }, - skip => [] - }, - #{ - name => no_cache, - desc => "No cache read or write", - opts => #{ - hashpath => ignore, - cache_control => [<<"no-cache">>, <<"no-store">>], - spawn_worker => false, - store => #{ - <<"store-module">> => hb_store_fs, - <<"name">> => <<"cache-TEST/fs">> - } - }, - skip => [load_as] - }, - #{ - name => only_store, - desc => "Store, don't read", - opts => #{ - hashpath => update, - cache_control => [<<"no-cache">>], - spawn_worker => false, - store => #{ - <<"store-module">> => hb_store_fs, - <<"name">> => <<"cache-TEST/fs">> - } - }, - skip => [ - denormalized_device_key, - deep_set_with_device, - load_as - ], - reset => false - }, - #{ - name => only_if_cached, - desc => "Only read, don't exec", - opts => #{ - hashpath => ignore, - cache_control => [<<"only-if-cached">>], - spawn_worker => false, - store => #{ - <<"store-module">> => hb_store_fs, - <<"name">> => <<"cache-TEST/fs">> - } - }, - skip => [ - % Exclude tests that return a list on its own for now, as raw - % lists cannot be cached yet. -``` - -### exec_dummy_device - -Ensure that we can read a device from the cache then execute it. By - -```erlang -exec_dummy_device(SigningWallet, Opts) -> - % Compile the test device and store it in an accessible cache to the execution - % environment. -``` - -### load_device_test - -```erlang -load_device_test() -> - % Establish an execution environment which trusts the device author. -``` - -### untrusted_load_device_test - -```erlang -untrusted_load_device_test() -> - % Establish an execution environment which does not trust the device author. -``` - -### resolve_simple_test - -```erlang -resolve_simple_test(Opts) -> - Res = hb_ao:resolve(#{ <<"a">> => <<"RESULT">> }, <<"a">>, Opts), - ?assertEqual({ok, <<"RESULT">>}, Res). -``` - -### resolve_id_test - -```erlang -resolve_id_test(Opts) -> - ?assertMatch( - ID when byte_size(ID) == 43, - hb_ao:get(id, #{ test_key => <<"1">> }, Opts) - ). -``` - -### resolve_key_twice_test - -```erlang -resolve_key_twice_test(Opts) -> - % Ensure that the same message can be resolved again. -``` - -### resolve_from_multiple_keys_test - -```erlang -resolve_from_multiple_keys_test(Opts) -> - ?assertEqual( - {ok, [<<"a">>]}, - hb_ao:resolve(#{ <<"a">> => <<"1">>, <<"priv_a">> => <<"2">> }, <<"keys">>, Opts) - ). -``` - -### resolve_path_element_test - -```erlang -resolve_path_element_test(Opts) -> - ?assertEqual( - {ok, [<<"test_path">>]}, - hb_ao:resolve(#{ <<"path">> => [<<"test_path">>] }, <<"path">>, Opts) - ), - ?assertEqual( - {ok, [<<"a">>]}, - hb_ao:resolve(#{ <<"Path">> => [<<"a">>] }, <<"Path">>, Opts) - ). -``` - -### key_to_binary_test - -```erlang -key_to_binary_test(Opts) -> - ?assertEqual(<<"a">>, hb_ao:normalize_key(a, Opts)), - ?assertEqual(<<"a">>, hb_ao:normalize_key(<<"a">>, Opts)), - ?assertEqual(<<"a">>, hb_ao:normalize_key("a", Opts)). -``` - -### resolve_binary_key_test - -```erlang -resolve_binary_key_test(Opts) -> - ?assertEqual( - {ok, <<"RESULT">>}, - hb_ao:resolve(#{ a => <<"RESULT">> }, <<"a">>, Opts) - ), - ?assertEqual( - {ok, <<"1">>}, - hb_ao:resolve( - #{ - <<"Test-Header">> => <<"1">> - }, - <<"Test-Header">>, - Opts - ) - ). -``` - -### generate_device_with_keys_using_args - -Generates a test device with three keys, each of which uses - -```erlang -generate_device_with_keys_using_args() -> - #{ - key_using_only_state => - fun(State) -> - {ok, - <<(hb_maps:get(<<"state_key">>, State))/binary>> - } - end, - key_using_state_and_msg => - fun(State, Msg) -> - {ok, - << - (hb_maps:get(<<"state_key">>, State))/binary, - (hb_maps:get(<<"msg_key">>, Msg))/binary - >> - } - end, - key_using_all => - fun(State, Msg, Opts) -> - {ok, - << - (hb_maps:get(<<"state_key">>, State, undefined, Opts))/binary, - (hb_maps:get(<<"msg_key">>, Msg, undefined, Opts))/binary, - (hb_maps:get(<<"opts_key">>, Opts, undefined, Opts))/binary - >> - } - end - }. -``` - -### gen_default_device - -Create a simple test device that implements the default handler. - -```erlang -gen_default_device() -> - #{ - info => - fun() -> - #{ - default => - fun(_, _State) -> - {ok, <<"DEFAULT">>} - end - } - end, - <<"state_key">> => - fun(_) -> - {ok, <<"STATE">>} - end - }. -``` - -### gen_handler_device - -Create a simple test device that implements the handler key. - -```erlang -gen_handler_device() -> - #{ - info => - fun() -> - #{ - handler => - fun(<<"set">>, M1, M2, Opts) -> - dev_message:set(M1, M2, Opts); - (_, _, _, _) -> - {ok, <<"HANDLER VALUE">>} - end - } - end - }. -``` - -### key_from_id_device_with_args_test - -Test that arguments are passed to a device key as expected. - -```erlang -key_from_id_device_with_args_test(Opts) -> - Msg = - #{ - device => generate_device_with_keys_using_args(), - state_key => <<"1">> - }, - ?assertEqual( - {ok, <<"1">>}, - hb_ao:resolve( - Msg, - #{ - <<"path">> => <<"key_using_only_state">>, - <<"msg_key">> => <<"2">> % Param message, which is ignored - }, - Opts - ) - ), - ?assertEqual( - {ok, <<"13">>}, - hb_ao:resolve( - Msg, - #{ - <<"path">> => <<"key_using_state_and_msg">>, - <<"msg_key">> => <<"3">> % Param message, with value to add - }, - Opts - ) - ), - ?assertEqual( - {ok, <<"1337">>}, - hb_ao:resolve( - Msg, - #{ - <<"path">> => <<"key_using_all">>, - <<"msg_key">> => <<"3">> % Param message - }, - Opts#{ - <<"opts_key">> => <<"37">>, - <<"cache_control">> => [<<"no-cache">>, <<"no-store">>] - } - ) - ). -``` - -### device_with_handler_function_test - -```erlang -device_with_handler_function_test(Opts) -> - Msg = - #{ - device => gen_handler_device(), - test_key => <<"BAD">> - }, - ?assertEqual( - {ok, <<"HANDLER VALUE">>}, - hb_ao:resolve(Msg, <<"test_key">>, Opts) - ). -``` - -### device_with_default_handler_function_test - -```erlang -device_with_default_handler_function_test(Opts) -> - Msg = - #{ - device => gen_default_device() - }, - ?assertEqual( - {ok, <<"STATE">>}, - hb_ao:resolve(Msg, <<"state_key">>, Opts) - ), - ?assertEqual( - {ok, <<"DEFAULT">>}, - hb_ao:resolve(Msg, <<"any_random_key">>, Opts) - ). -``` - -### basic_get_test - -```erlang -basic_get_test(Opts) -> - Msg = #{ <<"key1">> => <<"value1">>, <<"key2">> => <<"value2">> }, - ?assertEqual(<<"value1">>, hb_ao:get(<<"key1">>, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_ao:get([<<"key2">>], Msg, Opts)). -``` - -### recursive_get_test - -```erlang -recursive_get_test(Opts) -> - Msg = #{ - <<"key1">> => <<"value1">>, - <<"key2">> => #{ - <<"key3">> => <<"value3">>, - <<"key4">> => #{ - <<"key5">> => <<"value5">>, - <<"key6">> => #{ - <<"key7">> => <<"value7">> - } - } - } - }, - ?assertEqual( - {ok, <<"value1">>}, - hb_ao:resolve(Msg, #{ <<"path">> => <<"key1">> }, Opts) - ), - ?assertEqual(<<"value1">>, hb_ao:get(<<"key1">>, Msg, Opts)), - ?assertEqual( - {ok, <<"value3">>}, - hb_ao:resolve(Msg, #{ <<"path">> => [<<"key2">>, <<"key3">>] }, Opts) - ), - ?assertEqual(<<"value3">>, hb_ao:get([<<"key2">>, <<"key3">>], Msg, Opts)), - ?assertEqual(<<"value3">>, hb_ao:get(<<"key2/key3">>, Msg, Opts)). -``` - -### deep_recursive_get_test - -```erlang -deep_recursive_get_test(Opts) -> - Msg = #{ - <<"key1">> => <<"value1">>, - <<"key2">> => #{ - <<"key3">> => <<"value3">>, - <<"key4">> => #{ - <<"key5">> => <<"value5">>, - <<"key6">> => #{ - <<"key7">> => <<"value7">> - } - } - } - }, - ?assertEqual(<<"value7">>, hb_ao:get(<<"key2/key4/key6/key7">>, Msg, Opts)). -``` - -### basic_set_test - -```erlang -basic_set_test(Opts) -> - Msg = #{ <<"key1">> => <<"value1">>, <<"key2">> => <<"value2">> }, - UpdatedMsg = hb_ao:set(Msg, #{ <<"key1">> => <<"new_value1">> }, Opts), - ?event({set_key_complete, {key, <<"key1">>}, {value, <<"new_value1">>}}), - ?assertEqual(<<"new_value1">>, hb_ao:get(<<"key1">>, UpdatedMsg, Opts)), - ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, UpdatedMsg, Opts)). -``` - -### get_with_device_test - -```erlang -get_with_device_test(Opts) -> - Msg = - #{ - <<"device">> => generate_device_with_keys_using_args(), - <<"state_key">> => <<"STATE">> - }, - ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), - ?assertEqual(<<"STATE">>, hb_ao:get(<<"key_using_only_state">>, Msg, Opts)). -``` - -### get_as_with_device_test - -```erlang -get_as_with_device_test(Opts) -> - Msg = - #{ - <<"device">> => gen_handler_device(), - <<"test_key">> => <<"ACTUAL VALUE">> - }, - ?assertEqual( - <<"HANDLER VALUE">>, - hb_ao:get(test_key, Msg, Opts) - ), - ?assertEqual( - <<"ACTUAL VALUE">>, - hb_ao:get(test_key, {as, dev_message, Msg}, Opts) - ). -``` - -### set_with_device_test - -```erlang -set_with_device_test(Opts) -> - Msg = - #{ - <<"device">> => - #{ - <<"set">> => - fun(State, _Msg) -> - Acc = hb_maps:get(<<"set_count">>, State, <<"">>, Opts), - {ok, - State#{ - <<"set_count">> => << Acc/binary, "." >> - } - } - end - }, - <<"state_key">> => <<"STATE">> - }, - ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), - SetOnce = hb_ao:set(Msg, #{ <<"state_key">> => <<"SET_ONCE">> }, Opts), - ?assertEqual(<<".">>, hb_ao:get(<<"set_count">>, SetOnce, Opts)), - SetTwice = hb_ao:set(SetOnce, #{ <<"state_key">> => <<"SET_TWICE">> }, Opts), - ?assertEqual(<<"..">>, hb_ao:get(<<"set_count">>, SetTwice, Opts)), - ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, SetTwice, Opts)). -``` - -### deep_set_test - -```erlang -deep_set_test(Opts) -> - % First validate second layer changes are handled correctly. -``` - -### deep_set_new_messages_test - -```erlang -deep_set_new_messages_test() -> - Opts = hb_maps:get(opts, hd(test_opts())), - % Test that new messages are created when the path does not exist. -``` - -### deep_set_with_device_test - -```erlang -deep_set_with_device_test(Opts) -> - Device = #{ - set => - fun(Msg1, Msg2) -> - % A device where the set function modifies the key - % and adds a modified flag. -``` - -### device_exports_test - -```erlang -device_exports_test(Opts) -> - Msg = #{ <<"device">> => dev_message }, - ?assert(hb_ao:is_exported(Msg, dev_message, info, Opts)), - ?assert(hb_ao:is_exported(Msg, dev_message, set, Opts)), - ?assert( - hb_ao:is_exported( - Msg, - dev_message, - not_explicitly_exported, - Opts - ) - ), - Dev = #{ - info => fun() -> #{ exports => [set] } end, - set => fun(_, _) -> {ok, <<"SET">>} end - }, - Msg2 = #{ <<"device">> => Dev }, - ?assert(hb_ao:is_exported(Msg2, Dev, info, Opts)), - ?assert(hb_ao:is_exported(Msg2, Dev, set, Opts)), - ?assert(not hb_ao:is_exported(Msg2, Dev, not_exported, Opts)), - Dev2 = #{ - info => - fun() -> - #{ - exports => [test1, <<"test2">>], - handler => - fun() -> - {ok, <<"Handler-Value">>} - end - } - end - }, - Msg3 = #{ <<"device">> => Dev2, <<"test1">> => <<"BAD1">>, <<"test3">> => <<"GOOD3">> }, - ?assertEqual(<<"Handler-Value">>, hb_ao:get(<<"test1">>, Msg3, Opts)), - ?assertEqual(<<"Handler-Value">>, hb_ao:get(<<"test2">>, Msg3, Opts)), - ?assertEqual(<<"GOOD3">>, hb_ao:get(<<"test3">>, Msg3, Opts)), - ?assertEqual(<<"GOOD4">>, - hb_ao:get( - <<"test4">>, - hb_ao:set(Msg3, <<"test4">>, <<"GOOD4">>, Opts) - ) - ), - ?assertEqual(not_found, hb_ao:get(<<"test5">>, Msg3, Opts)). -``` - -### device_excludes_test - -```erlang -device_excludes_test(Opts) -> - % Create a device that returns an identifiable message for any key, but also - % sets excludes to [set], such that the message can be modified using the - % default handler. -``` - -### denormalized_device_key_test - -```erlang -denormalized_device_key_test(Opts) -> - Msg = #{ <<"device">> => dev_test }, - ?assertEqual(dev_test, hb_ao:get(device, Msg, Opts)), - ?assertEqual(dev_test, hb_ao:get(<<"device">>, Msg, Opts)), - ?assertEqual({module, dev_test}, - erlang:fun_info( - element(3, hb_ao:message_to_fun(Msg, test_func, Opts)), - module - ) - ). -``` - -### list_transform_test - -```erlang -list_transform_test(Opts) -> - Msg = [<<"A">>, <<"B">>, <<"C">>, <<"D">>, <<"E">>], - ?assertEqual(<<"A">>, hb_ao:get(1, Msg, Opts)), - ?assertEqual(<<"B">>, hb_ao:get(2, Msg, Opts)), - ?assertEqual(<<"C">>, hb_ao:get(3, Msg, Opts)), - ?assertEqual(<<"D">>, hb_ao:get(4, Msg, Opts)), - ?assertEqual(<<"E">>, hb_ao:get(5, Msg, Opts)). -``` - -### start_as_test - -```erlang -start_as_test(Opts) -> - ?assertEqual( - {ok, <<"GOOD_FUNCTION">>}, - hb_ao:resolve_many( - [ - {as, <<"test-device@1.0">>, #{ <<"path">> => <<>> }}, - #{ <<"path">> => <<"test_func">> } - ], - Opts - ) - ). -``` - -### start_as_with_parameters_test - -```erlang -start_as_with_parameters_test(Opts) -> - % Resolve a key on a message that has its device set with `as'. -``` - -### load_as_test - -```erlang -load_as_test(Opts) -> - % Load a message as a device with the `as' keyword. -``` - -### as_path_test - -```erlang -as_path_test(Opts) -> - % Create a message with the test device, which implements the test_func - % function. It normally returns `GOOD_FUNCTION'. -``` - -### continue_as_test - -```erlang -continue_as_test(Opts) -> - % Resolve a list of messages in sequence, swapping the device in the middle. -``` - -### multiple_as_subresolutions_test - -```erlang -multiple_as_subresolutions_test(Opts) -> - % Test that multiple as subresolutions in a sequence are handled correctly. -``` - -### step_hook_test - -```erlang -step_hook_test(InitOpts) -> - % Test that the step hook is called correctly. We do this by sending ourselves - % a message each time the hook is called. We also send a `reference', such - % that this test is uniquely identified and further/prior tests do not affect - % it. -``` - -### benchmark_simple_test - -```erlang -benchmark_simple_test(Opts) -> - Time = - hb_test_utils:benchmark_iterations( - fun(I) -> hb_ao:resolve(#{ <<"a">> => I }, <<"a">>, Opts) end, - ?BENCHMARK_ITERATIONS - ), - hb_test_utils:benchmark_print( - <<"Single-step resolutions:">>, - ?BENCHMARK_ITERATIONS, - Time - ). -``` - -### benchmark_multistep_test - -```erlang -benchmark_multistep_test(Opts) -> - Time = - hb_test_utils:benchmark_iterations( - fun(I) -> - hb_ao:resolve( - #{ - <<"iteration">> => I, - <<"a">> => #{ - <<"b">> => #{ <<"return">> => I } - } - }, - <<"a/b/return">>, - Opts - ) - end, - ?BENCHMARK_ITERATIONS - ), - hb_test_utils:benchmark_print( - <<"Multistep resolutions:">>, - ?BENCHMARK_ITERATIONS, - Time - ). -``` - -### benchmark_get_test - -```erlang -benchmark_get_test(Opts) -> - Time = - hb_test_utils:benchmark_iterations( - fun(I) -> - hb_ao:get( - <<"a">>, - #{ <<"a">> => <<"1">>, <<"iteration">> => I }, - Opts - ) - end, - ?BENCHMARK_ITERATIONS - ), - hb_test_utils:benchmark_print( - <<"Get operations:">>, - ?BENCHMARK_ITERATIONS, - Time - ). -``` - -### benchmark_set_test - -```erlang -benchmark_set_test(Opts) -> - Time = - hb_test_utils:benchmark_iterations( - fun(I) -> - hb_ao:set( - #{ <<"a">> => <<"1">>, <<"iteration">> => I }, - <<"a">>, - <<"2">>, - Opts - ) - end, - ?BENCHMARK_ITERATIONS - ), - hb_test_utils:benchmark_print( - <<"Single value set operations:">>, - ?BENCHMARK_ITERATIONS, - Time - ). -``` - -### benchmark_set_multiple_test - -```erlang -benchmark_set_multiple_test(Opts) -> - Time = - hb_test_utils:benchmark_iterations( - fun(I) -> - hb_ao:set( - #{ <<"a">> => <<"1">>, <<"iteration">> => I }, - #{ <<"a">> => <<"1a">>, <<"b">> => <<"2">> }, - Opts - ) - end, - ?BENCHMARK_ITERATIONS - ), - hb_test_utils:benchmark_print( - <<"Set two keys operations:">>, - ?BENCHMARK_ITERATIONS, - Time - ). -``` - -### benchmark_set_multiple_deep_test - -```erlang -benchmark_set_multiple_deep_test(Opts) -> - Time = - hb_test_utils:benchmark_iterations( - fun(I) -> - hb_ao:set( - #{ <<"a">> => #{ <<"b">> => <<"1">> } }, - #{ <<"a">> => #{ <<"b">> => <<"2">>, <<"c">> => I } }, - Opts - ) - end, - ?BENCHMARK_ITERATIONS - ), - hb_test_utils:benchmark_print( - <<"Set two keys operations:">>, - ?BENCHMARK_ITERATIONS, - Time -``` - ---- - -*Generated from [hb_ao_test_vectors.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_ao_test_vectors.erl)* diff --git a/docs/book/src/hb_app.erl.md b/docs/book/src/hb_app.erl.md deleted file mode 100644 index 5ad52cd50..000000000 --- a/docs/book/src/hb_app.erl.md +++ /dev/null @@ -1,37 +0,0 @@ -# hb_app - -[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_app.erl) - -The main HyperBEAM application module. - ---- - -## Exported Functions - -- `start/2` -- `stop/1` - ---- - -### start - -The main HyperBEAM application module. - -```erlang -start(_StartType, _StartArgs) -> - hb:init(), - hb_sup:start_link(), - ok = dev_scheduler_registry:start(), - _TimestampServer = ar_timestamp:start(), - {ok, _} = hb_http_server:start(). -``` - -### stop - -```erlang -stop(_State) -> -``` - ---- - -*Generated from [hb_app.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_app.erl)* diff --git a/docs/book/src/hb_beamr.erl.md b/docs/book/src/hb_beamr.erl.md deleted file mode 100644 index bc2f18a17..000000000 --- a/docs/book/src/hb_beamr.erl.md +++ /dev/null @@ -1,443 +0,0 @@ -# hb_beamr - -[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_beamr.erl) - -BEAMR: A WAMR wrapper for BEAM. -Beamr is a library that allows you to run WASM modules in BEAM, using the -Webassembly Micro Runtime (WAMR) as its engine. Each WASM module is -executed using a Linked-In Driver (LID) that is loaded into BEAM. It is -designed with a focus on supporting long-running WASM executions that -interact with Erlang functions and processes easily. -Because each WASM module runs as an independent async worker, if you plan -to run many instances in parallel, you should be sure to configure the -BEAM to have enough async worker threads enabled (see `erl +A N` in the -Erlang manuals). -The core API is simple: -`. -
- start(WasmBinary) -> {ok, Port, Imports, Exports}
- Where:
- WasmBinary is the WASM binary to load.
- Port is the port to the LID.
- Imports is a list of tuples of the form {Module, Function,
- Args, Signature}.
- Exports is a list of tuples of the form {Function, Args,
- Signature}.
- stop(Port) -> ok
- call(Port, FunctionName, Args) -> {ok, Result}
- Where:
- FunctionName is the name of the function to call.
- Args is a list of Erlang terms (converted to WASM values by
- BEAMR) that match the signature of the function.
- Result is a list of Erlang terms (converted from WASM values).
- call(Port, FunName, Args[, Import, State, Opts]) -> {ok, Res, NewState}
- Where:
- ImportFun is a function that will be called upon each import.
- ImportFun must have an arity of 2: Taking an arbitrary `state`
- term, and a map containing the `port`, `module`, `func`, `args`,
- `signature`, and the `options` map of the import.
- It must return a tuple of the form {ok, Response, NewState}.
- serialize(Port) -> {ok, Mem}
- Where:
- Port is the port to the LID.
- Mem is a binary representing the full WASM state.
- deserialize(Port, Mem) -> ok
- Where:
- Port is the port to the LID.
- Mem is a binary output of a previous `serialize/1` call.
-
-BEAMR was designed for use in the HyperBEAM project, but is suitable for
-deployment in other Erlang applications that need to run WASM modules. PRs
-are welcome.
-
----
-
-## Exported Functions
-
-- `call/3`
-- `call/4`
-- `call/5`
-- `call/6`
-- `deserialize/2`
-- `serialize/1`
-- `start/1`
-- `start/2`
-- `stop/1`
-- `stub/3`
-- `wasm_send/2`
-
----
-
-### load_driver
-
-BEAMR: A WAMR wrapper for BEAM.
-Load the driver for the WASM executor.
-
-```erlang
-load_driver() ->
- case erl_ddll:load(code:priv_dir(hb), ?MODULE) of
- ok -> ok;
- {error, already_loaded} -> ok;
- {error, Error} -> {error, Error}
- end.
-```
-
-### start
-
-Start a WASM executor context. Yields a port to the LID, and the
-
-```erlang
-start(WasmBinary) when is_binary(WasmBinary) ->
- start(WasmBinary, wasm).
-```
-
-### start
-
-```erlang
-start(WasmBinary, Mode) when is_binary(WasmBinary) ->
- ?event({loading_module, {bytes, byte_size(WasmBinary)}, Mode}),
- Self = self(),
- WASM = spawn(
- fun() ->
- ok = load_driver(),
- Port = open_port({spawn, "hb_beamr"}, []),
- Port ! {self(), {command, term_to_binary({init, WasmBinary, Mode})}},
- ?event({waiting_for_init_from, Port}),
- worker(Port, Self)
- end
- ),
- receive
- {execution_result, Imports, Exports} ->
- ?event(
- {wasm_init_success,
- {imports, Imports},
- {exports, Exports}}),
- {ok, WASM, Imports, Exports};
- {error, Error} ->
- ?event({wasm_init_error, Error}),
- stop(WASM),
- {error, Error}
- end.
-```
-
-### worker
-
-A worker process that is responsible for handling a WASM instance.
-
-```erlang
-worker(Port, Listener) ->
- receive
- stop ->
- ?event({stop_invoked_for_beamr, self()}),
- case erlang:port_info(Port, id) of
- undefined ->
- ok;
- _ ->
- port_close(Port),
- ok
- end,
- ok;
- {wasm_send, NewListener, Message} ->
- ?event({wasm_send, {listener, NewListener}, {message, Message}}),
- Port ! {self(), Message},
- worker(Port, NewListener);
- WASMResult ->
- ?event({wasm_result, {listener, Listener}, {result, WASMResult}}),
- Listener ! WASMResult,
- worker(Port, Listener)
- end.
-```
-
-### wasm_send
-
-```erlang
-wasm_send(WASM, Message) when is_pid(WASM) ->
- WASM ! {wasm_send, self(), Message},
- ok.
-```
-
-### stop
-
-Stop a WASM executor context.
-
-```erlang
-stop(WASM) when is_pid(WASM) ->
- WASM ! stop,
- ok.
-```
-
-### call
-
-Call a function in the WASM executor (see moduledoc for more details).
-
-```erlang
-call(PID, FuncRef, Args) ->
- {ok, Res, _} = call(PID, FuncRef, Args, fun stub/3),
- {ok, Res}.
-```
-
-### call
-
-```erlang
-call(PID, FuncRef, Args, ImportFun) ->
- call(PID, FuncRef, Args, ImportFun, #{}).
-```
-
-### call
-
-```erlang
-call(PID, FuncRef, Args, ImportFun, StateMsg) ->
- call(PID, FuncRef, Args, ImportFun, StateMsg, #{}).
-```
-
-### call
-
-```erlang
-call(PID, FuncRef, Args, ImportFun, StateMsg, Opts)
- when is_binary(FuncRef) ->
- call(PID, binary_to_list(FuncRef), Args, ImportFun, StateMsg, Opts);
-```
-
-### call
-
-```erlang
-call(WASM, FuncRef, Args, ImportFun, StateMsg, Opts)
- when is_pid(WASM)
- andalso (is_list(FuncRef) or is_integer(FuncRef))
- andalso is_list(Args)
- andalso is_function(ImportFun)
- andalso is_map(Opts) ->
- case is_valid_arg_list(Args) of
- true ->
- ?event(
- {call_started,
- WASM,
- FuncRef,
- Args,
- ImportFun,
- StateMsg,
- Opts}),
- wasm_send(WASM,
- {command,
- term_to_binary(
- case is_integer(FuncRef) of
- true -> {indirect_call, FuncRef, Args};
- false -> {call, FuncRef, Args}
- end
- )
- }
- ),
- ?event({waiting_for_call_result, self(), WASM}),
- monitor_call(WASM, ImportFun, StateMsg, Opts);
- false ->
- {error, {invalid_args, Args}}
- end.
-```
-
-### stub
-
-Stub import function for the WASM executor.
-
-```erlang
-stub(Msg1, _Msg2, _Opts) ->
- ?event(stub_stdlib_called),
- {ok, [0], Msg1}.
-```
-
-### monitor_call
-
-Synchonously monitor the WASM executor for a call result and any
-
-```erlang
-monitor_call(WASM, ImportFun, StateMsg, Opts) ->
- receive
- {execution_result, Result} ->
- ?event({call_result, Result}),
- {ok, Result, StateMsg};
- {import, Module, Func, Args, Signature} ->
- ?event({import_called, Module, Func, Args, Signature}),
- try
- {ok, Res, StateMsg2} =
- ImportFun(StateMsg,
- #{
- instance => WASM,
- module => Module,
- func => Func,
- args => Args,
- func_sig => Signature
- },
- Opts
- ),
- ?event({import_ret, Module, Func, {args, Args}, {res, Res}}),
- dispatch_response(WASM, Res),
- monitor_call(WASM, ImportFun, StateMsg2, Opts)
- catch
- Err:Reason:Stack ->
- % Signal the WASM executor to stop.
-```
-
-### dispatch_response
-
-Check the type of an import response and dispatch it to a Beamr port.
-
-```erlang
-dispatch_response(WASM, Term) when is_pid(WASM) ->
- case is_valid_arg_list(Term) of
- true ->
- wasm_send(WASM,
- {command, term_to_binary({import_response, Term})});
- false ->
- throw({error, {invalid_response, Term}})
- end;
-```
-
-### dispatch_response
-
-Check the type of an import response and dispatch it to a Beamr port.
-
-```erlang
-dispatch_response(_WASM, Term) ->
- throw({error, {invalid_response, Term}}).
-```
-
-### is_valid_arg_list
-
-Check that a list of arguments is valid for a WASM function call.
-
-```erlang
-is_valid_arg_list(Args) when is_list(Args) ->
- lists:all(fun(Arg) -> is_integer(Arg) or is_float(Arg) end, Args);
-```
-
-### is_valid_arg_list
-
-Check that a list of arguments is valid for a WASM function call.
-
-```erlang
-is_valid_arg_list(_) ->
- false.
-```
-
-### serialize
-
-Serialize the WASM state to a binary.
-
-```erlang
-serialize(WASM) when is_pid(WASM) ->
- ?event(starting_serialize),
- {ok, Size} = hb_beamr_io:size(WASM),
- ?event({image_size, Size}),
- {ok, Mem} = hb_beamr_io:read(WASM, 0, Size),
- ?event({finished_serialize, byte_size(Mem)}),
- {ok, Mem}.
-```
-
-### deserialize
-
-Deserialize a WASM state from a binary.
-
-```erlang
-deserialize(WASM, Bin) when is_pid(WASM) andalso is_binary(Bin) ->
- ?event(starting_deserialize),
- Res = hb_beamr_io:write(WASM, 0, Bin),
- ?event({finished_deserialize, Res}),
- ok.
-```
-
-### driver_loads_test
-
-```erlang
-driver_loads_test() ->
- ?assertEqual(ok, load_driver()).
-```
-
-### simple_wasm_test
-
-Test standalone `hb_beamr` correctly after loading a WASM module.
-
-```erlang
-simple_wasm_test() ->
- {ok, File} = file:read_file("test/test.wasm"),
- {ok, WASM, _Imports, _Exports} = start(File),
- {ok, [Result]} = call(WASM, "fac", [5.0]),
- ?assertEqual(120.0, Result).
-```
-
-### imported_function_test
-
-Test that imported functions can be called from the WASM module.
-
-```erlang
-imported_function_test() ->
- {ok, File} = file:read_file("test/pow_calculator.wasm"),
- {ok, WASM, _Imports, _Exports} = start(File),
- {ok, [Result], _} =
- call(WASM, <<"pow">>, [2, 5],
- fun(Msg1, #{ args := [Arg1, Arg2] }, _Opts) ->
- {ok, [Arg1 * Arg2], Msg1}
- end),
- ?assertEqual(32, Result).
-```
-
-### wasm64_test
-
-Test that WASM Memory64 modules load and execute correctly.
-
-```erlang
-wasm64_test() ->
- {ok, File} = file:read_file("test/test-64.wasm"),
- {ok, WASM, _ImportMap, _Exports} = start(File),
- {ok, [Result]} = call(WASM, "fac", [5.0]),
- ?assertEqual(120.0, Result).
-```
-
-### multiclient_test
-
-Ensure that processes outside of the initial one can interact with
-
-```erlang
-multiclient_test() ->
- Self = self(),
- ExecPID = spawn(fun() ->
- receive {wasm, WASM} ->
- {ok, [Result]} = call(WASM, "fac", [5.0]),
- Self ! {result, Result}
- end
- end),
- _StartPID = spawn(fun() ->
- {ok, File} = file:read_file("test/test.wasm"),
- {ok, WASM, _ImportMap, _Exports} = start(File),
- ExecPID ! {wasm, WASM}
- end),
- receive
- {result, Result} ->
- ?assertEqual(120.0, Result)
- end.
-```
-
-### benchmark_test
-
-```erlang
-benchmark_test() ->
- BenchTime = 1,
- {ok, File} = file:read_file("test/test-64.wasm"),
- {ok, WASM, _ImportMap, _Exports} = start(File),
- Iterations = hb_test_utils:benchmark(
- fun() ->
- {ok, [Result]} = call(WASM, "fac", [5.0]),
- ?assertEqual(120.0, Result)
- end,
- BenchTime
- ),
- ?event(benchmark, {scheduled, Iterations}),
- ?assert(Iterations > 1000),
- hb_test_utils:benchmark_print(
- <<"Direct beamr: Executed">>,
- <<"calls">>,
- Iterations,
- BenchTime
- ),
-```
-
----
-
-*Generated from [hb_beamr.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_beamr.erl)*
diff --git a/docs/book/src/hb_beamr_io.erl.md b/docs/book/src/hb_beamr_io.erl.md
deleted file mode 100644
index 181ac9e38..000000000
--- a/docs/book/src/hb_beamr_io.erl.md
+++ /dev/null
@@ -1,238 +0,0 @@
-# hb_beamr_io
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_beamr_io.erl)
-
-Simple interface for memory management for Beamr instances.
-It allows for reading and writing to memory, as well as allocating and
-freeing memory by calling the WASM module's exported malloc and free
-functions.
-Unlike the majority of HyperBEAM modules, this module takes a defensive
-approach to type checking, breaking from the conventional Erlang style,
-such that failures are caught in the Erlang-side of functions rather than
-in the C/WASM-side.
-
----
-
-## Exported Functions
-
-- `free/2`
-- `malloc/2`
-- `read_string/2`
-- `read/3`
-- `size/1`
-- `write_string/2`
-- `write/3`
-
----
-
-### size
-
-Simple interface for memory management for Beamr instances.
-Get the size (in bytes) of the native memory allocated in the Beamr
-
-```erlang
-size(WASM) when is_pid(WASM) ->
- hb_beamr:wasm_send(WASM, {command, term_to_binary({size})}),
- receive
- {execution_result, Size} ->
- {ok, Size}
- end.
-```
-
-### write
-
-Write a binary to the Beamr instance's native memory at a given offset.
-
-```erlang
-write(WASM, Offset, Data)
- when is_pid(WASM)
- andalso is_binary(Data)
- andalso is_integer(Offset) ->
- ?event(writing_to_mem),
- hb_beamr:wasm_send(WASM, {command, term_to_binary({write, Offset, Data})}),
- ?event(mem_written),
- receive
- ok -> ok;
- {error, Error} -> {error, Error}
- end.
-```
-
-### write_string
-
-Simple helper function to allocate space for (via malloc) and write a
-
-```erlang
-write_string(WASM, Data) when is_pid(WASM) andalso is_list(Data) ->
- write_string(WASM, iolist_to_binary(Data));
-```
-
-### write_string
-
-Simple helper function to allocate space for (via malloc) and write a
-
-```erlang
-write_string(WASM, Data) when is_pid(WASM) andalso is_binary(Data) ->
- DataSize = byte_size(Data) + 1,
- String = <>,
- case malloc(WASM, DataSize) of
- {ok, Ptr} ->
- case write(WASM, Ptr, String) of
- ok -> {ok, Ptr};
- {error, Error} -> {error, Error}
- end;
- Error -> Error
- end.
-```
-
-### read
-
-Read a binary from the Beamr instance's native memory at a given offset
-
-```erlang
-read(WASM, Offset, Size)
- when is_pid(WASM)
- andalso is_integer(Offset)
- andalso is_integer(Size) ->
- ?event({read_request, {port, WASM}, {location, Offset}, {size, Size}}),
- hb_beamr:wasm_send(WASM, {command, term_to_binary({read, Offset, Size})}),
- ?event(read_req_sent),
- receive
- {execution_result, Result} ->
- ?event(
- {read_result,
- {wasm, WASM},
- {location, Offset},
- {size, Size},
- {result, Result}}),
- {ok, Result};
- {error, Error} ->
- {error, Error}
- end.
-```
-
-### read_string
-
-Simple helper function to read a string from the Beamr instance's native
-
-```erlang
-read_string(Port, Offset) -> read_string(Port, Offset, 8).
-```
-
-### read_string
-
-Simple helper function to read a string from the Beamr instance's native
-
-```erlang
-read_string(WASM, Offset, ChunkSize)
- when is_pid(WASM)
- andalso is_integer(Offset)
- andalso is_integer(ChunkSize) ->
- {ok, iolist_to_binary(do_read_string(WASM, Offset, ChunkSize))}.
-```
-
-### do_read_string
-
-```erlang
-do_read_string(WASM, Offset, ChunkSize) ->
- {ok, Data} = read(WASM, Offset, ChunkSize),
- case binary:split(Data, [<<0>>]) of
- [Data|[]] -> [Data|do_read_string(WASM, Offset + ChunkSize, ChunkSize)];
- [FinalData|_Remainder] -> [FinalData]
- end.
-```
-
-### malloc
-
-Allocate space for (via an exported malloc function from the WASM) in
-
-```erlang
-malloc(WASM, Size) when is_pid(WASM) andalso is_integer(Size) ->
- case hb_beamr:call(WASM, "malloc", [Size]) of
- {ok, [0]} ->
- ?event({malloc_failed, Size}),
- {error, malloc_failed};
- {ok, [Ptr]} ->
- ?event({malloc_success, Ptr, Size}),
- {ok, Ptr};
- {error, Error} ->
- {error, Error}
- end.
-```
-
-### free
-
-Free space allocated in the Beamr instance's native memory via a
-
-```erlang
-free(WASM, Ptr) when is_pid(WASM) andalso is_integer(Ptr) ->
- case hb_beamr:call(WASM, "free", [Ptr]) of
- {ok, Res} ->
- ?event({free_result, Res}),
- ok;
- {error, Error} ->
- {error, Error}
- end.
-```
-
-### size_test
-
-```erlang
-size_test() ->
- WASMPageSize = 65536,
- File1Pages = 1,
- File2Pages = 193,
- {ok, File} = file:read_file("test/test-print.wasm"),
- {ok, WASM, _Imports, _Exports} = hb_beamr:start(File),
- ?assertEqual({ok, WASMPageSize * File1Pages}, hb_beamr_io:size(WASM)),
- hb_beamr:stop(WASM),
- {ok, File2} = file:read_file("test/aos-2-pure-xs.wasm"),
- {ok, WASM2, _Imports2, _Exports2} = hb_beamr:start(File2),
- ?assertEqual({ok, WASMPageSize * File2Pages}, hb_beamr_io:size(WASM2)),
- hb_beamr:stop(WASM2).
-```
-
-### write_test
-
-Test writing memory in and out of bounds.
-
-```erlang
-write_test() ->
- % Load the `test-print' WASM module, which has a simple print function.
-```
-
-### read_test
-
-Test reading memory in and out of bounds.
-
-```erlang
-read_test() ->
- % Our `test-print' module is hand-written in WASM, so we know that it
- % has a `Hello, World!` string at precisely offset 66.
-```
-
-### malloc_test
-
-Test allocating and freeing memory.
-
-```erlang
-malloc_test() ->
- {ok, File} = file:read_file("test/test-calling.wasm"),
- {ok, WASM, _Imports, _Exports} = hb_beamr:start(File),
- % Check that we can allocate memory inside the bounds of the WASM module.
-```
-
-### string_write_and_read_test
-
-Write and read strings to memory.
-
-```erlang
-string_write_and_read_test() ->
- {ok, File} = file:read_file("test/test-calling.wasm"),
- {ok, WASM, _Imports, _Exports} = hb_beamr:start(File),
- {ok, Ptr} = write_string(WASM, <<"Hello, World!">>),
- ?assertEqual({ok, <<"Hello, World!">>}, read_string(WASM, Ptr)).
-```
-
----
-
-*Generated from [hb_beamr_io.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_beamr_io.erl)*
diff --git a/docs/book/src/hb_cache.erl.md b/docs/book/src/hb_cache.erl.md
deleted file mode 100644
index 356becb5d..000000000
--- a/docs/book/src/hb_cache.erl.md
+++ /dev/null
@@ -1,967 +0,0 @@
-# hb_cache
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_cache.erl)
-
-A cache of AO-Core protocol messages and compute results.
-HyperBEAM stores all paths in key value stores, abstracted by the `hb_store`
-module. Each store has its own storage backend, but each works with simple
-key-value pairs. Each store can write binary keys at paths, and link between
-paths.
-There are three layers to HyperBEAMs internal data representation on-disk:
-1. The raw binary data, written to the store at the hash of the content.
- Storing binary paths in this way effectively deduplicates the data.
-2. The hashpath-graph of all content, stored as a set of links between
- hashpaths, their keys, and the data that underlies them. This allows
- all messages to share the same hashpath space, such that all requests
- from users additively fill-in the hashpath space, minimizing duplicated
- compute.
-3. Messages, referrable by their IDs (committed or uncommitted). These are
- stored as a set of links commitment IDs and the uncommitted message.
-Before writing a message to the store, we convert it to Type-Annotated
-Binary Messages (TABMs), such that each of the keys in the message is
-either a map or a direct binary.
-Nested keys are lazily loaded from the stores, such that large deeply
-nested messages where only a small part of the data is actually used are
-not loaded into memory unnecessarily. In order to ensure that a message is
-loaded from the cache after a `read`, we can use the `ensure_loaded/1` and
-`ensure_all_loaded/1` functions. Ensure loaded will load the exact value
-that has been requested, while ensure all loaded will load the entire
-structure of the message into memory.
-Lazily loadable `links` are expressed as a tuple of the following form:
-`{link, ID, LinkOpts}`, where `ID` is the path to the data in the store,
-and `LinkOpts` is a map of suggested options to use when loading the data.
-In particular, this module ensures to stash the `store` option in `LinkOpts`,
-such that the `read` function can use the correct store without having to
-search unnecessarily. By providing an `Opts` argument to `ensure_loaded` or
-`ensure_all_loaded`, the caller can specify additional options to use when
-loading the data -- overriding the suggested options in the link.
-
----
-
-## Exported Functions
-
-- `ensure_all_loaded/1`
-- `ensure_all_loaded/2`
-- `ensure_loaded/1`
-- `ensure_loaded/2`
-- `link/3`
-- `list_numbered/2`
-- `list/2`
-- `match/2`
-- `read_resolved/3`
-- `read/2`
-- `test_signed/1`
-- `test_unsigned/1`
-- `write_binary/3`
-- `write_hashpath/2`
-- `write/2`
-
----
-
-### ensure_loaded
-
-A cache of AO-Core protocol messages and compute results.
-Ensure that a value is loaded from the cache if it is an ID or a link.
-
-```erlang
-ensure_loaded(Msg) ->
- ensure_loaded(Msg, #{}).
-```
-
-### ensure_loaded
-
-```erlang
-ensure_loaded(Msg, Opts) ->
- ensure_loaded([], Msg, Opts).
-```
-
-### ensure_loaded
-
-```erlang
-ensure_loaded(Ref, {Status, Msg}, Opts) when Status == ok; Status == error ->
- {Status, ensure_loaded(Ref, Msg, Opts)};
-```
-
-### ensure_loaded
-
-```erlang
-ensure_loaded(Ref,
- Lk = {link, ID, LkOpts = #{ <<"type">> := <<"link">>, <<"lazy">> := Lazy }},
- RawOpts) ->
- % The link is to a submessage; either in lazy (unresolved) form, or direct
- % form.
-```
-
-### ensure_loaded
-
-```erlang
-ensure_loaded(Ref, Link = {link, ID, LinkOpts = #{ <<"lazy">> := true }}, RawOpts) ->
- % If the user provided their own options, we merge them and _overwrite_
- % the options that are already set in the link.
-```
-
-### ensure_loaded
-
-```erlang
-ensure_loaded(Ref, {link, ID, LinkOpts}, Opts) ->
- ensure_loaded(Ref, {link, ID, LinkOpts#{ <<"lazy">> => true}}, Opts);
-```
-
-### ensure_loaded
-
-```erlang
-ensure_loaded(_Ref, Msg, _Opts) when not ?IS_LINK(Msg) ->
- Msg.
-```
-
-### report_ensure_loaded_not_found
-
-Report that a value was not found in the cache. If a key is provided,
-
-```erlang
-report_ensure_loaded_not_found(Ref, Lk, Opts) ->
- ?event(link_error, {link_not_resolvable, {ref, Ref}, {link, Lk}, {opts, Opts}}),
- throw(
- {necessary_message_not_found,
- hb_path:to_binary(lists:reverse(Ref)),
- hb_link:format_unresolved(Lk, Opts, 0)
- }
- ).
-```
-
-### ensure_all_loaded
-
-Ensure that all of the components of a message (whether a map, list,
-
-```erlang
-ensure_all_loaded(Msg) ->
- ensure_all_loaded(Msg, #{}).
-```
-
-### ensure_all_loaded
-
-```erlang
-ensure_all_loaded(Msg, Opts) ->
- ensure_all_loaded([], Msg, Opts).
-```
-
-### ensure_all_loaded
-
-```erlang
-ensure_all_loaded(Ref, Link, Opts) when ?IS_LINK(Link) ->
- ensure_all_loaded(Ref, ensure_loaded(Ref, Link, Opts), Opts);
-```
-
-### ensure_all_loaded
-
-```erlang
-ensure_all_loaded(Ref, Msg, Opts) when is_map(Msg) ->
- maps:map(fun(K, V) -> ensure_all_loaded([K|Ref], V, Opts) end, Msg);
-```
-
-### ensure_all_loaded
-
-```erlang
-ensure_all_loaded(Ref, Msg, Opts) when is_list(Msg) ->
- lists:map(
- fun({N, V}) -> ensure_all_loaded([N|Ref], V, Opts) end,
- hb_util:number(Msg)
- );
-```
-
-### ensure_all_loaded
-
-```erlang
-ensure_all_loaded(Ref, Msg, Opts) ->
- ensure_loaded(Ref, Msg, Opts).
-```
-
-### list_numbered
-
-List all items in a directory, assuming they are numbered.
-
-```erlang
-list_numbered(Path, Opts) ->
- SlotDir = hb_store:path(hb_opts:get(store, no_viable_store, Opts), Path),
- [ hb_util:int(Name) || Name <- list(SlotDir, Opts) ].
-```
-
-### list
-
-List all items under a given path.
-
-```erlang
-list(Path, Opts) when is_map(Opts) and not is_map_key(<<"store-module">>, Opts) ->
- case hb_opts:get(store, no_viable_store, Opts) of
- not_found -> [];
- Store ->
- list(Path, Store)
- end;
-```
-
-### list
-
-List all items under a given path.
-
-```erlang
-list(Path, Store) ->
- ResolvedPath = hb_store:resolve(Store, Path),
- case hb_store:list(Store, ResolvedPath) of
- {ok, Names} -> Names;
- {error, _} -> [];
- not_found -> []
- end.
-```
-
-### match
-
-Match a template message against the cache, returning a list of IDs
-
-```erlang
-match(MatchSpec, Opts) ->
- Spec = hb_message:convert(MatchSpec, tabm, <<"structured@1.0">>, Opts),
- ConvertedMatchSpec =
- maps:map(
- fun(_, Value) ->
- generate_binary_path(Value, Opts)
- end,
- maps:without([<<"ao-types">>], hb_ao:normalize_keys(Spec, Opts))
- ),
- case hb_store:match(hb_opts:get(store, no_viable_store, Opts), ConvertedMatchSpec) of
- {ok, Matches} -> {ok, Matches};
- _ -> not_found
- end.
-```
-
-### generate_binary_path
-
-Generate the path at which a binary value should be stored.
-Write a message to the cache. For raw binaries, we write the data at
-
-```erlang
-generate_binary_path(Bin, Opts) ->
- Hashpath = hb_path:hashpath(Bin, Opts),
- <<"data/", Hashpath/binary>>.
-```
-
-### write
-
-Generate the path at which a binary value should be stored.
-Write a message to the cache. For raw binaries, we write the data at
-
-```erlang
-write(RawMsg, Opts) when is_map(RawMsg) ->
- {ok, Msg} = hb_message:with_only_committed(RawMsg, Opts),
- TABM = hb_message:convert(Msg, tabm, <<"structured@1.0">>, Opts),
- ?event(debug_cache, {writing_full_message, {msg, TABM}}),
- try
- do_write_message(
- TABM,
- hb_opts:get(store, no_viable_store, Opts),
- Opts
- )
- catch
- Type:Reason:Stacktrace ->
- ?event(error,
- {cache_write_error,
- {type, Type},
- {reason, Reason},
- {stacktrace, {trace, Stacktrace}}
- },
- Opts
- ),
- erlang:raise(Type, Reason, Stacktrace)
- end;
-```
-
-### write
-
-Generate the path at which a binary value should be stored.
-Write a message to the cache. For raw binaries, we write the data at
-
-```erlang
-write(List, Opts) when is_list(List) ->
- write(hb_message:convert(List, tabm, <<"structured@1.0">>, Opts), Opts);
-```
-
-### write
-
-Generate the path at which a binary value should be stored.
-Write a message to the cache. For raw binaries, we write the data at
-
-```erlang
-write(Bin, Opts) when is_binary(Bin) ->
- do_write_message(Bin, hb_opts:get(store, no_viable_store, Opts), Opts).
-```
-
-### do_write_message
-
-```erlang
-do_write_message(Bin, Store, Opts) when is_binary(Bin) ->
- % Write the binary in the store at its calculated content-hash.
-```
-
-### do_write_message
-
-```erlang
-do_write_message(List, Store, Opts) when is_list(List) ->
- do_write_message(
- hb_message:convert(List, tabm, <<"structured@1.0">>, Opts),
- Store,
- Opts
- );
-```
-
-### do_write_message
-
-```erlang
-do_write_message(Msg, Store, Opts) when is_map(Msg) ->
- ?event(debug_cache, {writing_message, Msg}),
- % Calculate the IDs of the message.
-```
-
-### write_key
-
-Write a single key for a message into the store.
-
-```erlang
-write_key(Base, <<"commitments">>, _HPAlg, RawCommitments, Store, Opts) ->
- % The commitments are a special case: We calculate the single-part hashpath
- % for the `baseID/commitments` key, then write each commitment to the store
- % and link it to `baseCommHP/commitmentID`.
-```
-
-### write_key
-
-```erlang
-write_key(Base, Key, HPAlg, Value, Store, Opts) ->
- KeyHashPath =
- hb_path:hashpath(
- Base,
- hb_path:to_binary(Key),
- HPAlg,
- Opts
- ),
- {ok, Path} = do_write_message(Value, Store, Opts),
- hb_store:make_link(Store, Path, KeyHashPath),
- {ok, Path}.
-```
-
-### prepare_commitments
-
-The `structured@1.0` encoder does not typically encode `commitments`,
-
-```erlang
-prepare_commitments(RawCommitments, Opts) ->
- Commitments = ensure_all_loaded(RawCommitments, Opts),
- maps:map(
- fun(_, StructuredCommitment) ->
- hb_message:convert(StructuredCommitment, tabm, Opts)
- end,
- Commitments
- ).
-```
-
-### commitment_path
-
-Generate the commitment path for a given base path.
-Calculate the IDs for a message.
-
-```erlang
-commitment_path(Base, Opts) ->
- hb_path:hashpath(<- Arweave TX/ANS-104 ==> dev_codec_ans104:from/1 ==> TABM - HTTP Signed Message ==> dev_codec_httpsig_conv:from/1 ==> TABM - Flat Maps ==> dev_codec_flat:from/1 ==> TABM - TABM ==> dev_codec_structured:to/1 ==> AO-Core Message - AO-Core Message ==> dev_codec_structured:from/1 ==> TABM - TABM ==> dev_codec_ans104:to/1 ==> Arweave TX/ANS-104 - TABM ==> dev_codec_httpsig_conv:to/1 ==> HTTP Signed Message - TABM ==> dev_codec_flat:to/1 ==> Flat Maps - ... --Additionally, this module provides a number of utility functions for -manipulating messages. For example, `hb_message:sign/2` to sign a message of -arbitrary type, or `hb_formatter:format_msg/1` to print an AO-Core/TABM message in -a human-readable format. -The `hb_cache` module is responsible for storing and retrieving messages in -the HyperBEAM stores configured on the node. Each store has its own storage -backend, but each works with simple key-value pairs. Subsequently, the -`hb_cache` module uses TABMs as the internal format for storing and -retrieving messages. -Test vectors to ensure the functioning of this module and the codecs that -interact with it are found in `hb_message_test_vectors.erl`. - ---- - -## Exported Functions - -- `commit/2` -- `commit/3` -- `commitment_devices/2` -- `commitment/2` -- `commitment/3` -- `commitments/3` -- `committed/3` -- `convert/3` -- `convert/4` -- `default_tx_list/0` -- `diff/3` -- `filter_default_keys/1` -- `find_target/3` -- `id/1` -- `id/2` -- `id/3` -- `is_signed_key/3` -- `match/2` -- `match/3` -- `match/4` -- `minimize/1` -- `normalize_commitments/2` -- `print/1` -- `signers/2` -- `type/1` -- `uncommitted/1` -- `uncommitted/2` -- `verify/1` -- `verify/2` -- `verify/3` -- `with_commitments/3` -- `with_only_committed/2` -- `with_only_committers/2` -- `with_only_committers/3` -- `without_commitments/3` -- `without_unless_signed/3` - ---- - -### convert - -This module acts an adapter between messages, as modeled in the -Convert a message from one format to another. Taking a message in the - -```erlang -convert(Msg, TargetFormat, Opts) -> - convert(Msg, TargetFormat, <<"structured@1.0">>, Opts). -``` - -### convert - -This module acts an adapter between messages, as modeled in the -Convert a message from one format to another. Taking a message in the - -```erlang -convert(Msg, TargetFormat, tabm, Opts) -> - OldPriv = - if is_map(Msg) -> maps:get(<<"priv">>, Msg, #{}); - true -> #{} - end, - from_tabm(Msg, TargetFormat, OldPriv, Opts); -``` - -### convert - -This module acts an adapter between messages, as modeled in the -Convert a message from one format to another. Taking a message in the - -```erlang -convert(Msg, TargetFormat, SourceFormat, Opts) -> - OldPriv = - if is_map(Msg) -> maps:get(<<"priv">>, Msg, #{}); - true -> #{} - end, - TABM = - to_tabm( - case is_map(Msg) of - true -> hb_maps:without([<<"priv">>], Msg, Opts); - false -> Msg - end, - SourceFormat, - Opts - ), - case TargetFormat of - tabm -> restore_priv(TABM, OldPriv, Opts); - _ -> from_tabm(TABM, TargetFormat, OldPriv, Opts) - end. -``` - -### to_tabm - -```erlang -to_tabm(Msg, SourceFormat, Opts) -> - {SourceCodecMod, Params} = conversion_spec_to_req(SourceFormat, Opts), - % We use _from_ here because the codecs are labelled from the perspective - % of their own format. `dev_codec_ans104:from/1' will convert _from_ - % an ANS-104 message _into_ a TABM. -``` - -### from_tabm - -```erlang -from_tabm(Msg, TargetFormat, OldPriv, Opts) -> - {TargetCodecMod, Params} = conversion_spec_to_req(TargetFormat, Opts), - % We use the _to_ function here because each of the codecs we may call in - % this step are labelled from the perspective of the target format. For - % example, `dev_codec_httpsig:to/1' will convert _from_ a TABM to an - % HTTPSig message. -``` - -### restore_priv - -Add the existing `priv` sub-map back to a converted message, honoring - -```erlang -restore_priv(Msg, EmptyPriv, _Opts) when map_size(EmptyPriv) == 0 -> Msg; -``` - -### restore_priv - -Add the existing `priv` sub-map back to a converted message, honoring -Get a codec device and request params from the given conversion request. - -```erlang -restore_priv(Msg, OldPriv, Opts) -> - MsgPriv = hb_maps:get(<<"priv">>, Msg, #{}, Opts), - ?event({restoring_priv, {msg_priv, MsgPriv}, {old_priv, OldPriv}}), - NewPriv = hb_util:deep_merge(MsgPriv, OldPriv, Opts), - ?event({new_priv, NewPriv}), - Msg#{ <<"priv">> => NewPriv }. -``` - -### conversion_spec_to_req - -Add the existing `priv` sub-map back to a converted message, honoring -Get a codec device and request params from the given conversion request. - -```erlang -conversion_spec_to_req(Spec, Opts) when is_binary(Spec) or (Spec == tabm) -> - conversion_spec_to_req(#{ <<"device">> => Spec }, Opts); -``` - -### conversion_spec_to_req - -Add the existing `priv` sub-map back to a converted message, honoring -Get a codec device and request params from the given conversion request. - -```erlang -conversion_spec_to_req(Spec, Opts) -> - try - Device = - hb_maps:get( - <<"device">>, - Spec, - no_codec_device_in_conversion_spec, - Opts - ), - { - case Device of - tabm -> tabm; - _ -> - hb_ao:message_to_device( - #{ - <<"device">> => Device - }, - Opts - ) - end, - hb_maps:without([<<"device">>], Spec, Opts) - } - catch _:_ -> - throw({message_codec_not_extractable, Spec}) - end. -``` - -### id - -Return the ID of a message. - -```erlang -id(Msg) -> id(Msg, uncommitted). -``` - -### id - -Return the ID of a message. - -```erlang -id(Msg, Opts) when is_map(Opts) -> id(Msg, uncommitted, Opts); -``` - -### id - -Return the ID of a message. - -```erlang -id(Msg, Committers) -> id(Msg, Committers, #{}). -``` - -### id - -Return the ID of a message. - -```erlang -id(Msg, RawCommitters, Opts) -> - CommSpec = - case RawCommitters of - none -> #{ <<"committers">> => <<"none">> }; - uncommitted -> #{ <<"committers">> => <<"none">> }; - unsigned -> #{ <<"committers">> => <<"none">> }; - all -> #{ <<"committers">> => <<"all">> }; - signed -> #{ <<"committers">> => <<"all">> }; - List when is_list(List) -> #{ <<"committers">> => List } - end, - ?event({getting_id, {msg, Msg}, {spec, CommSpec}}), - {ok, ID} = - dev_message:id( - Msg, - CommSpec#{ <<"path">> => <<"id">> }, - Opts - ), - hb_util:human_id(ID). -``` - -### normalize_commitments - -Normalize the IDs in a message, ensuring that there is at least one - -```erlang -normalize_commitments(Msg, Opts) when is_map(Msg) -> - NormMsg = - maps:map( - fun(Key, Val) when Key == <<"commitments">> orelse Key == <<"priv">> -> - Val; - (_Key, Val) -> normalize_commitments(Val, Opts) - end, - Msg - ), - case hb_maps:get(<<"commitments">>, NormMsg, not_found, Opts) of - not_found -> - {ok, #{ <<"commitments">> := Commitments }} = - dev_message:commit( - NormMsg, - #{ <<"type">> => <<"unsigned">> }, - Opts - ), - NormMsg#{ <<"commitments">> => Commitments }; - _ -> NormMsg - end; -``` - -### normalize_commitments - -Normalize the IDs in a message, ensuring that there is at least one - -```erlang -normalize_commitments(Msg, Opts) when is_list(Msg) -> - lists:map(fun(X) -> normalize_commitments(X, Opts) end, Msg); -``` - -### normalize_commitments - -Normalize the IDs in a message, ensuring that there is at least one - -```erlang -normalize_commitments(Msg, _Opts) -> - Msg. -``` - -### with_only_committed - -Return a message with only the committed keys. If no commitments are - -```erlang -with_only_committed(Msg, Opts) when is_map(Msg) -> - ?event({with_only_committed, {msg, Msg}, {opts, Opts}}), - Comms = hb_maps:get(<<"commitments">>, Msg, not_found, Opts), - case is_map(Msg) andalso Comms /= not_found of - true -> - try - CommittedKeys = - hb_message:committed( - Msg, - #{ <<"commitments">> => <<"all">> }, - Opts - ), - % Add the ao-body-key to the committed list if it is not - % already present. -``` - -### with_only_committed - -```erlang -with_only_committed(Msg, _) -> - % If the message is not a map, it cannot be signed. -``` - -### with_links - -Filter keys from a map that do not match either the list of keys or - -```erlang -with_links(Keys, Map, Opts) -> - hb_maps:with( - Keys ++ - lists:map( - fun(Key) -> - <<(hb_link:remove_link_specifier(Key))/binary, "+link">> - end, - Keys - ), - Map, - Opts - ). -``` - -### with_only_committers - -Return the message with only the specified committers attached. - -```erlang -with_only_committers(Msg, Committers) -> - with_only_committers(Msg, Committers, #{}). -``` - -### with_only_committers - -```erlang -with_only_committers(Msg, Committers, Opts) when is_map(Msg) -> - NewCommitments = - hb_maps:filter( - fun(_, #{ <<"committer">> := Committer }) -> - lists:member(Committer, Committers); - (_, _) -> false - end, - hb_maps:get(<<"commitments">>, Msg, #{}, Opts), - Opts - ), - Msg#{ <<"commitments">> => NewCommitments }; -``` - -### with_only_committers - -```erlang -with_only_committers(Msg, _Committers, _Opts) -> - throw({unsupported_message_type, Msg}). -``` - -### is_signed_key - -Determine whether a specific key is part of a message's commitments. - -```erlang -is_signed_key(Key, Msg, Opts) -> - lists:member(Key, hb_message:committed(Msg, all, Opts)). -``` - -### without_unless_signed - -Remove the any of the given keys that are not signed from a message. - -```erlang -without_unless_signed(Key, Msg, Opts) when not is_list(Key) -> - without_unless_signed([Key], Msg, Opts); -``` - -### without_unless_signed - -Remove the any of the given keys that are not signed from a message. - -```erlang -without_unless_signed(Keys, Msg, Opts) -> - SignedKeys = hb_message:committed(Msg, all, Opts), - maps:without( - lists:filter(fun(K) -> not lists:member(K, SignedKeys) end, Keys), - Msg - ). -``` - -### commit - -Sign a message with the given wallet. - -```erlang -commit(Msg, WalletOrOpts) -> - commit( - Msg, - WalletOrOpts, - hb_opts:get( - commitment_device, - no_viable_commitment_device, - case is_map(WalletOrOpts) of - true -> WalletOrOpts; - false -> #{ priv_wallet => WalletOrOpts } - end - ) - ). -``` - -### commit - -```erlang -commit(Msg, Wallet, Format) when not is_map(Wallet) -> - commit(Msg, #{ priv_wallet => Wallet }, Format); -``` - -### commit - -```erlang -commit(Msg, Opts, CodecName) when is_binary(CodecName) -> - commit(Msg, Opts, #{ <<"commitment-device">> => CodecName }); -``` - -### commit - -```erlang -commit(Msg, Opts, Spec) -> - {ok, Signed} = - dev_message:commit( - Msg, - Spec#{ - <<"commitment-device">> => - case hb_maps:get(<<"commitment-device">>, Spec, none, Opts) of - none -> - case hb_maps:get(<<"device">>, Spec, none, Opts) of - none -> - throw( - { - no_commitment_device_in_codec_spec, - Spec - } - ); - Device -> Device - end; - CommitmentDevice -> CommitmentDevice - end - }, - Opts - ), - Signed. -``` - -### committed - -Return the list of committed keys from a message. - -```erlang -committed(Msg, all, Opts) -> - committed(Msg, #{ <<"committers">> => <<"all">> }, Opts); -``` - -### committed - -Return the list of committed keys from a message. - -```erlang -committed(Msg, none, Opts) -> - committed(Msg, #{ <<"committers">> => <<"none">> }, Opts); -``` - -### committed - -Return the list of committed keys from a message. - -```erlang -committed(Msg, List, Opts) when is_list(List) -> - committed(Msg, #{ <<"commitments">> => List }, Opts); -``` - -### committed - -Return the list of committed keys from a message. - -```erlang -committed(Msg, CommittersMsg, Opts) -> - ?event( - {committed, - {msg, {explicit, Msg}}, - {committers_msg, {explicit, CommittersMsg}}, - {opts, Opts} - } - ), - {ok, CommittedKeys} = dev_message:committed(Msg, CommittersMsg, Opts), - CommittedKeys. -``` - -### verify - -wrapper function to verify a message. - -```erlang -verify(Msg) -> verify(Msg, all). -``` - -### verify - -wrapper function to verify a message. - -```erlang -verify(Msg, Committers) -> - verify(Msg, Committers, #{}). -``` - -### verify - -```erlang -verify(Msg, all, Opts) -> - verify(Msg, <<"all">>, Opts); -``` - -### verify - -```erlang -verify(Msg, signers, Opts) -> - verify(Msg, hb_message:signers(Msg, Opts), Opts); -``` - -### verify - -```erlang -verify(Msg, Committers, Opts) when not is_map(Committers) -> - verify( - Msg, - #{ - <<"committers">> => - case ?IS_ID(Committers) of - true -> [Committers]; - false -> Committers - end - }, - Opts - ); -``` - -### verify - -```erlang -verify(Msg, Spec, Opts) -> - ?event(verify, {verify, {spec, Spec}}), - {ok, Res} = - dev_message:verify( - Msg, - Spec, - Opts - ), - Res. -``` - -### uncommitted - -Return the unsigned version of a message in AO-Core format. - -```erlang -uncommitted(Msg) -> uncommitted(Msg, #{}). -``` - -### uncommitted - -Return the unsigned version of a message in AO-Core format. - -```erlang -uncommitted(Bin, _Opts) when is_binary(Bin) -> Bin; -``` - -### uncommitted - -Return the unsigned version of a message in AO-Core format. -Return all of the committers on a message that have 'normal', 256 bit, - -```erlang -uncommitted(Msg, Opts) -> - hb_maps:remove(<<"commitments">>, Msg, Opts). -``` - -### signers - -Return the unsigned version of a message in AO-Core format. -Return all of the committers on a message that have 'normal', 256 bit, - -```erlang -signers(Msg, Opts) -> - hb_util:ok(dev_message:committers(Msg, #{}, Opts)). -``` - -### print - -Pretty-print a message. - -```erlang -print(Msg) -> print(Msg, 0). -``` - -### print - -Pretty-print a message. -Return the type of an encoded message. - -```erlang -print(Msg, Indent) -> - io:format(standard_error, "~s", [lists:flatten(hb_format:message(Msg, #{}, Indent))]). -``` - -### type - -Pretty-print a message. -Return the type of an encoded message. - -```erlang -type(TX) when is_record(TX, tx) -> tx; -``` - -### type - -Pretty-print a message. -Return the type of an encoded message. - -```erlang -type(Binary) when is_binary(Binary) -> binary; -``` - -### type - -Pretty-print a message. -Return the type of an encoded message. - -```erlang -type(Msg) when is_map(Msg) -> - IsDeep = lists:any( - fun({_, Value}) -> is_map(Value) end, - lists:filter( - fun({Key, _}) -> not hb_private:is_private(Key) end, - hb_maps:to_list(Msg) - ) - ), - case IsDeep of - true -> deep; - false -> shallow - end. -``` - -### match - -Check if two maps match, including recursively checking nested maps. - -```erlang -match(Map1, Map2) -> - match(Map1, Map2, strict). -``` - -### match - -```erlang -match(Map1, Map2, Mode) -> - match(Map1, Map2, Mode, #{}). -``` - -### match - -```erlang -match(Map1, Map2, Mode, Opts) -> - try unsafe_match(Map1, Map2, Mode, [], Opts) - catch _:Details -> Details - end. -``` - -### unsafe_match - -Match two maps, returning `true` if they match, or throwing an error - -```erlang -unsafe_match(Map1, Map2, Mode, Path, Opts) -> - Keys1 = - hb_maps:keys( - NormMap1 = hb_util:lower_case_key_map(minimize( - normalize(hb_ao:normalize_keys(Map1, Opts), Opts), - [<<"content-type">>, <<"ao-body-key">>] - ), Opts) - ), - Keys2 = - hb_maps:keys( - NormMap2 = hb_util:lower_case_key_map(minimize( - normalize(hb_ao:normalize_keys(Map2, Opts), Opts), - [<<"content-type">>, <<"ao-body-key">>] - ), Opts) - ), - PrimaryKeysPresent = - (Mode == primary) andalso - lists:all( - fun(Key) -> lists:member(Key, Keys1) end, - Keys1 - ), - ?event(match, - {match, - {keys1, Keys1}, - {keys2, Keys2}, - {mode, Mode}, - {primary_keys_present, PrimaryKeysPresent}, - {msg1, Map1}, - {msg2, Map2} - } - ), - case (Keys1 == Keys2) or (Mode == only_present) or PrimaryKeysPresent of - true -> - lists:all( - fun(Key) -> - ?event(match, {matching_key, Key}), - Val1 = - hb_ao:normalize_keys( - hb_maps:get(Key, NormMap1, not_found, Opts), - Opts - ), - Val2 = - hb_ao:normalize_keys( - hb_maps:get(Key, NormMap2, not_found, Opts), - Opts - ), - BothPresent = (Val1 =/= not_found) and (Val2 =/= not_found), - case (not BothPresent) and (Mode == only_present) of - true -> true; - false -> - case is_map(Val1) andalso is_map(Val2) of - true -> - unsafe_match(Val1, Val2, Mode, Path ++ [Key], Opts); - false -> - case {Val1, Val2} of - {V, V} -> true; - {V, '_'} when V =/= not_found -> true; - {'_', V} when V =/= not_found -> true; - {'_', '_'} -> true; - _ -> - throw( - {value_mismatch, - hb_format:short_id( - hb_path:to_binary( - Path ++ [Key] - ) - ), - {val1, Val1}, - {val2, Val2} - } - ) - end - end - end - end, - Keys1 - ); - false -> - throw( - {keys_mismatch, - {path, hb_format:short_id(hb_path:to_binary(Path))}, - {keys1, Keys1}, - {keys2, Keys2} - } - ) - end. -``` - -### matchable_keys - -```erlang -matchable_keys(Map) -> - lists:sort(lists:map(fun hb_ao:normalize_key/1, hb_maps:keys(Map))). -``` - -### diff - -Return the numeric differences between two messages, matching deeply - -```erlang -diff(Msg1, Msg2, Opts) when is_map(Msg1) andalso is_map(Msg2) -> - maps:filtermap( - fun(Key, Val2) -> - case hb_maps:get(Key, Msg1, not_found, Opts) of - Val2 -> - % The key is present in both maps, and the values match. -``` - -### diff - -```erlang -diff(_Val1, _Val2, _Opts) -> - not_found. -``` - -### with_commitments - -Filter messages that do not match the 'spec' given. The underlying match - -```erlang -with_commitments(ID, Msg, Opts) when ?IS_ID(ID) -> - with_commitments([ID], Msg, Opts); -``` - -### with_commitments - -Filter messages that do not match the 'spec' given. The underlying match - -```erlang -with_commitments(Spec, Msg = #{ <<"commitments">> := Commitments }, Opts) -> - ?event({with_commitments, {spec, Spec}, {commitments, Commitments}}), - FilteredCommitments = - hb_maps:filter( - fun(ID, CommMsg) -> - if is_list(Spec) -> - lists:member(ID, Spec); - is_map(Spec) -> - match(Spec, CommMsg, primary, Opts) == true - end - end, - Commitments, - Opts - ), - ?event({with_commitments, {filtered_commitments, FilteredCommitments}}), - Msg#{ <<"commitments">> => FilteredCommitments }; -``` - -### with_commitments - -Filter messages that do not match the 'spec' given. The underlying match - -```erlang -with_commitments(_Spec, Msg, _Opts) -> - Msg. -``` - -### without_commitments - -Filter messages that match the 'spec' given. Inverts the `with_commitments/2` - -```erlang -without_commitments(Spec, Msg = #{ <<"commitments">> := Commitments }, Opts) -> - ?event({without_commitments, {spec, Spec}, {msg, Msg}, {commitments, Commitments}}), - FilteredCommitments = - hb_maps:without( - hb_maps:keys( - hb_maps:get( - <<"commitments">>, - with_commitments(Spec, Msg, Opts), - #{}, - Opts - ) - ), - Commitments - ), - ?event({without_commitments, {filtered_commitments, FilteredCommitments}}), - Msg#{ <<"commitments">> => FilteredCommitments }; -``` - -### without_commitments - -Filter messages that match the 'spec' given. Inverts the `with_commitments/2` - -```erlang -without_commitments(_Spec, Msg, _Opts) -> - Msg. -``` - -### commitment - -Extract a commitment from a message given a `committer` or `commitment` - -```erlang -commitment(ID, Msg) -> - commitment(ID, Msg, #{}). -``` - -### commitment - -```erlang -commitment(ID, Link, Opts) when ?IS_LINK(Link) -> - commitment(ID, hb_cache:ensure_loaded(Link, Opts), Opts); -``` - -### commitment - -```erlang -commitment(ID, #{ <<"commitments">> := Commitments }, Opts) - when is_binary(ID), is_map_key(ID, Commitments) -> - hb_maps:get( - ID, - Commitments, - not_found, - Opts - ); -``` - -### commitment - -```erlang -commitment(Spec, Msg, Opts) -> - Matches = commitments(Spec, Msg, Opts), - ?event(debug_commitment, {commitment, {spec, Spec}, {matches, Matches}}), - if - map_size(Matches) == 0 -> not_found; - map_size(Matches) == 1 -> - CommID = hd(hb_maps:keys(Matches)), - {ok, CommID, hb_util:ok(hb_maps:find(CommID, Matches, Opts))}; - true -> - ?event(commitment, {multiple_matches, {matches, Matches}}), - multiple_matches - end; -``` - -### commitment - -```erlang -commitment(_Spec, _Msg, _Opts) -> - % The message has no commitments, so the spec can never match. -``` - -### commitments - -Return a list of all commitments that match the spec. - -```erlang -commitments(ID, Link, Opts) when ?IS_LINK(Link) -> - commitments(ID, hb_cache:ensure_loaded(Link, Opts), Opts); -``` - -### commitments - -Return a list of all commitments that match the spec. - -```erlang -commitments(CommitterID, Msg, Opts) when is_binary(CommitterID) -> - commitments(#{ <<"committer">> => CommitterID }, Msg, Opts); -``` - -### commitments - -Return a list of all commitments that match the spec. - -```erlang -commitments(Spec, #{ <<"commitments">> := Commitments }, Opts) -> - hb_maps:filtermap( - fun(_ID, CommMsg) -> - case match(Spec, CommMsg, primary, Opts) of - true -> {true, CommMsg}; - _ -> false - end - end, - Commitments, - Opts - ); -``` - -### commitments - -Return a list of all commitments that match the spec. - -```erlang -commitments(_Spec, _Msg, _Opts) -> - #{}. -``` - -### commitment_devices - -Return the devices for which there are commitments on a message. - -```erlang -commitment_devices(#{ <<"commitments">> := Commitments }, Opts) -> - lists:map( - fun(CommMsg) -> - hb_ao:get(<<"commitment-device">>, CommMsg, Opts) - end, - maps:values(Commitments) - ); -``` - -### commitment_devices - -Return the devices for which there are commitments on a message. - -```erlang -commitment_devices(_Msg, _Opts) -> - []. -``` - -### find_target - -Implements a standard pattern in which the target for an operation is - -```erlang -find_target(Self, Req, Opts) -> - GetOpts = Opts#{ - hashpath => ignore, - cache_control => [<<"no-cache">>, <<"no-store">>] - }, - {ok, - case hb_maps:get(<<"target">>, Req, <<"self">>, GetOpts) of - <<"self">> -> Self; - Key -> - hb_maps:get( - Key, - Req, - hb_maps:get(<<"body">>, Req, GetOpts), - GetOpts - ) - end - }. -``` - -### minimize - -Remove keys from the map that can be regenerated. Optionally takes an - -```erlang -minimize(Msg) -> minimize(Msg, []). -``` - -### minimize - -Remove keys from the map that can be regenerated. Optionally takes an - -```erlang -minimize(RawVal, _) when not is_map(RawVal) -> RawVal; -``` - -### minimize - -Remove keys from the map that can be regenerated. Optionally takes an - -```erlang -minimize(Map, ExtraKeys) -> - NormKeys = - lists:map(fun hb_ao:normalize_key/1, ?REGEN_KEYS) - ++ lists:map(fun hb_ao:normalize_key/1, ExtraKeys), - maps:filter( - fun(Key, _) -> - (not lists:member(hb_ao:normalize_key(Key), NormKeys)) - andalso (not hb_private:is_private(Key)) - end, - maps:map(fun(_K, V) -> minimize(V) end, Map) - ). -``` - -### normalize - -Return a map with only the keys that necessary, without those that can - -```erlang -normalize(Map, Opts) when is_map(Map) orelse is_list(Map) -> - NormalizedMap = hb_ao:normalize_keys(Map, Opts), - FilteredMap = filter_default_keys(NormalizedMap), - hb_maps:with(matchable_keys(FilteredMap), FilteredMap); -``` - -### normalize - -Return a map with only the keys that necessary, without those that can - -```erlang -normalize(Other, _Opts) -> - Other. -``` - -### filter_default_keys - -Remove keys from a map that have the default values found in the tx - -```erlang -filter_default_keys(Map) -> - DefaultsMap = default_tx_message(), - maps:filter( - fun(Key, Value) -> - case hb_maps:find(hb_ao:normalize_key(Key), DefaultsMap) of - {ok, Value} -> false; - _ -> true - end - end, - Map - ). -``` - -### default_tx_message - -Get the normalized fields and default values of the tx record. - -```erlang -default_tx_message() -> - hb_maps:from_list(default_tx_list()). -``` - -### default_tx_list - -Get the ordered list of fields as AO-Core keys and default values of - -```erlang -default_tx_list() -> - Keys = lists:map(fun hb_ao:normalize_key/1, record_info(fields, tx)), -``` - ---- - -*Generated from [hb_message.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_message.erl)* diff --git a/docs/book/src/hb_message_test_vectors.erl.md b/docs/book/src/hb_message_test_vectors.erl.md deleted file mode 100644 index 36e40e41e..000000000 --- a/docs/book/src/hb_message_test_vectors.erl.md +++ /dev/null @@ -1,1736 +0,0 @@ -# hb_message_test_vectors - -[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_message_test_vectors.erl) - -A battery of test vectors for message codecs, implementing the -`message@1.0` encoding and commitment APIs. Additionally, this module -houses tests that ensure the general functioning of the `hb_message` API. - ---- - -### run_test - -A battery of test vectors for message codecs, implementing the -Test invocation function, making it easier to run a specific test. - -```erlang -run_test() -> - hb:init(), - nested_structured_fields_test( - #{ <<"device">> => <<"json@1.0">>, <<"bundle">> => true }, - test_opts(normal) - ). -``` - -### test_codecs - -Return a list of codecs to test. Disable these as necessary if you need - -```erlang -test_codecs() -> - [ - <<"structured@1.0">>, - <<"httpsig@1.0">>, - #{ <<"device">> => <<"httpsig@1.0">>, <<"bundle">> => true }, - <<"flat@1.0">>, - <<"ans104@1.0">>, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, - <<"json@1.0">>, - #{ <<"device">> => <<"json@1.0">>, <<"bundle">> => true } - ]. -``` - -### suite_test_opts - -Return a set of options for testing, taking the codec name as an - -```erlang -suite_test_opts() -> - [ - #{ - name => normal, - desc => <<"Default opts">>, - opts => test_opts(normal) - } - ]. -``` - -### suite_test_opts - -```erlang -suite_test_opts(OptsName) -> - [ O || O = #{ name := OName } <- suite_test_opts(), OName == OptsName ]. -``` - -### test_opts - -```erlang -test_opts(normal) -> - #{ - store => hb_test_utils:test_store(), - priv_wallet => hb:wallet() - }. -``` - -### test_suite - -```erlang -test_suite() -> - [ - % Basic operations - {<<"Binary to binary">>, - fun binary_to_binary_test/2}, - {<<"Match">>, - fun match_test/2}, - {<<"Basic message encoding and decoding">>, - fun basic_message_codec_test/2}, - {<<"Priv survives conversion">>, - fun priv_survives_conversion_test/2}, - {<<"Message with body">>, - fun set_body_codec_test/2}, - {<<"Message with large keys">>, - fun message_with_large_keys_test/2}, - {<<"Structured field atom parsing">>, - fun structured_field_atom_parsing_test/2}, - {<<"Structured field decimal parsing">>, - fun structured_field_decimal_parsing_test/2}, - {<<"Unsigned id">>, - fun unsigned_id_test/2}, - % Nested structures - {<<"Simple nested message">>, - fun simple_nested_message_test/2}, - {<<"Message with simple embedded list">>, - fun message_with_simple_embedded_list_test/2}, - {<<"Nested empty map">>, - fun nested_empty_map_test/2}, - {<<"Empty body">>, - fun empty_body_test/2}, - {<<"Nested structured fields">>, - fun nested_structured_fields_test/2}, - {<<"Single layer message to encoding">>, - fun single_layer_message_to_encoding_test/2}, - {<<"Nested body list">>, - fun nested_body_list_test/2}, - {<<"Empty string in nested tag">>, - fun empty_string_in_nested_tag_test/2}, - {<<"Deep typed message ID">>, - fun deep_typed_message_id_test/2}, - {<<"Encode small balance table">>, - fun encode_small_balance_table_test/2}, - {<<"Encode large balance table">>, - fun encode_large_balance_table_test/2}, - {<<"Normalize commitments">>, - fun normalize_commitments_test/2}, - % Signed messages - {<<"Signed message to message and back">>, - fun signed_message_encode_decode_verify_test/2}, - {<<"Specific order signed message">>, - fun specific_order_signed_message_test/2}, - {<<"Specific order deeply nested signed message">>, - fun specific_order_deeply_nested_signed_message_test/2}, - {<<"Signed only committed data field">>, - fun signed_only_committed_data_field_test/2}, - {<<"Signed simple nested message">>, - fun simple_signed_nested_message_test/2}, - {<<"Signed nested message">>, - fun signed_nested_message_with_child_test/2}, - {<<"Committed keys">>, - fun committed_keys_test/2}, - {<<"Committed empty keys">>, - fun committed_empty_keys_test/2}, - {<<"Signed list HTTP response">>, - fun signed_list_test/2}, - {<<"Sign node message">>, - fun sign_node_message_test/2}, - {<<"Complex signed message">>, - fun complex_signed_message_test/2}, - {<<"Nested message with large keys">>, - fun nested_message_with_large_keys_test/2}, - {<<"Signed nested complex signed message">>, - fun verify_nested_complex_signed_test/2}, - % Complex structures - {<<"Nested message with large keys and content">>, - fun nested_message_with_large_keys_and_content_test/2}, - {<<"Nested message with large content">>, - fun nested_message_with_large_content_test/2}, - {<<"Deeply nested message with content">>, - fun deeply_nested_message_with_content_test/2}, - {<<"Deeply nested message with only content">>, - fun deeply_nested_message_with_only_content/2}, - {<<"Signed deep serialize and deserialize">>, - fun signed_deep_message_test/2}, - {<<"Signed nested data key">>, - fun signed_nested_data_key_test/2}, - {<<"Signed message with hashpath">>, - fun hashpath_sign_verify_test/2}, - {<<"Message with derived components">>, - fun signed_message_with_derived_components_test/2}, - {<<"Large body committed keys">>, - fun large_body_committed_keys_test/2}, - {<<"Signed with inner signed">>, - fun signed_with_inner_signed_message_test/2}, - {<<"Recursive nested list">>, - fun recursive_nested_list_test/2}, - {<<"Sign links">>, - fun sign_links_test/2}, - {<<"ID of linked message">>, - fun id_of_linked_message_test/2}, - {<<"Sign deep message from lazy cache read">>, - fun sign_deep_message_from_lazy_cache_read_test/2}, - {<<"ID of deep message and link message match">>, - fun id_of_deep_message_and_link_message_match_test/2}, - {<<"Signed non-bundle is bundlable">>, - fun signed_non_bundle_is_bundlable_test/2}, - {<<"Bundled ordering">>, - fun bundled_ordering_test/2}, - {<<"Codec round-trip conversion is idempotent">>, - fun codec_roundtrip_conversion_is_idempotent_test/2}, - {<<"Bundled and unbundled IDs differ">>, - fun bundled_and_unbundled_ids_differ_test/2}, - {<<"Tabm conversion is idempotent">>, - fun tabm_conversion_is_idempotent_test/2} - ]. -``` - -### suite_test_ - -Organizes a test battery for the `hb_message` module and its codecs. - -```erlang -suite_test_() -> - hb_test_utils:suite_with_opts( - codec_test_suite( - test_codecs(), - normal - ), - suite_test_opts(normal) - ). -``` - -### codec_test_suite - -Run the test suite for a set of codecs, using the given options type. - -```erlang -codec_test_suite(Codecs, OptsType) -> - lists:flatmap( - fun(CodecName) -> - lists:map(fun({Desc, Test}) -> - TestName = - binary_to_list( - << (suite_name(CodecName))/binary, ": ", Desc/binary >> - ), - TestSpecificOpts = test_opts(OptsType), - { - Desc, - TestName, - fun(_SuiteOpts) -> Test(CodecName, TestSpecificOpts) end - } - end, test_suite()) - end, - Codecs - ). -``` - -### suite_name - -Create a name for a suite from a codec spec. - -```erlang -suite_name(CodecSpec) when is_binary(CodecSpec) -> CodecSpec; -``` - -### suite_name - -Create a name for a suite from a codec spec. - -```erlang -suite_name(CodecSpec) when is_map(CodecSpec) -> - CodecName = maps:get(<<"device">>, CodecSpec, <<"[! NO CODEC !]">>), - case maps:get(<<"bundle">>, CodecSpec, false) of - false -> CodecName; - true -> << CodecName/binary, " (bundle)">> - end. -``` - -### is_idempotent - -Tests a message transforming function to ensure that it is idempotent. - -```erlang -is_idempotent(Func, Msg, Opts) -> - Run = fun(M) -> case Func(M) of {ok, Res} -> Res; Res -> Res end end, - After1 = Run(Msg), - After2 = Run(After1), - After3 = Run(After2), - MatchRes1 = hb_message:match(After1, After2, strict, Opts), - MatchRes2 = hb_message:match(After2, After3, strict, Opts), - ?event({is_idempotent, {match_res1, MatchRes1}, {match_res2, MatchRes2}}), - MatchRes1 andalso MatchRes2. -``` - -### tabm_conversion_is_idempotent_test - -Ensure that converting a message to/from TABM multiple times repeatedly - -```erlang -tabm_conversion_is_idempotent_test(_Codec, Opts) -> - From = fun(M) -> hb_message:convert(M, <<"structured@1.0">>, tabm, Opts) end, - To = fun(M) -> hb_message:convert(M, tabm, <<"structured@1.0">>, Opts) end, - SimpleMsg = #{ <<"a">> => <<"x">>, <<"b">> => <<"y">>, <<"c">> => <<"z">> }, - ComplexMsg = - #{ - <<"path">> => <<"schedule">>, - <<"method">> => <<"POST">>, - <<"body">> => - Signed = hb_message:commit( - #{ - <<"type">> => <<"Message">>, - <<"function">> => <<"fac">>, - <<"parameters">> => #{ - <<"a">> => 1 - }, - <<"content-type">> => <<"application/html">>, - <<"body">> => - << - """ - -
- Msg1.HashPath = Msg1.ID
- Msg3.HashPath = Msg1.Hash(Msg1.HashPath, Msg2.ID)
- Msg3.{...} = AO-Core.apply(Msg1, Msg2)
- ...
-
-A message's ID itself includes its HashPath, leading to the mixing of
-a Msg2's merkle list into the resulting Msg3's HashPath. This allows a single
-message to represent a history _tree_ of all of the messages that were
-applied to generate it -- rather than just a linear history.
-A message may also specify its own algorithm for generating its HashPath,
-which allows for custom logic to be used for representing the history of a
-message. When Msg2's are applied to a Msg1, the resulting Msg3's HashPath
-will be generated according to Msg1's algorithm choice.
-
----
-
-## Exported Functions
-
-- `from_message/3`
-- `hashpath_alg/2`
-- `hashpath/2`
-- `hashpath/3`
-- `hashpath/4`
-- `hd/2`
-- `matches/2`
-- `normalize/1`
-- `pop_request/2`
-- `priv_remaining/2`
-- `priv_store_remaining/2`
-- `priv_store_remaining/3`
-- `push_request/2`
-- `push_request/3`
-- `queue_request/2`
-- `queue_request/3`
-- `regex_matches/2`
-- `term_to_path_parts/1`
-- `term_to_path_parts/2`
-- `tl/2`
-- `to_binary/1`
-- `verify_hashpath/2`
-
----
-
-### hd
-
-This module provides utilities for manipulating the paths of a
-Extract the first key from a `Message2`'s `Path` field.
-
-```erlang
-hd(Msg2, Opts) ->
- %?event({key_from_path, Msg2, Opts}),
- case pop_request(Msg2, Opts) of
- undefined -> undefined;
- {Head, _} ->
- % `term_to_path' returns the full path, so we need to take the
- % `hd' of our `Head'.
-```
-
-### tl
-
-Return the message without its first path element. Note that this
-
-```erlang
-tl(Msg2, Opts) when is_map(Msg2) ->
- case pop_request(Msg2, Opts) of
- undefined -> undefined;
- {_, Rest} -> Rest
- end;
-```
-
-### tl
-
-Return the message without its first path element. Note that this
-
-```erlang
-tl(Path, Opts) when is_list(Path) ->
- case tl(#{ <<"path">> => Path }, Opts) of
- [] -> undefined;
- undefined -> undefined;
- #{ <<"path">> := Rest } -> Rest
- end.
-```
-
-### priv_remaining
-
-Return the `Remaining-Path` of a message, from its hidden `AO-Core`
-Store the remaining path of a message in its hidden `AO-Core` key.
-
-```erlang
-priv_remaining(Msg, Opts) ->
- Priv = hb_private:from_message(Msg),
- AOCore = hb_maps:get(<<"ao-core">>, Priv, #{}, Opts),
- hb_maps:get(<<"remaining">>, AOCore, undefined, Opts).
-```
-
-### priv_store_remaining
-
-Return the `Remaining-Path` of a message, from its hidden `AO-Core`
-Store the remaining path of a message in its hidden `AO-Core` key.
-
-```erlang
-priv_store_remaining(Msg, RemainingPath) ->
- priv_store_remaining(Msg, RemainingPath, #{}).
-```
-
-### priv_store_remaining
-
-```erlang
-priv_store_remaining(Msg, RemainingPath, Opts) ->
- Priv = hb_private:from_message(Msg),
- AOCore = hb_maps:get(<<"ao-core">>, Priv, #{}, Opts),
- Msg#{
- <<"priv">> =>
- Priv#{
- <<"ao-core">> =>
- AOCore#{
- <<"remaining">> => RemainingPath
- }
- }
- }.
-```
-
-### hashpath
-
-Add an ID of a Msg2 to the HashPath of another message.
-
-```erlang
-hashpath(Bin, _Opts) when is_binary(Bin) ->
- % Default hashpath for a binary message is its SHA2-256 hash.
-```
-
-### hashpath
-
-```erlang
-hashpath(RawMsg1, Opts) ->
- Msg1 = hb_ao:normalize_keys(RawMsg1, Opts),
- case hb_private:from_message(Msg1) of
- #{ <<"hashpath">> := HP } -> HP;
- _ ->
- % Note: We do not use `hb_message:id' here because it will call
- % hb_ao:resolve, which will call `hashpath' recursively.
-```
-
-### hashpath
-
-```erlang
-hashpath(Msg1, Msg2, Opts) when is_map(Msg1) ->
- Msg1Hashpath = hashpath(Msg1, Opts),
- HashpathAlg = hashpath_alg(Msg1, Opts),
- hashpath(Msg1Hashpath, Msg2, HashpathAlg, Opts);
-```
-
-### hashpath
-
-```erlang
-hashpath(Msg1, Msg2, Opts) ->
- throw({hashpath_not_viable, Msg1, Msg2, Opts}).
-```
-
-### hashpath
-
-```erlang
-hashpath(Msg1, Msg2, HashpathAlg, Opts) when is_map(Msg2) ->
- Msg2WithoutMeta = hb_maps:without(?AO_CORE_KEYS, Msg2, Opts),
- ReqPath = from_message(request, Msg2, Opts),
- case {map_size(Msg2WithoutMeta), ReqPath} of
- {0, _} when ReqPath =/= undefined ->
- hashpath(Msg1, to_binary(hd(ReqPath)), HashpathAlg, Opts);
- _ ->
- {ok, Msg2ID} =
- dev_message:id(
- Msg2,
- #{ <<"commitments">> => <<"all">> },
- Opts
- ),
- hashpath(Msg1, hb_util:human_id(Msg2ID), HashpathAlg, Opts)
- end;
-```
-
-### hashpath
-
-```erlang
-hashpath(Msg1Hashpath, HumanMsg2ID, HashpathAlg, Opts) ->
- ?event({hashpath, {msg1hp, {explicit, Msg1Hashpath}}, {msg2id, {explicit, HumanMsg2ID}}}),
- HP =
- case term_to_path_parts(Msg1Hashpath, Opts) of
- [_] ->
- << Msg1Hashpath/binary, "/", HumanMsg2ID/binary >>;
- [Prev1, Prev2] ->
- % Calculate the new base of the hashpath. We check whether the key is
- % a human-readable binary ID, or a path part, and convert or pass
- % through accordingly.
-```
-
-### hashpath_alg
-
-Get the hashpath function for a message from its HashPath-Alg.
-
-```erlang
-hashpath_alg(Msg, Opts) ->
- case dev_message:get(<<"hashpath-alg">>, Msg, Opts) of
- {ok, <<"sha-256-chain">>} ->
- fun hb_crypto:sha256_chain/2;
- {ok, <<"accumulate-256">>} ->
- fun hb_crypto:accumulate/2;
- {error, not_found} ->
- fun hb_crypto:sha256_chain/2
- end.
-```
-
-### push_request
-
-Add a message to the head (next to execute) of a request path.
-
-```erlang
-push_request(Msg, Path) ->
- push_request(Msg, Path, #{}).
-```
-
-### push_request
-
-Pop the next element from a request path or path list.
-
-```erlang
-push_request(Msg, Path, Opts) ->
- hb_maps:put(<<"path">>, term_to_path_parts(Path, Opts) ++ from_message(request, Msg, Opts), Msg, Opts).
-```
-
-### pop_request
-
-Pop the next element from a request path or path list.
-
-```erlang
-pop_request(undefined, _Opts) -> undefined;
-```
-
-### pop_request
-
-Pop the next element from a request path or path list.
-
-```erlang
-pop_request(Msg, Opts) when is_map(Msg) ->
- %?event({popping_request, {msg, Msg}, {opts, Opts}}),
- case pop_request(from_message(request, Msg, Opts), Opts) of
- undefined -> undefined;
- {undefined, _} -> undefined;
- {Head, []} -> {Head, undefined};
- {Head, Rest} ->
- ?event({popped_request, Head, Rest}),
- {Head, hb_maps:put(<<"path">>, Rest, Msg, Opts)}
- end;
-```
-
-### pop_request
-
-Pop the next element from a request path or path list.
-
-```erlang
-pop_request([], _Opts) -> undefined;
-```
-
-### pop_request
-
-Pop the next element from a request path or path list.
-
-```erlang
-pop_request([Head|Rest], _Opts) ->
- {Head, Rest}.
-```
-
-### queue_request
-
-Queue a message at the back of a request path. `path` is the only
-
-```erlang
-queue_request(Msg, Path) ->
- queue_request(Msg, Path, #{}).
-```
-
-### queue_request
-
-Verify the HashPath of a message, given a list of messages that
-
-```erlang
-queue_request(Msg, Path, Opts) ->
- hb_maps:put(<<"path">>, from_message(request, Msg, Opts) ++ term_to_path_parts(Path), Msg, Opts).
-```
-
-### verify_hashpath
-
-Verify the HashPath of a message, given a list of messages that
-
-```erlang
-verify_hashpath([Msg1, Msg2, Msg3|Rest], Opts) ->
- CorrectHashpath = hashpath(Msg1, Msg2, Opts),
- FromMsg3 = from_message(hashpath, Msg3, Opts),
- CorrectHashpath == FromMsg3 andalso
- case Rest of
- [] -> true;
- _ -> verify_hashpath([Msg2, Msg3|Rest], Opts)
- end.
-```
-
-### from_message
-
-Extract the request path or hashpath from a message. We do not use
-
-```erlang
-from_message(Type, Link, Opts) when ?IS_LINK(Link) ->
- from_message(Type, hb_cache:ensure_loaded(Link, Opts), Opts);
-```
-
-### from_message
-
-Extract the request path or hashpath from a message. We do not use
-
-```erlang
-from_message(hashpath, Msg, Opts) -> hashpath(Msg, Opts);
-```
-
-### from_message
-
-Extract the request path or hashpath from a message. We do not use
-
-```erlang
-from_message(request, #{ path := Path }, Opts) -> term_to_path_parts(Path, Opts);
-```
-
-### from_message
-
-Extract the request path or hashpath from a message. We do not use
-
-```erlang
-from_message(request, #{ <<"path">> := Path }, Opts) -> term_to_path_parts(Path, Opts);
-```
-
-### from_message
-
-Extract the request path or hashpath from a message. We do not use
-
-```erlang
-from_message(request, #{ <<"Path">> := Path }, Opts) -> term_to_path_parts(Path, Opts);
-```
-
-### from_message
-
-Extract the request path or hashpath from a message. We do not use
-Convert a term into an executable path. Supports binaries, lists, and
-
-```erlang
-from_message(request, _, _Opts) -> undefined.
-```
-
-### term_to_path_parts
-
-Extract the request path or hashpath from a message. We do not use
-Convert a term into an executable path. Supports binaries, lists, and
-
-```erlang
-term_to_path_parts(Path) ->
- term_to_path_parts(Path, #{ error_strategy => throw }).
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts(Link, Opts) when ?IS_LINK(Link) ->
- term_to_path_parts(hb_cache:ensure_loaded(Link, Opts), Opts);
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts([], _Opts) -> undefined;
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts(<<>>, _Opts) -> undefined;
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts(<<"/">>, _Opts) -> [];
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts(Binary, Opts) when is_binary(Binary) ->
- case binary:match(Binary, <<"/">>) of
- nomatch -> [Binary];
- _ ->
- term_to_path_parts(
- binary:split(Binary, <<"/">>, [global, trim_all]),
- Opts
- )
- end;
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts(Path = [ASCII | _], _Opts) when is_integer(ASCII) ->
- [hb_ao:normalize_key(Path)];
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts(List, Opts) when is_list(List) ->
- lists:flatten(lists:map(
- fun(Part) ->
- term_to_path_parts(Part, Opts)
- end,
- List
- ));
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts(Atom, _Opts) when is_atom(Atom) -> [Atom];
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts(Integer, _Opts) when is_integer(Integer) ->
- [hb_ao:normalize_key(Integer)];
-```
-
-### term_to_path_parts
-
-```erlang
-term_to_path_parts({as, DevName, Msgs}, _Opts) ->
- [{as, hb_ao:normalize_key(DevName), Msgs}].
-```
-
-### to_binary
-
-Convert a path of any form to a binary.
-
-```erlang
-to_binary(Path) ->
- Parts = binary:split(do_to_binary(Path), <<"/">>, [global, trim_all]),
- iolist_to_binary(lists:join(<<"/">>, Parts)).
-```
-
-### do_to_binary
-
-Convert a path of any form to a binary.
-
-```erlang
-do_to_binary(Path) when is_list(Path) ->
- case hb_util:is_string_list(Path) of
- false ->
- iolist_to_binary(
- lists:join(
- "/",
- lists:filtermap(
- fun(Part) ->
- case do_to_binary(Part) of
- <<>> -> false;
- BinPart -> {true, BinPart}
- end
- end,
- Path
- )
- )
- );
- true ->
- to_binary(list_to_binary(Path))
- end;
-```
-
-### do_to_binary
-
-Convert a path of any form to a binary.
-
-```erlang
-do_to_binary(Path) when is_binary(Path) ->
- Path;
-```
-
-### do_to_binary
-
-Convert a path of any form to a binary.
-
-```erlang
-do_to_binary(Other) ->
- hb_ao:normalize_key(Other).
-```
-
-### matches
-
-Check if two keys match.
-
-```erlang
-matches(Key1, Key2) ->
- hb_util:to_lower(hb_ao:normalize_key(Key1)) ==
- hb_util:to_lower(hb_ao:normalize_key(Key2)).
-```
-
-### regex_matches
-
-Check if two keys match using regex.
-
-```erlang
-regex_matches(Path1, Path2) ->
- NormP1 = normalize(hb_ao:normalize_key(Path1)),
- NormP2 =
- case hb_ao:normalize_key(Path2) of
- Normalized = <<"^", _/binary>> -> Normalized;
- Normalized -> normalize(Normalized)
- end,
- try re:run(NormP1, NormP2) =/= nomatch
- catch _A:_B:_C -> false
- end.
-```
-
-### normalize
-
-Normalize a path to a binary, removing the leading slash if present.
-
-```erlang
-normalize(Path) ->
- case iolist_to_binary([Path]) of
- BinPath = <<"/", _/binary>> -> BinPath;
- Binary -> <<"/", Binary/binary>>
- end.
-```
-
-### hashpath_test
-
-```erlang
-hashpath_test() ->
- Msg1 = #{ priv => #{<<"empty">> => <<"message">>} },
- Msg2 = #{ priv => #{<<"exciting">> => <<"message2">>} },
- Hashpath = hashpath(Msg1, Msg2, #{}),
- ?assert(is_binary(Hashpath) andalso byte_size(Hashpath) == 87).
-```
-
-### hashpath_direct_msg2_test
-
-```erlang
-hashpath_direct_msg2_test() ->
- Msg1 = #{ <<"base">> => <<"message">> },
- Msg2 = #{ <<"path">> => <<"base">> },
- Hashpath = hashpath(Msg1, Msg2, #{}),
- [_, KeyName] = term_to_path_parts(Hashpath),
- ?assert(matches(KeyName, <<"base">>)).
-```
-
-### multiple_hashpaths_test
-
-```erlang
-multiple_hashpaths_test() ->
- Msg1 = #{ <<"empty">> => <<"message">> },
- Msg2 = #{ <<"exciting">> => <<"message2">> },
- Msg3 = #{ priv => #{<<"hashpath">> => hashpath(Msg1, Msg2, #{}) } },
- Msg4 = #{ <<"exciting">> => <<"message4">> },
- Msg5 = hashpath(Msg3, Msg4, #{}),
- ?assert(is_binary(Msg5)).
-```
-
-### verify_hashpath_test
-
-```erlang
-verify_hashpath_test() ->
- Msg1 = #{ <<"test">> => <<"initial">> },
- Msg2 = #{ <<"firstapplied">> => <<"msg2">> },
- Msg3 = #{ priv => #{<<"hashpath">> => hashpath(Msg1, Msg2, #{})} },
- Msg4 = #{ priv => #{<<"hashpath">> => hashpath(Msg2, Msg3, #{})} },
- Msg3Fake = #{ priv => #{<<"hashpath">> => hashpath(Msg4, Msg2, #{})} },
- ?assert(verify_hashpath([Msg1, Msg2, Msg3, Msg4], #{})),
- ?assertNot(verify_hashpath([Msg1, Msg2, Msg3Fake, Msg4], #{})).
-```
-
-### validate_path_transitions
-
-```erlang
-validate_path_transitions(X, Opts) ->
- {Head, X2} = pop_request(X, Opts),
- ?assertEqual(<<"a">>, Head),
- {H2, X3} = pop_request(X2, Opts),
- ?assertEqual(<<"b">>, H2),
- {H3, X4} = pop_request(X3, Opts),
- ?assertEqual(<<"c">>, H3),
- ?assertEqual(undefined, pop_request(X4, Opts)).
-```
-
-### pop_from_message_test
-
-```erlang
-pop_from_message_test() ->
- validate_path_transitions(#{ <<"path">> => [<<"a">>, <<"b">>, <<"c">>] }, #{}).
-```
-
-### pop_from_path_list_test
-
-```erlang
-pop_from_path_list_test() ->
- validate_path_transitions([<<"a">>, <<"b">>, <<"c">>], #{}).
-```
-
-### hd_test
-
-```erlang
-hd_test() ->
- ?assertEqual(<<"a">>, hd(#{ <<"path">> => [<<"a">>, <<"b">>, <<"c">>] }, #{})),
- ?assertEqual(undefined, hd(#{ <<"path">> => undefined }, #{})).
-```
-
-### tl_test
-
-```erlang
-tl_test() ->
- ?assertMatch([<<"b">>, <<"c">>], hb_maps:get(<<"path">>, tl(#{ <<"path">> => [<<"a">>, <<"b">>, <<"c">>] }, #{}))),
- ?assertEqual(undefined, tl(#{ <<"path">> => [] }, #{})),
- ?assertEqual(undefined, tl(#{ <<"path">> => <<"a">> }, #{})),
- ?assertEqual(undefined, tl(#{ <<"path">> => undefined }, #{})),
- ?assertEqual([<<"b">>, <<"c">>], tl([<<"a">>, <<"b">>, <<"c">>], #{ })),
- ?assertEqual(undefined, tl([<<"c">>], #{ })).
-```
-
-### to_binary_test
-
-```erlang
-to_binary_test() ->
- ?assertEqual(<<"a/b/c">>, to_binary([<<"a">>, <<"b">>, <<"c">>])),
- ?assertEqual(<<"a/b/c">>, to_binary(<<"a/b/c">>)),
- ?assertEqual(<<"a/b/c">>, to_binary([<<"a">>, <<"b">>, <<"c">>])),
- ?assertEqual(<<"a/b/c">>, to_binary(["a", <<"b">>, <<"c">>])),
- ?assertEqual(<<"a/b/b/c">>, to_binary([<<"a">>, [<<"b">>, <<"//b">>], <<"c">>])).
-```
-
-### term_to_path_parts_test
-
-```erlang
-term_to_path_parts_test() ->
- ?assert(matches([<<"a">>, <<"b">>, <<"c">>],
- term_to_path_parts(<<"a/b/c">>))),
- ?assert(matches([<<"a">>, <<"b">>, <<"c">>],
- term_to_path_parts([<<"a">>, <<"b">>, <<"c">>]))),
- ?assert(matches([<<"a">>, <<"b">>, <<"c">>],
- term_to_path_parts(["a", <<"b">>, <<"c">>]))),
- ?assert(matches([<<"a">>, <<"b">>, <<"b">>, <<"c">>],
- term_to_path_parts([[<<"/a">>, [<<"b">>, <<"//b">>], <<"c">>]]))),
- ?assertEqual([], term_to_path_parts(<<"/">>)).
-% calculate_multistage_hashpath_test() ->
-% Msg1 = #{ <<"base">> => <<"message">> },
-% Msg2 = #{ <<"path">> => <<"2">> },
-% Msg3 = #{ <<"path">> => <<"3">> },
-% Msg4 = #{ <<"path">> => <<"4">> },
-% Msg5 = hashpath(Msg1, [Msg2, Msg3, Msg4], #{}),
-% ?assert(is_binary(Msg5)),
-% Msg3Path = <<"3">>,
-% Msg5b = hashpath(Msg1, [Msg2, Msg3Path, Msg4]),
-% ?assertEqual(Msg5, Msg5b).
-```
-
-### regex_matches_test
-
-```erlang
-regex_matches_test() ->
- ?assert(regex_matches(<<"a/b/c">>, <<"a/.*/c">>)),
- ?assertNot(regex_matches(<<"a/b/c">>, <<"a/.*/d">>)),
- ?assert(regex_matches(<<"a/abcd/c">>, <<"a/abc.*/c">>)),
- ?assertNot(regex_matches(<<"a/bcd/c">>, <<"a/abc.*/c">>)),
- ?assert(regex_matches(<<"a/bcd/ignored/c">>, <<"a/.*/c">>)),
-```
-
----
-
-*Generated from [hb_path.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_path.erl)*
diff --git a/docs/book/src/hb_persistent.erl.md b/docs/book/src/hb_persistent.erl.md
deleted file mode 100644
index 16e4bcf83..000000000
--- a/docs/book/src/hb_persistent.erl.md
+++ /dev/null
@@ -1,585 +0,0 @@
-# hb_persistent
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_persistent.erl)
-
-Creates and manages long-lived AO-Core resolution processes.
-These can be useful for situations where a message is large and expensive
-to serialize and deserialize, or when executions should be deliberately
-serialized to avoid parallel executions of the same computation. This
-module is called during the core `hb_ao` execution process, so care
-must be taken to avoid recursive spawns/loops.
-Built using the `pg` module, which is a distributed Erlang process group
-manager.
-
----
-
-## Exported Functions
-
-- `await/4`
-- `default_await/5`
-- `default_grouper/3`
-- `default_worker/3`
-- `find_or_register/3`
-- `forward_work/2`
-- `group/3`
-- `notify/4`
-- `start_monitor/0`
-- `start_monitor/1`
-- `start_worker/2`
-- `start_worker/3`
-- `stop_monitor/1`
-- `unregister_notify/4`
-
----
-
-### start
-
-Creates and manages long-lived AO-Core resolution processes.
-Ensure that the `pg` module is started.
-Start a monitor that prints the current members of the group every
-
-```erlang
-start() -> hb_name:start().
-```
-
-### start_monitor
-
-Creates and manages long-lived AO-Core resolution processes.
-Ensure that the `pg` module is started.
-Start a monitor that prints the current members of the group every
-
-```erlang
-start_monitor() ->
- start_monitor(global).
-```
-
-### start_monitor
-
-```erlang
-start_monitor(Group) ->
- start_monitor(Group, #{}).
-```
-
-### start_monitor
-
-```erlang
-start_monitor(Group, Opts) ->
- start(),
- ?event({worker_monitor, {start_monitor, Group, hb_name:all()}}),
- spawn(fun() -> do_monitor(Group, #{}, Opts) end).
-```
-
-### stop_monitor
-
-```erlang
-stop_monitor(PID) ->
- PID ! stop.
-```
-
-### do_monitor
-
-```erlang
-do_monitor(Group, Last, Opts) ->
- Groups = lists:map(fun({Name, _}) -> Name end, hb_name:all()),
- New =
- hb_maps:from_list(
- lists:map(
- fun(G) ->
- Pid = hb_name:lookup(G),
- {
- G,
- #{
- pid => Pid,
- messages =>
- case Pid of
- undefined -> 0;
- _ ->
- length(
- element(2,
- erlang:process_info(Pid, messages)
- )
- )
- end
- }
- }
- end,
- case Group of
- global -> Groups;
- TargetGroup ->
- case lists:member(TargetGroup, Groups) of
- true -> [TargetGroup];
- false -> []
- end
- end
- )
- ),
- Delta =
- hb_maps:filter(
- fun(G, NewState) ->
- case hb_maps:get(G, Last, []) of
- NewState -> false;
- _ -> true
- end
- end,
- New,
- Opts
- ),
- case hb_maps:size(Delta, Opts) of
- 0 -> ok;
- Deltas ->
- io:format(standard_error, "== Sitrep ==> ~p named processes. ~p changes. ~n",
- [hb_maps:size(New, Opts), Deltas]),
- hb_maps:map(
- fun(G, #{pid := P, messages := Msgs}) ->
- io:format(standard_error, "[~p: ~p] #M: ~p~n", [G, P, Msgs])
- end,
- Delta,
- Opts
- ),
- io:format(standard_error, "~n", [])
- end,
- timer:sleep(1000),
- receive stop -> stopped
- after 0 -> do_monitor(Group, New, Opts)
- end.
-```
-
-### find_or_register
-
-Register the process to lead an execution if none is found, otherwise
-
-```erlang
-find_or_register(Msg1, Msg2, Opts) ->
- GroupName = group(Msg1, Msg2, Opts),
- find_or_register(GroupName, Msg1, Msg2, Opts).
-```
-
-### find_or_register
-
-```erlang
-find_or_register(ungrouped_exec, _Msg1, _Msg2, _Opts) ->
- {leader, ungrouped_exec};
-```
-
-### find_or_register
-
-```erlang
-find_or_register(GroupName, _Msg1, _Msg2, Opts) ->
- case hb_opts:get(await_inprogress, false, Opts) of
- false -> {leader, GroupName};
- _ ->
- Self = self(),
- case find_execution(GroupName, Opts) of
- {ok, Leader} when Leader =/= Self ->
- ?event({found_leader, GroupName, {leader, Leader}}),
- {wait, Leader};
- {ok, Leader} when Leader =:= Self ->
- {infinite_recursion, GroupName};
- _ ->
- ?event({register_resolver, {group, GroupName}}),
- register_groupname(GroupName, Opts),
- {leader, GroupName}
- end
- end.
-```
-
-### unregister_notify
-
-Unregister as the leader for an execution and notify waiting processes.
-
-```erlang
-unregister_notify(ungrouped_exec, _Msg2, _Msg3, _Opts) -> ok;
-```
-
-### unregister_notify
-
-Unregister as the leader for an execution and notify waiting processes.
-
-```erlang
-unregister_notify(GroupName, Msg2, Msg3, Opts) ->
- unregister_groupname(GroupName, Opts),
- notify(GroupName, Msg2, Msg3, Opts).
-```
-
-### find_execution
-
-Find a group with the given name.
-
-```erlang
-find_execution(Groupname, _Opts) ->
- start(),
- case hb_name:lookup(Groupname) of
- undefined -> not_found;
- Pid -> {ok, Pid}
- end.
-```
-
-### group
-
-Calculate the group name for a Msg1 and Msg2 pair. Uses the Msg1's
-
-```erlang
-group(Msg1, Msg2, Opts) ->
- Grouper =
- hb_maps:get(grouper, hb_ao:info(Msg1, Opts), fun default_grouper/3, Opts),
- apply(
- Grouper,
- hb_ao:truncate_args(Grouper, [Msg1, Msg2, Opts])
- ).
-```
-
-### register_groupname
-
-Register for performing an AO-Core resolution.
-
-```erlang
-register_groupname(Groupname, _Opts) ->
- ?event({registering_as, Groupname}),
- hb_name:register(Groupname).
-```
-
-### unregister
-
-Unregister for being the leader on an AO-Core resolution.
-
-```erlang
-unregister(Msg1, Msg2, Opts) ->
- start(),
- unregister_groupname(group(Msg1, Msg2, Opts), Opts).
-```
-
-### unregister_groupname
-
-```erlang
-unregister_groupname(Groupname, _Opts) ->
- ?event({unregister_resolver, {explicit, Groupname}}),
- hb_name:unregister(Groupname).
-```
-
-### await
-
-If there was already an Erlang process handling this execution,
-
-```erlang
-await(Worker, Msg1, Msg2, Opts) ->
- % Get the device's await function, if it exists.
-```
-
-### default_await
-
-Default await function that waits for a resolution from a worker.
-
-```erlang
-default_await(Worker, GroupName, Msg1, Msg2, Opts) ->
- % Wait for the result.
-```
-
-### notify
-
-Check our inbox for processes that are waiting for the resolution
-
-```erlang
-notify(GroupName, Msg2, Msg3, Opts) ->
- case is_binary(GroupName) of
- true ->
- ?event({notifying_all, {group, GroupName}});
- false ->
- ok
- end,
- receive
- {resolve, Listener, GroupName, Msg2, _ListenerOpts} ->
- ?event({notifying_listener, {listener, Listener}, {group, GroupName}}),
- send_response(Listener, GroupName, Msg2, Msg3),
- notify(GroupName, Msg2, Msg3, Opts)
- after 0 ->
- ?event(finished_notify),
- ok
- end.
-```
-
-### forward_work
-
-Forward requests to a newly delegated execution process.
-
-```erlang
-forward_work(NewPID, Opts) ->
- Gather =
- fun Gather() ->
- receive
- Req = {resolve, _, _, _, _} -> [Req | Gather()]
- after 0 -> []
- end
- end,
- ToForward = Gather(),
- lists:foreach(
- fun(Req) ->
- NewPID ! Req
- end,
- ToForward
- ),
- case length(ToForward) > 0 of
- true ->
- ?event({fwded, {reqs, length(ToForward)}, {pid, NewPID}}, Opts);
- false -> ok
- end,
- ok.
-```
-
-### send_response
-
-Helper function that wraps responding with a new Msg3.
-
-```erlang
-send_response(Listener, GroupName, Msg2, Msg3) ->
- ?event(worker,
- {send_response,
- {listener, Listener},
- {group, GroupName}
- }
- ),
- Listener ! {resolved, self(), GroupName, Msg2, Msg3}.
-```
-
-### start_worker
-
-Start a worker process that will hold a message in memory for
-
-```erlang
-start_worker(Msg, Opts) ->
- start_worker(group(Msg, undefined, Opts), Msg, Opts).
-```
-
-### start_worker
-
-```erlang
-start_worker(_, NotMsg, _) when not is_map(NotMsg) -> not_started;
-```
-
-### start_worker
-
-```erlang
-start_worker(GroupName, Msg, Opts) ->
- start(),
- ?event(worker_spawns,
- {starting_worker, {group, GroupName}, {msg, Msg}, {opts, Opts}}
- ),
- WorkerPID = spawn(
- fun() ->
- % If the device's info contains a `worker' function we
- % use that instead of the default implementation.
-```
-
-### default_worker
-
-A server function for handling persistent executions.
-
-```erlang
-default_worker(GroupName, Msg1, Opts) ->
- Timeout = hb_opts:get(worker_timeout, 10000, Opts),
- worker_event(GroupName, default_worker_waiting_for_req, Msg1, undefined, Opts),
- receive
- {resolve, Listener, GroupName, Msg2, ListenerOpts} ->
- ?event(worker,
- {work_received,
- {listener, Listener},
- {group, GroupName}
- }
- ),
- Res =
- hb_ao:resolve(
- Msg1,
- Msg2,
- hb_maps:merge(ListenerOpts, Opts, Opts)
- ),
- send_response(Listener, GroupName, Msg2, Res),
- notify(GroupName, Msg2, Res, Opts),
- case hb_opts:get(static_worker, false, Opts) of
- true ->
- % Reregister for the existing group name.
-```
-
-### default_grouper
-
-Create a group name from a Msg1 and Msg2 pair as a tuple.
-
-```erlang
-default_grouper(Msg1, Msg2, Opts) ->
- %?event({calculating_default_group_name, {msg1, Msg1}, {msg2, Msg2}}),
- % Use Erlang's `phash2' to hash the result of the Grouper function.
-```
-
-### worker_event
-
-Log an event with the worker process. If we used the default grouper
-
-```erlang
-worker_event(Group, Data, Msg1, Msg2, Opts) when is_integer(Group) ->
- ?event(worker, {worker_event, Group, Data, {msg1, Msg1}, {msg2, Msg2}}, Opts);
-```
-
-### worker_event
-
-Log an event with the worker process. If we used the default grouper
-
-```erlang
-worker_event(Group, Data, _, _, Opts) ->
- ?event(worker, {worker_event, Group, Data}, Opts).
-```
-
-### test_device
-
-```erlang
-test_device() -> test_device(#{}).
-```
-
-### test_device
-
-```erlang
-test_device(Base) ->
- #{
- info =>
- fun() ->
- hb_maps:merge(
- #{
- grouper =>
- fun(M1, _M2, _Opts) ->
- erlang:phash2(M1)
- end
- },
- Base
- )
- end,
- slow_key =>
- fun(_, #{ <<"wait">> := Wait }) ->
- ?event({slow_key_wait_started, Wait}),
- receive after Wait ->
- {ok,
- #{
- waited => Wait,
- pid => self(),
- random_bytes =>
- hb_util:encode(crypto:strong_rand_bytes(4))
- }
- }
- end
- end,
- self =>
- fun(M1, #{ <<"wait">> := Wait }) ->
- ?event({self_waiting, {wait, Wait}}),
- receive after Wait ->
- ?event({self_returning, M1, {wait, Wait}}),
- {ok, M1}
- end
- end
- }.
-```
-
-### spawn_test_client
-
-```erlang
-spawn_test_client(Msg1, Msg2) ->
- spawn_test_client(Msg1, Msg2, #{}).
-```
-
-### spawn_test_client
-
-```erlang
-spawn_test_client(Msg1, Msg2, Opts) ->
- Ref = make_ref(),
- TestParent = self(),
- spawn_link(fun() ->
- ?event({new_concurrent_test_resolver, Ref, {executing, Msg2}}),
- Res = hb_ao:resolve(Msg1, Msg2, Opts),
- ?event({test_worker_got_result, Ref, {result, Res}}),
- TestParent ! {result, Ref, Res}
- end),
- Ref.
-```
-
-### wait_for_test_result
-
-```erlang
-wait_for_test_result(Ref) ->
- receive {result, Ref, Res} -> Res end.
-```
-
-### deduplicated_execution_test
-
-Test merging and returning a value with a persistent worker.
-
-```erlang
-deduplicated_execution_test() ->
- TestTime = 200,
- Msg1 = #{ <<"device">> => test_device() },
- Msg2 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => TestTime },
- T0 = hb:now(),
- Ref1 = spawn_test_client(Msg1, Msg2),
- receive after 100 -> ok end,
- Ref2 = spawn_test_client(Msg1, Msg2),
- Res1 = wait_for_test_result(Ref1),
- Res2 = wait_for_test_result(Ref2),
- T1 = hb:now(),
- % Check the result is the same.
-```
-
-### persistent_worker_test
-
-Test spawning a default persistent worker.
-
-```erlang
-persistent_worker_test() ->
- TestTime = 200,
- Msg1 = #{ <<"device">> => test_device() },
- link(start_worker(Msg1, #{ static_worker => true })),
- receive after 10 -> ok end,
- Msg2 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => TestTime },
- Msg3 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.1) },
- Msg4 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.2) },
- T0 = hb:now(),
- Ref1 = spawn_test_client(Msg1, Msg2),
- Ref2 = spawn_test_client(Msg1, Msg3),
- Ref3 = spawn_test_client(Msg1, Msg4),
- Res1 = wait_for_test_result(Ref1),
- Res2 = wait_for_test_result(Ref2),
- Res3 = wait_for_test_result(Ref3),
- T1 = hb:now(),
- ?assertNotEqual(Res1, Res2),
- ?assertNotEqual(Res2, Res3),
- ?assert(T1 - T0 >= (3*TestTime)).
-```
-
-### spawn_after_execution_test
-
-```erlang
-spawn_after_execution_test() ->
- ?event(<<"">>),
- TestTime = 500,
- Msg1 = #{ <<"device">> => test_device() },
- Msg2 = #{ <<"path">> => <<"self">>, <<"wait">> => TestTime },
- Msg3 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.1) },
- Msg4 = #{ <<"path">> => <<"slow_key">>, <<"wait">> => trunc(TestTime*1.2) },
- T0 = hb:now(),
- Ref1 =
- spawn_test_client(
- Msg1,
- Msg2,
- #{
- spawn_worker => true,
- static_worker => true,
- hashpath => ignore
- }
- ),
- receive after 10 -> ok end,
- Ref2 = spawn_test_client(Msg1, Msg3),
- Ref3 = spawn_test_client(Msg1, Msg4),
- Res1 = wait_for_test_result(Ref1),
- Res2 = wait_for_test_result(Ref2),
- Res3 = wait_for_test_result(Ref3),
- T1 = hb:now(),
- ?assertNotEqual(Res1, Res2),
- ?assertNotEqual(Res2, Res3),
-```
-
----
-
-*Generated from [hb_persistent.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_persistent.erl)*
diff --git a/docs/book/src/hb_private.erl.md b/docs/book/src/hb_private.erl.md
deleted file mode 100644
index 353a46deb..000000000
--- a/docs/book/src/hb_private.erl.md
+++ /dev/null
@@ -1,279 +0,0 @@
-# hb_private
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_private.erl)
-
-This module provides basic helper utilities for managing the
-private element of a message, which can be used to store state that is
-not included in serialized messages, or those granted to users via the
-APIs. Private elements of a message can be useful for storing state that
-is only relevant temporarily. For example, a device might use the private
-element to store a cache of values that are expensive to recompute. They
-should _not_ be used for encoding state that makes the execution of a
-device non-deterministic (unless you are sure you know what you are doing).
-The `set` and `get` functions of this module allow you to run those keys
-as AO-Core paths if you would like to have private `devices` in the
-messages non-public zone.
-See `hb_ao` for more information about the AO-Core protocol
-and private elements of messages.
-
----
-
-## Exported Functions
-
-- `from_message/1`
-- `get/3`
-- `get/4`
-- `is_private/1`
-- `merge/3`
-- `opts/1`
-- `reset/1`
-- `set_priv/2`
-- `set/3`
-- `set/4`
-
----
-
-### from_message
-
-This module provides basic helper utilities for managing the
-Return the `private` key from a message. If the key does not exist, an
-
-```erlang
-from_message(Msg) when is_map(Msg) ->
- case maps:is_key(<<"priv">>, Msg) of
- true -> maps:get(<<"priv">>, Msg, #{});
- false -> maps:get(priv, Msg, #{})
- end;
-```
-
-### from_message
-
-This module provides basic helper utilities for managing the
-Return the `private` key from a message. If the key does not exist, an
-Helper for getting a value from the private element of a message. Uses
-
-```erlang
-from_message(_NonMapMessage) -> #{}.
-```
-
-### get
-
-This module provides basic helper utilities for managing the
-Return the `private` key from a message. If the key does not exist, an
-Helper for getting a value from the private element of a message. Uses
-
-```erlang
-get(Key, Msg, Opts) ->
- get(Key, Msg, not_found, Opts).
-```
-
-### get
-
-```erlang
-get(InputPath, Msg, Default, Opts) ->
- % Resolve the path against the private element of the message.
-```
-
-### set
-
-Helper function for setting a key in the private element of a message.
-
-```erlang
-set(Msg, InputPath, Value, Opts) ->
- Path = remove_private_specifier(InputPath, Opts),
- Priv = from_message(Msg),
- ?event({set_private, {in, Path}, {out, Path}, {value, Value}, {opts, Opts}}),
- NewPriv = hb_util:deep_set(Path, Value, Priv, opts(Opts)),
- ?event({set_private_res, {out, NewPriv}}),
- set_priv(Msg, NewPriv).
-```
-
-### set
-
-```erlang
-set(Msg, PrivMap, Opts) ->
- CurrentPriv = from_message(Msg),
- ?event({set_private, {in, PrivMap}, {opts, Opts}}),
- NewPriv = hb_util:deep_merge(CurrentPriv, PrivMap, opts(Opts)),
- ?event({set_private_res, {out, NewPriv}}),
- set_priv(Msg, NewPriv).
-```
-
-### merge
-
-Merge the private elements of two messages into one. The keys in the
-
-```erlang
-merge(Msg1, Msg2, Opts) ->
- % Merge the private elements of the two messages.
-```
-
-### set_priv
-
-Helper function for setting the complete private element of a message.
-
-```erlang
-set_priv(Msg, PrivMap)
- when map_size(PrivMap) =:= 0 andalso not is_map_key(<<"priv">>, Msg) ->
- Msg;
-```
-
-### set_priv
-
-Helper function for setting the complete private element of a message.
-Check if a key is private.
-
-```erlang
-set_priv(Msg, PrivMap) ->
- Msg#{ <<"priv">> => PrivMap }.
-```
-
-### is_private
-
-Helper function for setting the complete private element of a message.
-Check if a key is private.
-
-```erlang
-is_private(Key) ->
- try hb_util:bin(Key) of
- <<"priv", _/binary>> -> true;
- _ -> false
- catch _:_ -> false
- end.
-```
-
-### remove_private_specifier
-
-Remove the first key from the path if it is a private specifier.
-
-```erlang
-remove_private_specifier(InputPath, Opts) ->
- case is_private(hd(Path = hb_path:term_to_path_parts(InputPath, Opts))) of
- true -> tl(Path);
- false -> Path
- end.
-```
-
-### opts
-
-The opts map that should be used when resolving paths against the
-
-```erlang
-opts(Opts) ->
- PrivStore =
- case hb_opts:get(priv_store, undefined, Opts) of
- undefined -> [];
- PrivateStores when is_list(PrivateStores) -> PrivateStores;
- PrivateStore -> [PrivateStore]
- end,
- BaseStore =
- case hb_opts:get(store, [], Opts) of
- SingleStore when is_map(SingleStore) -> [SingleStore];
- Stores when is_list(Stores) -> Stores
- end,
- NormStore = PrivStore ++ BaseStore,
- Opts#{
- hashpath => ignore,
- cache_control => [<<"no-cache">>, <<"no-store">>],
- store => NormStore
- }.
-```
-
-### reset
-
-Unset all of the private keys in a message or deep Erlang term.
-
-```erlang
-reset(Msg) when is_map(Msg) ->
- maps:map(
- fun(_Key, Val) -> reset(Val) end,
- maps:without(
- lists:filter(fun is_private/1, maps:keys(Msg)),
- Msg
- )
- );
-```
-
-### reset
-
-Unset all of the private keys in a message or deep Erlang term.
-
-```erlang
-reset(List) when is_list(List) ->
- % Check if any of the terms in the list are private specifiers, return an
- % empty list if so.
-```
-
-### reset
-
-```erlang
-reset(Tuple) when is_tuple(Tuple) ->
- list_to_tuple(reset(tuple_to_list(Tuple)));
-```
-
-### reset
-
-```erlang
-reset(NonMapMessage) ->
- NonMapMessage.
-```
-
-### set_private_test
-
-```erlang
-set_private_test() ->
- ?assertEqual(
- #{<<"a">> => 1, <<"priv">> => #{<<"b">> => 2}},
- set(#{<<"a">> => 1}, <<"b">>, 2, #{})
- ),
- Res = set(#{<<"a">> => 1}, <<"a">>, 1, #{}),
- ?assertEqual(#{<<"a">> => 1, <<"priv">> => #{<<"a">> => 1}}, Res),
- ?assertEqual(
- #{<<"a">> => 1, <<"priv">> => #{<<"a">> => 1}},
- set(Res, <<"a">>, 1, #{})
- ).
-```
-
-### get_private_key_test
-
-```erlang
-get_private_key_test() ->
- M1 = #{<<"a">> => 1, <<"priv">> => #{<<"b">> => 2}},
- ?assertEqual(not_found, get(<<"a">>, M1, #{})),
- {ok, [<<"a">>]} = hb_ao:resolve(M1, <<"keys">>, #{}),
- ?assertEqual(2, get(<<"b">>, M1, #{})),
- {error, _} = hb_ao:resolve(M1, <<"priv/a">>, #{}),
- {error, _} = hb_ao:resolve(M1, <<"priv">>, #{}).
-```
-
-### get_deep_key_test
-
-```erlang
-get_deep_key_test() ->
- M1 = #{<<"a">> => 1, <<"priv">> => #{<<"b">> => #{<<"c">> => 3}}},
- ?assertEqual(3, get(<<"b/c">>, M1, #{})).
-```
-
-### priv_opts_store_read_link_test
-
-```erlang
-priv_opts_store_read_link_test() ->
- % Write a message to the public store.
-```
-
-### priv_opts_cache_read_message_test
-
-```erlang
-priv_opts_cache_read_message_test() ->
- hb:init(),
- PublicStore = [hb_test_utils:test_store(hb_store_lmdb)],
- OnlyPrivStore = [hb_test_utils:test_store(hb_store_fs)],
- Opts = #{ store => PublicStore, priv_store => OnlyPrivStore },
- PrivOpts = opts(Opts),
- % Use the `~scheduler@1.0' and `~process@1.0' infrastructure to write a
- % complex message into the public store.
-```
-
----
-
-*Generated from [hb_private.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_private.erl)*
diff --git a/docs/book/src/hb_process_monitor.erl.md b/docs/book/src/hb_process_monitor.erl.md
deleted file mode 100644
index ea96dae16..000000000
--- a/docs/book/src/hb_process_monitor.erl.md
+++ /dev/null
@@ -1,106 +0,0 @@
-# hb_process_monitor
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_process_monitor.erl)
-
-## Exported Functions
-
-- `start/1`
-- `start/2`
-- `start/3`
-- `stop/1`
-
----
-
-### start
-
-```erlang
-start(ProcID) ->
- start(ProcID, hb_opts:get(default_cron_rate)).
-```
-
-### start
-
-```erlang
-start(ProcID, Rate) ->
- start(ProcID, Rate, hb_client:cron_cursor(ProcID)).
-```
-
-### start
-
-```erlang
-start(ProcID, Rate, Cursor) ->
- Logger = hb_logger:start(),
- Monitor = spawn(
- fun() ->
- server(
- #state{
- proc_id = ProcID,
- cursor = Cursor,
- logger = Logger
- }
- )
- end),
- Ticker = spawn(fun() -> ticker(Monitor, Rate) end),
- hb_logger:register(Monitor),
- hb_logger:log(Monitor, {ok, started_monitor, {ProcID, Rate, Cursor}}),
- hb_logger:register(Ticker),
- {Monitor, Logger}.
-```
-
-### stop
-
-```erlang
-stop(PID) ->
- PID ! stop.
-```
-
-### server
-
-```erlang
-server(State) ->
- receive
- stop -> ok;
- tick ->server(handle_crons(State))
- end.
-```
-
-### handle_crons
-
-```erlang
-handle_crons(State) ->
- case hb_client:cron(State#state.proc_id, State#state.cursor) of
- {ok, HasNextPage, Results, Cursor} ->
- lists:map(
- fun(Res) ->
- % TODO: Validate this
- dev_mu:push(#{ message => Res }, State)
- end,
- Results
- ),
- NS = State#state{cursor = Cursor},
- case HasNextPage of
- true -> NS;
- false -> handle_crons(NS)
- end;
- Error ->
- hb_logger:log(State#state.logger, Error),
- State
- end.
-```
-
-### ticker
-
-```erlang
-ticker(Monitor, Rate) ->
- case erlang:is_process_alive(Monitor) of
- true ->
- timer:sleep(Rate),
- Monitor ! tick,
- ticker(Monitor, Rate);
- false ->
- ok
-```
-
----
-
-*Generated from [hb_process_monitor.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_process_monitor.erl)*
diff --git a/docs/book/src/hb_router.erl.md b/docs/book/src/hb_router.erl.md
deleted file mode 100644
index 06c229ae3..000000000
--- a/docs/book/src/hb_router.erl.md
+++ /dev/null
@@ -1,44 +0,0 @@
-# hb_router
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_router.erl)
-
-Locate a service in the AO network. This module uses
-URLs to locate services, so it can be used to locate
-nodes using IP addresses or domain names. This also
-allows us to use different protocols later, potentially.
-
----
-
-## Exported Functions
-
-- `find/2`
-- `find/3`
-
----
-
-### find
-
-```erlang
-find(Type, ID) ->
- find(Type, ID, '_').
-```
-
-### find
-
-```erlang
-find(Type, ID, Address) ->
- find(Type, ID, Address, #{}).
-```
-
-### find
-
-```erlang
-find(Type, _ID, Address, Opts) ->
- case hb_maps:get(Type, hb_opts:get(nodes), undefined, Opts) of
- #{ Address := Node } -> {ok, Node};
- undefined -> {error, service_type_not_found}
-```
-
----
-
-*Generated from [hb_router.erl](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_router.erl)*
diff --git a/docs/book/src/hb_singleton.erl.md b/docs/book/src/hb_singleton.erl.md
deleted file mode 100644
index 12a3cdbca..000000000
--- a/docs/book/src/hb_singleton.erl.md
+++ /dev/null
@@ -1,1104 +0,0 @@
-# hb_singleton
-
-[View source on GitHub](https://github.com/permaweb/HyperBEAM/blob/edge/src/hb_singleton.erl)
-
-A parser that translates AO-Core HTTP API requests in TABM format
-into an ordered list of messages to evaluate. The details of this format
-are described in `docs/ao-core-http-api.md`.
-Syntax overview:
-
- Singleton: Message containing keys and a `path` field,
- which may also contain a query string of key-value pairs.
- Path:
- - /Part1/Part2/.../PartN/ => [Part1, Part2, ..., PartN]
- - /ID/Part2/.../PartN => [ID, Part2, ..., PartN]
- Part: (Key + Resolution), Device?, #{ K => V}?
- - Part => #{ path => Part }
- - `Part&Key=Value => #{ path => Part, Key => Value }`
- - `Part=Value&... => #{ path => Part, Part => Value, ... }`
- - `Part&Key => #{ path => Part, Key => true }`
- - `Part&k1=v1&k2=v2 => #{ path => Part, k1 => `<<"v1">>`, k2 => `<<"v2">>` }'
- - `Part~Device => {as, Device, #{ path => Part }}`
- - `Part~D&K1=V1 => {as, D, #{ path => Part, K1 => `<<"v1">>` }}'
- - `pt&k1+int=1 => #{ path => pt, k1 => 1 }`
- - `pt~d&k1+int=1 => {as, d, #{ path => pt, k1 => 1 }}`
- - `(/nested/path) => Resolution of the path /nested/path`
- - `(/nested/path&k1=v1) => (resolve /nested/path)#{k1 => v1}`
- - `(/nested/path~D&K1=V1) => (resolve /nested/path)#{K1 => V1}`
- - `pt&k1+res=(/a/b/c) => #{ path => pt, k1 => (resolve /a/b/c) }`
- Key:
- - key: `<<"value">>` => #{ key => `<<"value">>`, ... } for all messages
- - n.key: `<<"value">>` => #{ key => `<<"value">>`, ... } for Nth message
- - key+int: 1 => #{ key => 1, ... }
- - key+res: /nested/path => #{ key => (resolve /nested/path), ... }
- - N.Key+res=(/a/b/c) => #{ Key => (resolve /a/b/c), ... }
-
-
----
-
-## Exported Functions
-
-- `from_path/1`
-- `from/2`
-- `to/1`
-
----
-
-### append_path
-
-```erlang
--spec to(list(ao_message())) -> tabm_message().
-to(Messages) ->
- % Iterate through all AO-Core messages folding them into the TABM message
- % Scopes contains the following map: #{Key => [StageIndex, StageIndex2...]}
- % that allows to scope keys to the given stage.
-```
-
-```erlang
-append_path(PathPart, #{<<"path">> := Path} = Message) ->
- hb_maps:put(<<"path">>, < tags with structured content
+ text = text.replace(/([\s\S]*?)<\/pre>/g, (match, content) => {
+ return this.formatPreContent(content);
+ });
+
// Convert Erlang doc syntax to Markdown
return text
.replace(/`([^']*?)'/g, '`$1`') // Convert `code' to `code`
.replace(/<</g, '<<') // Fix HTML entities
.replace(/>>/g, '>>')
.replace(/@doc\s*/g, '') // Remove @doc tags
- .trim();
+ .replace(/\n\s*\n\s*\n/g, '\n\n') // Normalize multiple empty lines to double newlines
+ .replace(/[ \t]+$/gm, '') // Trim trailing spaces per line
+ .replace(/^\s+|\s+$/g, ''); // Final trim
+ }
+
+ formatPreContent(content) {
+ // First, let's look at the actual structure of the content more carefully
+ // The issue is that definitions span multiple lines with varying indentation
+
+ const lines = content.trim().split('\n');
+ const formatted = [];
+
+ let i = 0;
+ while (i < lines.length) {
+ const line = lines[i].trim();
+
+ if (!line) {
+ i++;
+ continue;
+ }
+
+ // Look for definition pattern: starts with word(s), colon, then description
+ // Pattern: "DevMod:ExportedFunc : Description" or "info/exports : Description"
+ const defMatch = line.match(/^(\S+(?:\s*:\s*\S+)?)\s*:\s*(.*)$/);
+
+ if (defMatch) {
+ const [, term, initialDesc] = defMatch;
+ let fullDescription = initialDesc.trim();
+
+ // Collect continuation lines for this definition
+ let j = i + 1;
+ while (j < lines.length) {
+ const nextLine = lines[j];
+
+ // Empty line - check if there's more content
+ if (!nextLine.trim()) {
+ j++;
+ continue;
+ }
+
+ // If it looks like a new definition, stop
+ if (nextLine.trim().match(/^\S+(?:\s*:\s*\S+)?\s*:\s*/)) {
+ break;
+ }
+
+ // This is a continuation line - add it to the description
+ if (nextLine.trim()) {
+ fullDescription += ' ' + nextLine.trim();
+ }
+ j++;
+ }
+
+ // Format the definition
+ formatted.push('');
+ formatted.push(`**${term.trim()}**`);
+ formatted.push('');
+ formatted.push(fullDescription);
+
+ i = j; // Move to the next unprocessed line
+ } else {
+ // Not a definition - handle as regular content
+ if (line.toLowerCase().includes('hyperbeam') && line.includes('options')) {
+ formatted.push('');
+ formatted.push(`### ${line}`);
+ formatted.push('');
+ } else if (line.match(/^`[^`]+`\s*:/)) {
+ // Special case for option definitions like `update_hashpath`:
+ const optMatch = line.match(/^(`[^`]+`)\s*:\s*(.*)$/);
+ if (optMatch) {
+ const [, optName, optDesc] = optMatch;
+ formatted.push('');
+ formatted.push(`**${optName}**`);
+ formatted.push('');
+ formatted.push(optDesc);
+ } else {
+ formatted.push(line);
+ }
+ } else {
+ formatted.push(line);
+ }
+ i++;
+ }
+ }
+
+ return formatted.join('\n');
}
parseDocumentation(docText) {
@@ -460,78 +639,124 @@ class ErlangLiterateParser {
md.push('');
}
+ // Group functions by name to merge overloaded functions
+ const groupedFunctions = this.groupFunctionsByName(this.functions);
+
// Functions
- for (const func of this.functions) {
- md.push(`## ${func.name}`);
+ for (const group of groupedFunctions) {
+ md.push(`## ${group.name}`);
md.push('');
- // Parse and format documentation
+ // Combine documentation from all functions in the group
+ const combinedDoc = this.combineFunctionDocs(group.functions);
+ if (combinedDoc) {
+ md.push(combinedDoc);
+ md.push('');
+ }
+
+ // Add all specs and bodies for the function group
+ for (const func of group.functions) {
+ // Spec
+ if (func.spec) {
+ md.push('```erlang');
+ md.push(func.spec.trim());
+ md.push('```');
+ md.push('');
+ }
+
+ // Function body with inline comments
+ if (func.body && func.body.length > 0) {
+ md.push('');
+
+ for (const segment of func.body) {
+ if (segment.type === 'comment') {
+ md.push(segment.content);
+ md.push('');
+ } else if (segment.type === 'doc') {
+ // Insert structured params/returns adjacent to the preceding code
+ md.push(segment.content);
+ md.push('');
+ } else if (segment.type === 'code') {
+ md.push('```erlang');
+ md.push(segment.content.trim());
+ md.push('```');
+ md.push('');
+ }
+ }
+ }
+ }
+
+ md.push('');
+ }
+
+ // Footer
+ md.push('---');
+ md.push('');
+ md.push(`*Generated from [${fileName}](${githubUrl})*`);
+
+ return md.join('\n');
+ }
+
+ groupFunctionsByName(functions) {
+ const groups = [];
+ let currentGroup = null;
+
+ for (const func of functions) {
+ if (!currentGroup || currentGroup.name !== func.name) {
+ // Start a new group
+ currentGroup = {
+ name: func.name,
+ functions: [func]
+ };
+ groups.push(currentGroup);
+ } else {
+ // Add to current group
+ currentGroup.functions.push(func);
+ }
+ }
+
+ return groups;
+ }
+
+ combineFunctionDocs(functions) {
+ // Use the documentation from the first function that has it
+ // In practice, usually only the first clause of an overloaded function has detailed docs
+ for (const func of functions) {
if (func.doc) {
const parsed = this.parseDocumentation(func.doc);
+ let combinedDoc = [];
// Description
if (parsed.description.length > 0) {
- md.push(this.cleanDocumentation(parsed.description.join(' ')));
- md.push('');
+ combinedDoc.push(this.cleanDocumentation(parsed.description.join('\n')));
+ combinedDoc.push('');
}
// Parameters
if (parsed.params.length > 0) {
- md.push('### Parameters');
- md.push('');
+ combinedDoc.push('### Parameters');
+ combinedDoc.push('');
for (const param of parsed.params) {
const desc = this.cleanDocumentation(param.description);
- md.push(`- \`${param.name}\` - ${desc}`);
+ combinedDoc.push(`- \`${param.name}\` - ${desc}`);
}
- md.push('');
+ combinedDoc.push('');
}
// Returns
if (parsed.returns.length > 0) {
- md.push('### Returns');
- md.push('');
+ combinedDoc.push('### Returns');
+ combinedDoc.push('');
for (const ret of parsed.returns) {
- md.push(`- ${this.cleanDocumentation(ret)}`);
+ combinedDoc.push(`- ${this.cleanDocumentation(ret)}`);
}
- md.push('');
+ combinedDoc.push('');
}
- }
-
- // Spec
- if (func.spec) {
- md.push('```erlang');
- md.push(func.spec.trim());
- md.push('```');
- md.push('');
- }
-
- // Function body with inline comments
- if (func.body && func.body.length > 0) {
- // md.push('### Function');
- md.push('');
- for (const segment of func.body) {
- if (segment.type === 'comment') {
- md.push(segment.content);
- md.push('');
- } else if (segment.type === 'code') {
- md.push('```erlang');
- md.push(segment.content.trim());
- md.push('```');
- md.push('');
- }
- }
+ return combinedDoc.join('\n');
}
-
- md.push('');
}
-
- // Footer
- md.push('---');
- md.push('');
- md.push(`*Generated from [${fileName}](${githubUrl})*`);
-
- return md.join('\n');
+ return null;
}
}
From 621269882ebbd0cefa95f3a5f94e4928f9d08055 Mon Sep 17 00:00:00 2001
From: Dylan Shade <63427984+dpshade@users.noreply.github.com>
Date: Fri, 19 Sep 2025 16:57:52 -0400
Subject: [PATCH 14/17] docs: Refactor Erlang literate parser for improved
return formatting and documentation handling
- Update return processing to utilize `splitReturnsIntoOutcomes` and `formatReturnsText` for better output formatting.
- Enhance handling of continuation lines in return and parameter sections to maintain clarity in documentation.
- Introduce `reflowNumberedLists` method to ensure proper formatting of numbered lists in generated documentation.
---
docs/erlang-literate-parser.js | 154 +++++++++++++++++++++++++++++++--
1 file changed, 146 insertions(+), 8 deletions(-)
diff --git a/docs/erlang-literate-parser.js b/docs/erlang-literate-parser.js
index 82cf91ea3..5f276307c 100755
--- a/docs/erlang-literate-parser.js
+++ b/docs/erlang-literate-parser.js
@@ -383,8 +383,9 @@ class ErlangLiterateParser {
if (parsed.returns.length > 0) {
docParts.push('### Returns');
docParts.push('');
- for (const r of parsed.returns) {
- docParts.push(`- ${this.cleanDocumentation(r)}`);
+ const expanded = parsed.returns.flatMap(r => this.splitReturnsIntoOutcomes(r));
+ for (const r of expanded) {
+ docParts.push(`- ${this.formatReturnsText(r)}`);
}
docParts.push('');
}
@@ -426,6 +427,13 @@ class ErlangLiterateParser {
: `@returns ${cleaned.trim()}`;
// Accumulate tag lines
pendingTagLines.push(lineAsTag);
+ } else if (
+ pendingTagLines.length > 0 &&
+ (pendingTagLines[pendingTagLines.length - 1].startsWith('@returns') ||
+ pendingTagLines[pendingTagLines.length - 1].startsWith('@param'))
+ ) {
+ // Continuation of the previous @returns/@param block; append line
+ pendingTagLines.push(cleaned.trim());
} else {
// Flush any pending tag block before emitting a normal comment
flushTags();
@@ -459,7 +467,7 @@ class ErlangLiterateParser {
});
// Convert Erlang doc syntax to Markdown
- return text
+ let cleaned = text
.replace(/`([^']*?)'/g, '`$1`') // Convert `code' to `code`
.replace(/<</g, '<<') // Fix HTML entities
.replace(/>>/g, '>>')
@@ -467,6 +475,118 @@ class ErlangLiterateParser {
.replace(/\n\s*\n\s*\n/g, '\n\n') // Normalize multiple empty lines to double newlines
.replace(/[ \t]+$/gm, '') // Trim trailing spaces per line
.replace(/^\s+|\s+$/g, ''); // Final trim
+
+ // Reflow numbered lists and ensure separation from following headings/labels
+ cleaned = this.reflowNumberedLists(cleaned);
+
+ return cleaned;
+ }
+
+ formatReturnsText(text) {
+ if (!text) return '';
+ // First, clean the documentation text
+ let result = this.cleanDocumentation(text);
+
+ // Wrap leading return token if it's a tuple/list or common atom
+ const leadingMatch = result.match(/^(\s*)(\{[^}]+\}|\[[^\]]+\]|ok|error|not_found|true|false)(\b|\s|$)/i);
+ if (leadingMatch) {
+ const [, leadSpace, token, trail] = leadingMatch;
+ result = leadSpace + '`' + token + '`' + result.slice(leadSpace.length + token.length);
+ }
+
+ // Wrap any standalone tuple occurrences not already inside backticks
+ result = result.replace(/(^|[^`])(\{[^}]+\})([^`]|$)/g, (m, pre, tuple, post) => {
+ return `${pre}\`${tuple}\`${post}`;
+ });
+
+ return result;
+ }
+
+ splitReturnsIntoOutcomes(text) {
+ if (!text) return [];
+ const s = this.cleanDocumentation(text);
+ const tokenRegex = /(\{[^}]+\}|\bok\b|\berror\b|\bnot_found\b|\btrue\b|\bfalse\b)/gi;
+ const parts = [];
+ let match;
+ const matches = [];
+ while ((match = tokenRegex.exec(s)) !== null) {
+ matches.push({ index: match.index, token: match[0] });
+ }
+ // If no tokens or prose exists before the first token, don't split; keep as one descriptive line
+ if (matches.length === 0 || (matches.length > 0 && matches[0].index > 0)) {
+ return [s.trim()];
+ }
+ for (let i = 0; i < matches.length; i++) {
+ const start = matches[i].index;
+ const nextStart = (i + 1 < matches.length) ? matches[i + 1].index : s.length;
+ let segment = s.slice(start, nextStart).trim();
+ // Remove leading commas that were used as separators
+ segment = segment.replace(/^,\s*/, '');
+ // If there's trailing comma before next token, trim it but keep sentence end
+ segment = segment.replace(/,\s*$/, '');
+ if (segment) parts.push(segment.trim());
+ }
+ // If we accidentally merged two outcomes without clear token boundaries, ensure uniqueness
+ return parts.filter(p => p.length > 0);
+ }
+
+ reflowNumberedLists(text) {
+ if (!text) return '';
+ const lines = text.split('\n');
+ const out = [];
+ let inNumbered = false;
+ let lastNumIndex = -1;
+ for (let i = 0; i < lines.length; i++) {
+ const raw = lines[i];
+ const trimmed = raw.trim();
+
+ const isNumbered = /^\d+\.\s/.test(trimmed);
+ const isBullet = /^[-*]\s/.test(trimmed);
+ const isHeading = /^#{1,6}\s/.test(trimmed);
+ const isCodeFence = /^```/.test(trimmed);
+
+ if (isNumbered) {
+ out.push(trimmed);
+ inNumbered = true;
+ lastNumIndex = out.length - 1;
+ continue;
+ }
+
+ if (inNumbered) {
+ if (trimmed === '') {
+ out.push('');
+ inNumbered = false;
+ lastNumIndex = -1;
+ continue;
+ }
+ if (!isNumbered && !isBullet && !isHeading && !isCodeFence) {
+ // Continuation of previous numbered item; append
+ out[lastNumIndex] = out[lastNumIndex] + ' ' + trimmed;
+ continue;
+ }
+ // Different kind of line; end numbered block and fall through
+ inNumbered = false;
+ lastNumIndex = -1;
+ }
+
+ out.push(raw);
+ }
+
+ // Ensure a blank line between last numbered item and a label/heading line like 'Config options ...:'
+ const separated = [];
+ for (let i = 0; i < out.length; i++) {
+ const cur = out[i];
+ const next = i + 1 < out.length ? out[i + 1] : '';
+ separated.push(cur);
+ if (/^\d+\.\s/.test(cur.trim()) && next && !/^\s*$/.test(next) && /:\s*$/.test(next.trim())) {
+ // Insert a blank line if not already present
+ if (separated[separated.length - 1] !== '') {
+ separated.push('');
+ }
+ }
+ }
+
+ return separated.join('\n');
}
formatPreContent(content) {
@@ -561,6 +681,7 @@ class ErlangLiterateParser {
let currentSection = 'description';
let currentParam = null;
+ let lastReturnIndex = -1;
for (const line of lines) {
const trimmed = line.trim();
@@ -587,17 +708,33 @@ class ErlangLiterateParser {
}
const returnsText = trimmed.replace(/^@returns?\s*/, '');
result.returns.push(returnsText);
+ lastReturnIndex = result.returns.length - 1;
currentSection = 'returns';
continue;
}
// Add to current section
- if (currentSection === 'description' && trimmed) {
- result.description.push(trimmed);
+ if (currentSection === 'description') {
+ if (trimmed) {
+ result.description.push(trimmed);
+ } else {
+ // Preserve a single blank line to break paragraphs/lists
+ const last = result.description[result.description.length - 1];
+ if (last !== '') {
+ result.description.push('');
+ }
+ }
} else if (currentSection === 'param' && currentParam && trimmed) {
currentParam.description += ' ' + trimmed;
} else if (currentSection === 'returns' && trimmed) {
- result.returns.push(trimmed);
+ if (lastReturnIndex >= 0) {
+ // Append continuation text to the last returns entry
+ result.returns[lastReturnIndex] =
+ (result.returns[lastReturnIndex] + ' ' + trimmed).replace(/\s+/g, ' ').trim();
+ } else {
+ result.returns.push(trimmed);
+ lastReturnIndex = result.returns.length - 1;
+ }
}
}
@@ -747,8 +884,9 @@ class ErlangLiterateParser {
if (parsed.returns.length > 0) {
combinedDoc.push('### Returns');
combinedDoc.push('');
- for (const ret of parsed.returns) {
- combinedDoc.push(`- ${this.cleanDocumentation(ret)}`);
+ const expanded = parsed.returns.flatMap(r => this.splitReturnsIntoOutcomes(r));
+ for (const ret of expanded) {
+ combinedDoc.push(`- ${this.formatReturnsText(ret)}`);
}
combinedDoc.push('');
}
From 1dd667e6516b7b9f65ffb612a47c47bf5bb220a6 Mon Sep 17 00:00:00 2001
From: Dylan Shade <63427984+dpshade@users.noreply.github.com>
Date: Fri, 19 Sep 2025 17:27:39 -0400
Subject: [PATCH 15/17] docs: Refactor Erlang literate parser for improved
performance and documentation handling
- Introduce character codes and string constants for faster comparisons and reduced memory allocations.
- Optimize regex patterns for performance and clarity in parsing Erlang files.
- Enhance the reset method to initialize module information with default values.
- Update function processing to utilize constants for improved readability and maintainability.
- Revise documentation generation methods to ensure consistent formatting and structure in output.
---
docs/book/src/SUMMARY.md | 126 +++---
docs/erlang-literate-parser.js | 773 ++++++++++++++++-----------------
2 files changed, 429 insertions(+), 470 deletions(-)
diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md
index b2f5260c3..64394767e 100644
--- a/docs/book/src/SUMMARY.md
+++ b/docs/book/src/SUMMARY.md
@@ -11,6 +11,24 @@
- [ar_tx](ar_tx.erl.md)
- [ar_wallet](ar_wallet.erl.md)
+# Codec Modules
+
+- [dev_codec_ans104](dev_codec_ans104.erl.md)
+- [dev_codec_ans104_from](dev_codec_ans104_from.erl.md)
+- [dev_codec_ans104_to](dev_codec_ans104_to.erl.md)
+- [dev_codec_cookie](dev_codec_cookie.erl.md)
+- [dev_codec_cookie_auth](dev_codec_cookie_auth.erl.md)
+- [dev_codec_cookie_test_vectors](dev_codec_cookie_test_vectors.erl.md)
+- [dev_codec_flat](dev_codec_flat.erl.md)
+- [dev_codec_http_auth](dev_codec_http_auth.erl.md)
+- [dev_codec_httpsig](dev_codec_httpsig.erl.md)
+- [dev_codec_httpsig_conv](dev_codec_httpsig_conv.erl.md)
+- [dev_codec_httpsig_keyid](dev_codec_httpsig_keyid.erl.md)
+- [dev_codec_httpsig_proxy](dev_codec_httpsig_proxy.erl.md)
+- [dev_codec_httpsig_siginfo](dev_codec_httpsig_siginfo.erl.md)
+- [dev_codec_json](dev_codec_json.erl.md)
+- [dev_codec_structured](dev_codec_structured.erl.md)
+
# Devices
- [dev_apply](dev_apply.erl.md)
@@ -30,70 +48,50 @@
- [dev_arweave_block_cache](dev_arweave_block_cache.erl.md)
- [dev_auth_hook](dev_auth_hook.erl.md)
- [dev_secret](dev_secret.erl.md)
-
-# Codec Modules
-
-- [dev_codec_ans104](dev_codec_ans104.erl.md)
-- [dev_codec_ans104_from](dev_codec_ans104_from.erl.md)
-- [dev_codec_ans104_to](dev_codec_ans104_to.erl.md)
-- [dev_codec_cookie](dev_codec_cookie.erl.md)
-- [dev_codec_cookie_auth](dev_codec_cookie_auth.erl.md)
-- [dev_codec_cookie_test_vectors](dev_codec_cookie_test_vectors.erl.md)
-- [dev_codec_flat](dev_codec_flat.erl.md)
-- [dev_codec_http_auth](dev_codec_http_auth.erl.md)
-- [dev_codec_httpsig](dev_codec_httpsig.erl.md)
-- [dev_codec_httpsig_conv](dev_codec_httpsig_conv.erl.md)
-- [dev_codec_httpsig_keyid](dev_codec_httpsig_keyid.erl.md)
-- [dev_codec_httpsig_proxy](dev_codec_httpsig_proxy.erl.md)
-- [dev_codec_httpsig_siginfo](dev_codec_httpsig_siginfo.erl.md)
-- [dev_codec_json](dev_codec_json.erl.md)
-- [dev_codec_structured](dev_codec_structured.erl.md)
-
-
- - [dev_delegated_compute](dev_delegated_compute.erl.md)
- - [dev_genesis_wasm](dev_genesis_wasm.erl.md)
- - [dev_green_zone](dev_green_zone.erl.md)
- - [dev_hook](dev_hook.erl.md)
- - [dev_hyperbuddy](dev_hyperbuddy.erl.md)
- - [dev_json_iface](dev_json_iface.erl.md)
- - [dev_local_name](dev_local_name.erl.md)
- - [dev_lookup](dev_lookup.erl.md)
- - [dev_lua](dev_lua.erl.md)
- - [dev_lua_lib](dev_lua_lib.erl.md)
- - [dev_lua_test](dev_lua_test.erl.md)
- - [dev_lua_test_ledgers](dev_lua_test_ledgers.erl.md)
- - [dev_manifest](dev_manifest.erl.md)
- - [dev_message](dev_message.erl.md)
- - [dev_meta](dev_meta.erl.md)
- - [dev_name](dev_name.erl.md)
- - [dev_node_process](dev_node_process.erl.md)
- - [dev_p4](dev_p4.erl.md)
- - [dev_patch](dev_patch.erl.md)
- - [dev_poda](dev_poda.erl.md)
- - [dev_process](dev_process.erl.md)
- - [dev_process_cache](dev_process_cache.erl.md)
- - [dev_process_worker](dev_process_worker.erl.md)
- - [dev_profile](dev_profile.erl.md)
- - [dev_push](dev_push.erl.md)
- - [dev_query](dev_query.erl.md)
- - [dev_query_arweave](dev_query_arweave.erl.md)
- - [dev_query_graphql](dev_query_graphql.erl.md)
- - [dev_query_test_vectors](dev_query_test_vectors.erl.md)
- - [dev_relay](dev_relay.erl.md)
- - [dev_router](dev_router.erl.md)
- - [dev_scheduler](dev_scheduler.erl.md)
- - [dev_scheduler_cache](dev_scheduler_cache.erl.md)
- - [dev_scheduler_formats](dev_scheduler_formats.erl.md)
- - [dev_scheduler_registry](dev_scheduler_registry.erl.md)
- - [dev_scheduler_server](dev_scheduler_server.erl.md)
- - [dev_simple_pay](dev_simple_pay.erl.md)
- - [dev_snp](dev_snp.erl.md)
- - [dev_snp_nif](dev_snp_nif.erl.md)
- - [dev_stack](dev_stack.erl.md)
- - [dev_volume](dev_volume.erl.md)
- - [dev_wasi](dev_wasi.erl.md)
- - [dev_wasm](dev_wasm.erl.md)
- - [dev_whois](dev_whois.erl.md)
+- [dev_delegated_compute](dev_delegated_compute.erl.md)
+- [dev_genesis_wasm](dev_genesis_wasm.erl.md)
+- [dev_green_zone](dev_green_zone.erl.md)
+- [dev_hook](dev_hook.erl.md)
+- [dev_hyperbuddy](dev_hyperbuddy.erl.md)
+- [dev_json_iface](dev_json_iface.erl.md)
+- [dev_local_name](dev_local_name.erl.md)
+- [dev_lookup](dev_lookup.erl.md)
+- [dev_lua](dev_lua.erl.md)
+- [dev_lua_lib](dev_lua_lib.erl.md)
+- [dev_lua_test](dev_lua_test.erl.md)
+- [dev_lua_test_ledgers](dev_lua_test_ledgers.erl.md)
+- [dev_manifest](dev_manifest.erl.md)
+- [dev_message](dev_message.erl.md)
+- [dev_meta](dev_meta.erl.md)
+- [dev_name](dev_name.erl.md)
+- [dev_node_process](dev_node_process.erl.md)
+- [dev_p4](dev_p4.erl.md)
+- [dev_patch](dev_patch.erl.md)
+- [dev_poda](dev_poda.erl.md)
+- [dev_process](dev_process.erl.md)
+- [dev_process_cache](dev_process_cache.erl.md)
+- [dev_process_worker](dev_process_worker.erl.md)
+- [dev_profile](dev_profile.erl.md)
+- [dev_push](dev_push.erl.md)
+- [dev_query](dev_query.erl.md)
+- [dev_query_arweave](dev_query_arweave.erl.md)
+- [dev_query_graphql](dev_query_graphql.erl.md)
+- [dev_query_test_vectors](dev_query_test_vectors.erl.md)
+- [dev_relay](dev_relay.erl.md)
+- [dev_router](dev_router.erl.md)
+- [dev_scheduler](dev_scheduler.erl.md)
+- [dev_scheduler_cache](dev_scheduler_cache.erl.md)
+- [dev_scheduler_formats](dev_scheduler_formats.erl.md)
+- [dev_scheduler_registry](dev_scheduler_registry.erl.md)
+- [dev_scheduler_server](dev_scheduler_server.erl.md)
+- [dev_simple_pay](dev_simple_pay.erl.md)
+- [dev_snp](dev_snp.erl.md)
+- [dev_snp_nif](dev_snp_nif.erl.md)
+- [dev_stack](dev_stack.erl.md)
+- [dev_volume](dev_volume.erl.md)
+- [dev_wasi](dev_wasi.erl.md)
+- [dev_wasm](dev_wasm.erl.md)
+- [dev_whois](dev_whois.erl.md)
# HyperBEAM Core
diff --git a/docs/erlang-literate-parser.js b/docs/erlang-literate-parser.js
index 5f276307c..56dfd914c 100755
--- a/docs/erlang-literate-parser.js
+++ b/docs/erlang-literate-parser.js
@@ -12,7 +12,79 @@
import fs from 'fs';
import path from 'path';
-import { execSync } from 'child_process';
+
+// Character codes for faster comparisons
+const CHAR_CODES = {
+ PERCENT: 37, // '%'
+ DASH: 45, // '-'
+ OPEN_PAREN: 40, // '('
+ CLOSE_PAREN: 41, // ')'
+ OPEN_BRACE: 123, // '{'
+ CLOSE_BRACE: 125, // '}'
+ OPEN_BRACKET: 91, // '['
+ CLOSE_BRACKET: 93, // ']'
+ DOT: 46 // '.'
+};
+
+// String constants to avoid repeated allocations
+const STRINGS = {
+ EMPTY: '',
+ SPACE: ' ',
+ NEWLINE: '\n',
+ TRIPLE_PERCENT: '%%%',
+ DOUBLE_PERCENT: '%%',
+ SINGLE_PERCENT: '%',
+ SPEC_PREFIX: '-spec ',
+ ERLANG: 'erlang',
+ BACKTICK: '`',
+ PARAM_TAG: '@param',
+ RETURNS_TAG: '@returns',
+ PARAMETERS_HEADER: '### Parameters',
+ RETURNS_HEADER: '### Returns',
+ EXPORTED_FUNCTIONS: '## Exported Functions',
+ SEPARATOR: '---'
+};
+
+// Precompiled regex patterns for performance
+const REGEX = {
+ MODULE: /^-module\(([^)]+)\)/,
+ EXPORT: /^-export\(\[([^\]]+)\]\)/,
+ EXPORT_PREFIX: /^-export\(\[/,
+ SPEC: /^-spec\s+([a-z][a-z0-9_]*)\s*\(/,
+ FUNCTION: /^([a-z][a-z0-9_]*)\s*\(/,
+ DOC_BLOCK_START: /^%% @doc/,
+ COMMENT_SINGLE: /^\s*%[^%]/,
+ COMMENT_DOUBLE: /^\s*%%[^%]/,
+ BACKTICK_QUOTE: /`([^']*?)'/g,
+ HTML_ENTITIES_LT: /<</g,
+ HTML_ENTITIES_GT: />>/g,
+ PRE_TAG: /([\s\S]*?)<\/pre>/g,
+ // Match numbered list items that start with either "1." or "1:" (allow optional space before the punctuation)
+ NUMBERED_LIST: /^\d+\s*[.:]\s/,
+ BULLET_LIST: /^[-*]\s/,
+ HEADING: /^#{1,6}\s/,
+ CODE_FENCE: /^```/,
+ RETURNS_TOKENS: /(\{[^}]+\}|\bok\b|\berror\b|\bnot_found\b|\btrue\b|\bfalse\b)/gi,
+ LEADING_RETURN_TOKEN: /^(\s*)(\{[^}]+\}|\[[^\]]+\]|ok|error|not_found|true|false)(\b|\s|$)/i,
+ STANDALONE_TUPLE: /(^|[^`])(\{[^}]+\})([^`]|$)/g,
+ PARAM: /^@param\s+(\S+)\s*(.*)/,
+ RETURNS: /^@returns?\s*/,
+ OPTION_DEF: /^(`[^`]+`)\s*:\s*(.*)$/,
+ DEFINITION: /^(\S+(?:\s*:\s*\S+)?)\s*:\s*(.*)$/,
+ MULTIPLE_NEWLINES: /\n\s*\n\s*\n/g,
+ TRAILING_SPACES: /[ \t]+$/gm,
+ TRIM: /^\s+|\s+$/g,
+ RETURNS_LIKE_TUPLE: /^`?\{[^}]+\}`?/,
+ RETURNS_LIKE_ATOM: /^`?(ok|error|not_found|true|false)\b/i,
+ REMOVE_DOC: /@doc\s*/g,
+ WHITESPACE_NORMALIZE: /\s+/g,
+ COMMA_START: /^,\s*/,
+ COMMA_END: /,\s*$/,
+ EMPTY_LINE: /^\s*$/,
+ COLON_END: /:\s*$/,
+ COMMENT_DOUBLE_PREFIX: /^\s*%%\s?/,
+ COMMENT_SINGLE_PREFIX: /^\s*%\s?/
+};
class ErlangLiterateParser {
constructor(options = {}) {
@@ -26,16 +98,16 @@ class ErlangLiterateParser {
reset() {
this.lines = [];
- this.moduleInfo = {};
+ this.moduleInfo = { name: null, doc: null, exports: null };
this.functions = [];
this.currentState = {
inFunction: false,
- functionName: '',
- functionSpec: '',
- functionDoc: '',
+ functionName: STRINGS.EMPTY,
+ functionSpec: STRINGS.EMPTY,
+ functionDoc: STRINGS.EMPTY,
functionLines: [],
- pendingDoc: '',
- specFunctionName: '',
+ pendingDoc: STRINGS.EMPTY,
+ specFunctionName: STRINGS.EMPTY,
braceDepth: 0,
parenDepth: 0,
inlineDocTags: []
@@ -45,296 +117,248 @@ class ErlangLiterateParser {
parseFile(filePath) {
const content = fs.readFileSync(filePath, 'utf8');
this.reset();
- this.lines = content.split('\n');
+ this.lines = content.split(STRINGS.NEWLINE);
- // Extract module-level information
this.extractModuleInfo();
-
- // Process all lines for functions
this.processFunctions();
- // Generate the markdown
return this.generateMarkdown(path.basename(filePath));
}
extractModuleInfo() {
- let moduleDoc = [];
+ const moduleDoc = [];
let inModuleDoc = false;
+ const linesLength = this.lines.length;
- for (let i = 0; i < this.lines.length; i++) {
+ for (let i = 0; i < linesLength; i++) {
const line = this.lines[i];
const trimmed = line.trim();
- // Module name
- if (trimmed.match(/^-module\(([^)]+)\)/)) {
- this.moduleInfo.name = trimmed.match(/^-module\(([^)]+)\)/)[1];
+ if (!trimmed) {
+ if (inModuleDoc) moduleDoc.push(STRINGS.EMPTY);
+ continue;
+ }
+
+ // Fast character check before regex
+ const firstChar = trimmed.charCodeAt(0);
+
+ // Module name - check for dash first
+ if (firstChar === CHAR_CODES.DASH && trimmed.startsWith('-module(')) {
+ const moduleMatch = trimmed.match(REGEX.MODULE);
+ if (moduleMatch) {
+ this.moduleInfo.name = moduleMatch[1];
+ }
+ continue;
}
- // Exports
- if (trimmed.match(/^-export\(\[/)) {
- const exportMatch = trimmed.match(/^-export\(\[([^\]]+)\]\)/);
+ // Exports - check for dash first
+ if (firstChar === CHAR_CODES.DASH && REGEX.EXPORT_PREFIX.test(trimmed)) {
+ const exportMatch = trimmed.match(REGEX.EXPORT);
if (exportMatch) {
this.moduleInfo.exports = exportMatch[1]
.split(',')
.map(e => e.trim())
- .filter(e => e);
+ .filter(Boolean);
}
+ continue;
}
- // Module documentation (%%% comments at the top)
- if (trimmed.startsWith('%%%')) {
+ // Module documentation - check for percent first
+ if (firstChar === CHAR_CODES.PERCENT && trimmed.startsWith(STRINGS.TRIPLE_PERCENT)) {
inModuleDoc = true;
let docLine = trimmed.substring(3).trim();
- // Remove @doc if present
- docLine = docLine.replace(/^@doc\s*/, '');
- // Always push the line, even if empty (for paragraph breaks)
+ docLine = docLine.replace(REGEX.REMOVE_DOC, STRINGS.EMPTY);
moduleDoc.push(docLine);
- } else if (inModuleDoc && trimmed === '') {
- // Empty line in module doc, preserve it for paragraph breaks
- moduleDoc.push('');
- } else if (inModuleDoc && trimmed.startsWith('%')) {
- // Continue with other comment types but end module doc
- inModuleDoc = false;
- break;
- } else if (inModuleDoc && trimmed.startsWith('-')) {
- // Hit a module directive, end of module doc
+ } else if (inModuleDoc && (firstChar === CHAR_CODES.PERCENT || firstChar === CHAR_CODES.DASH)) {
break;
}
}
- this.moduleInfo.doc = this.cleanDocumentation(moduleDoc.join('\n'));
+ this.moduleInfo.doc = this.cleanDocumentation(moduleDoc.join(STRINGS.NEWLINE));
}
processFunctions() {
- for (let i = 0; i < this.lines.length; i++) {
+ const linesLength = this.lines.length;
+
+ for (let i = 0; i < linesLength; i++) {
const line = this.lines[i];
const trimmed = line.trim();
+ if (!trimmed) continue;
+
// Check for start of function documentation block
- if (trimmed.startsWith('%% @doc')) {
+ if (REGEX.DOC_BLOCK_START.test(trimmed)) {
this.collectFunctionDoc(i);
continue;
}
// Check for -spec
- if (trimmed.startsWith('-spec ')) {
- // Before collecting the spec, check if there are @param/@returns tags immediately before
+ if (trimmed.startsWith(STRINGS.SPEC_PREFIX)) {
this.collectParamTagsBeforeSpec(i);
this.collectSpec(i);
- // Extract function name from spec line
- const specMatch = trimmed.match(/-spec\s+([a-z][a-z0-9_]*)\s*\(/);
+ const specMatch = trimmed.match(REGEX.SPEC);
if (specMatch) {
this.currentState.specFunctionName = specMatch[1];
}
continue;
}
- // Check for function start (handles both start-of-line and indented functions)
- const funcMatch = trimmed.match(/^([a-z][a-z0-9_]*)\s*\(/);
+ // Check for function start
+ const funcMatch = trimmed.match(REGEX.FUNCTION);
if (funcMatch && !this.currentState.inFunction) {
- // Save any pending doc
if (this.currentState.pendingDoc) {
this.currentState.functionDoc = this.currentState.pendingDoc;
- this.currentState.pendingDoc = '';
+ this.currentState.pendingDoc = STRINGS.EMPTY;
}
- // Use function name from spec if available, otherwise use detected name
const functionName = this.currentState.specFunctionName || funcMatch[1];
- this.startFunction(functionName, i);
-
- // Clear the spec function name after use
- this.currentState.specFunctionName = '';
+ this.startFunction(functionName);
+ this.currentState.specFunctionName = STRINGS.EMPTY;
}
// If in function, collect lines
if (this.currentState.inFunction) {
- this.collectFunctionLine(line, i);
-
- // Check for function end
+ this.collectFunctionLine(line);
if (this.isFunctionEnd(line)) {
this.endFunction();
}
}
}
- // Handle any remaining function
if (this.currentState.inFunction) {
this.endFunction();
}
}
- isFunctionDoc(line) {
- return line.startsWith('%% @doc') ||
- (line.startsWith('%%') && !line.startsWith('%%%'));
- }
-
collectFunctionDoc(startIdx) {
const docLines = [];
+ const linesLength = this.lines.length;
- for (let i = startIdx; i < this.lines.length; i++) {
+ for (let i = startIdx; i < linesLength; i++) {
const line = this.lines[i];
const trimmed = line.trim();
- if (trimmed.startsWith('%%')) {
+ if (trimmed.startsWith(STRINGS.DOUBLE_PERCENT)) {
let docLine = trimmed.substring(2).trim();
- // Remove @doc prefix if on first line
if (i === startIdx) {
- docLine = docLine.replace(/^@doc\s*/, '');
+ docLine = docLine.replace(REGEX.REMOVE_DOC, STRINGS.EMPTY);
}
docLines.push(docLine);
- } else if (trimmed === '') {
- // Empty line - might be part of doc block or end
- // Look ahead to see if more %% comments follow
+ } else if (!trimmed) {
+ // Look ahead for more %% comments
let j = i + 1;
- let foundMoreDoc = false;
- while (j < this.lines.length && this.lines[j].trim() === '') {
- j++;
- }
- if (j < this.lines.length && this.lines[j].trim().startsWith('%%')) {
- // More doc coming, include empty line
- docLines.push('');
- foundMoreDoc = true;
- }
- if (!foundMoreDoc) {
- // End of doc block
+ while (j < linesLength && !this.lines[j].trim()) j++;
+
+ if (j < linesLength && this.lines[j].trim().startsWith(STRINGS.DOUBLE_PERCENT)) {
+ docLines.push(STRINGS.EMPTY);
+ } else {
break;
}
- } else if (trimmed.startsWith('%')) {
- // Single % comment, continue but don't include
- continue;
- } else {
- // End of doc block
+ } else if (!trimmed.startsWith(STRINGS.SINGLE_PERCENT)) {
break;
}
}
- this.currentState.pendingDoc = docLines.join('\n');
+ this.currentState.pendingDoc = docLines.join(STRINGS.NEWLINE);
}
collectParamTagsBeforeSpec(specIdx) {
const paramLines = [];
let hitDocBlock = false;
- // Look backwards from the -spec line to find @param and @returns tags
for (let i = specIdx - 1; i >= 0; i--) {
- const line = this.lines[i];
- const trimmed = line.trim();
+ const trimmed = this.lines[i].trim();
- // If we hit a @doc line, we already collected this documentation
- if (trimmed.startsWith('%% @doc')) {
+ if (REGEX.DOC_BLOCK_START.test(trimmed)) {
hitDocBlock = true;
break;
}
- // If we hit an empty line or a non-comment line, stop looking backwards
- if (trimmed === '' || (!trimmed.startsWith('%%') && !trimmed.startsWith('%'))) {
+ if (!trimmed || (!trimmed.startsWith(STRINGS.DOUBLE_PERCENT) && !trimmed.startsWith(STRINGS.SINGLE_PERCENT))) {
break;
}
- // Check if this line contains @param or @returns
- if (trimmed.startsWith('%%') && (trimmed.includes('@param') || trimmed.includes('@returns'))) {
- let docLine = trimmed.substring(2).trim();
- paramLines.unshift(docLine); // Add to beginning to maintain order
+ if (trimmed.startsWith(STRINGS.DOUBLE_PERCENT) &&
+ (trimmed.includes(STRINGS.PARAM_TAG) || trimmed.includes(STRINGS.RETURNS_TAG))) {
+ paramLines.unshift(trimmed.substring(2).trim());
}
}
- // Only add the tags if we didn't find a @doc block (meaning these are standalone tags)
if (paramLines.length > 0 && !hitDocBlock) {
- const existingDoc = this.currentState.pendingDoc || '';
- const newDoc = paramLines.join('\n');
-
- if (existingDoc) {
- this.currentState.pendingDoc = existingDoc + '\n' + newDoc;
- } else {
- this.currentState.pendingDoc = newDoc;
- }
+ const existingDoc = this.currentState.pendingDoc;
+ const newDoc = paramLines.join(STRINGS.NEWLINE);
+ this.currentState.pendingDoc = existingDoc ?
+ existingDoc + STRINGS.NEWLINE + newDoc : newDoc;
}
}
collectSpec(startIdx) {
const specLines = [];
let depth = 0;
+ const linesLength = this.lines.length;
- for (let i = startIdx; i < this.lines.length; i++) {
+ for (let i = startIdx; i < linesLength; i++) {
const line = this.lines[i];
specLines.push(line);
- // Track parentheses to handle multi-line specs
- for (const char of line) {
- if (char === '(') depth++;
- if (char === ')') depth--;
+ // Fast character-based depth tracking
+ for (let j = 0, len = line.length; j < len; j++) {
+ const charCode = line.charCodeAt(j);
+ if (charCode === CHAR_CODES.OPEN_PAREN) depth++;
+ else if (charCode === CHAR_CODES.CLOSE_PAREN) depth--;
}
- // Check if spec is complete
if (line.trim().endsWith('.') && depth === 0) {
break;
}
}
- this.currentState.functionSpec = specLines.join('\n');
-
- // After collecting spec, look for the actual function definition
- // that should follow shortly after
- for (let j = startIdx + specLines.length; j < this.lines.length; j++) {
- const nextLine = this.lines[j].trim();
-
- // Skip empty lines and comments
- if (nextLine === '' || nextLine.startsWith('%')) {
- continue;
- }
-
- // Look for function definition
- const funcMatch = nextLine.match(/^([a-z][a-z0-9_]*)\s*\(/);
- if (funcMatch) {
- // This is likely the function that corresponds to this spec
- // But don't start the function here, let the main loop handle it
- break;
- }
-
- // If we hit another -spec or module directive, stop looking
- if (nextLine.startsWith('-spec') || nextLine.startsWith('-')) {
- break;
- }
- }
+ this.currentState.functionSpec = specLines.join(STRINGS.NEWLINE);
}
- startFunction(name, lineIdx) {
+ startFunction(name) {
this.currentState.inFunction = true;
this.currentState.functionName = name;
- this.currentState.functionLines = [];
+ this.currentState.functionLines.length = 0;
this.currentState.braceDepth = 0;
this.currentState.parenDepth = 0;
- this.currentState.inlineDocTags = [];
+ this.currentState.inlineDocTags.length = 0;
}
- collectFunctionLine(line, lineIdx) {
+ collectFunctionLine(line) {
this.currentState.functionLines.push(line);
- // Track depth for function end detection
- for (const char of line) {
- if (char === '{' || char === '[') this.currentState.braceDepth++;
- if (char === '}' || char === ']') this.currentState.braceDepth--;
- if (char === '(') this.currentState.parenDepth++;
- if (char === ')') this.currentState.parenDepth--;
+ // Fast character-based depth tracking
+ for (let i = 0, len = line.length; i < len; i++) {
+ const charCode = line.charCodeAt(i);
+ switch (charCode) {
+ case CHAR_CODES.OPEN_BRACE:
+ case CHAR_CODES.OPEN_BRACKET:
+ this.currentState.braceDepth++;
+ break;
+ case CHAR_CODES.CLOSE_BRACE:
+ case CHAR_CODES.CLOSE_BRACKET:
+ this.currentState.braceDepth--;
+ break;
+ case CHAR_CODES.OPEN_PAREN:
+ this.currentState.parenDepth++;
+ break;
+ case CHAR_CODES.CLOSE_PAREN:
+ this.currentState.parenDepth--;
+ break;
+ }
}
}
isFunctionEnd(line) {
const trimmed = line.trim();
-
- // Function ends with . at depth 0
- if (this.currentState.braceDepth === 0 &&
- this.currentState.parenDepth === 0 &&
- trimmed.endsWith('.') &&
- !trimmed.startsWith('%')) {
- return true;
- }
-
- return false;
+ return this.currentState.braceDepth === 0 &&
+ this.currentState.parenDepth === 0 &&
+ trimmed.charCodeAt(trimmed.length - 1) === CHAR_CODES.DOT &&
+ trimmed.charCodeAt(0) !== CHAR_CODES.PERCENT;
}
endFunction() {
- // Process the function body to extract inline comments
const processedBody = this.processFunctionBody(this.currentState.functionLines);
this.functions.push({
@@ -344,206 +368,191 @@ class ErlangLiterateParser {
body: processedBody
});
- // Reset state
+ // Reset state efficiently
this.currentState.inFunction = false;
- this.currentState.functionName = '';
- this.currentState.functionSpec = '';
- this.currentState.functionDoc = '';
- this.currentState.functionLines = [];
- this.currentState.specFunctionName = '';
- this.currentState.inlineDocTags = [];
+ this.currentState.functionName = STRINGS.EMPTY;
+ this.currentState.functionSpec = STRINGS.EMPTY;
+ this.currentState.functionDoc = STRINGS.EMPTY;
+ this.currentState.functionLines.length = 0;
+ this.currentState.specFunctionName = STRINGS.EMPTY;
+ this.currentState.inlineDocTags.length = 0;
}
processFunctionBody(lines) {
const segments = [];
- let currentCode = [];
- let pendingTagLines = [];
+ const currentCode = [];
+ const pendingTagLines = [];
const flushCode = () => {
if (currentCode.length > 0) {
- segments.push({ type: 'code', content: currentCode.join('\n') });
- currentCode = [];
+ segments.push({ type: 'code', content: currentCode.join(STRINGS.NEWLINE) });
+ currentCode.length = 0;
}
};
const flushTags = () => {
if (pendingTagLines.length > 0) {
- const tagText = pendingTagLines.join('\n');
+ const tagText = pendingTagLines.join(STRINGS.NEWLINE);
const parsed = this.parseDocumentation(tagText);
- let docParts = [];
+ const docParts = [];
+
if (parsed.params.length > 0) {
- docParts.push('### Parameters');
- docParts.push('');
- for (const p of parsed.params) {
- const desc = this.cleanDocumentation(p.description || '');
- docParts.push(`- \`${p.name}\` - ${desc}`);
- }
- docParts.push('');
+ docParts.push(STRINGS.PARAMETERS_HEADER, STRINGS.EMPTY);
+ parsed.params.forEach(p => {
+ const desc = this.cleanDocumentation(p.description || STRINGS.EMPTY);
+ docParts.push(`- ${STRINGS.BACKTICK}${p.name}${STRINGS.BACKTICK} - ${desc}`);
+ });
+ docParts.push(STRINGS.EMPTY);
}
+
if (parsed.returns.length > 0) {
- docParts.push('### Returns');
- docParts.push('');
+ docParts.push(STRINGS.RETURNS_HEADER, STRINGS.EMPTY);
const expanded = parsed.returns.flatMap(r => this.splitReturnsIntoOutcomes(r));
- for (const r of expanded) {
- docParts.push(`- ${this.formatReturnsText(r)}`);
- }
- docParts.push('');
+ expanded.forEach(r => docParts.push(`- ${this.formatReturnsText(r)}`));
+ docParts.push(STRINGS.EMPTY);
}
+
if (docParts.length > 0) {
- segments.push({ type: 'doc', content: docParts.join('\n') });
+ segments.push({ type: 'doc', content: docParts.join(STRINGS.NEWLINE) });
}
- pendingTagLines = [];
+ pendingTagLines.length = 0;
}
};
for (const line of lines) {
const trimmed = line.trim();
- if (trimmed.match(/^\s*%[^%]/) || trimmed.match(/^\s*%%[^%]/)) {
- // It's a comment line
- // Save any accumulated code block first
- if (currentCode.length > 0) {
- flushCode();
- }
+ if (REGEX.COMMENT_SINGLE.test(trimmed) || REGEX.COMMENT_DOUBLE.test(trimmed)) {
+ flushCode();
- // Extract comment text (remove % or %% prefix)
- let commentText;
- if (trimmed.match(/^\s*%%[^%]/)) {
- commentText = line.replace(/^\s*%%\s?/, '');
- } else {
- commentText = line.replace(/^\s*%\s?/, '');
- }
+ const commentText = REGEX.COMMENT_DOUBLE.test(trimmed)
+ ? line.replace(REGEX.COMMENT_DOUBLE_PREFIX, STRINGS.EMPTY)
+ : line.replace(REGEX.COMMENT_SINGLE_PREFIX, STRINGS.EMPTY);
const cleaned = this.cleanInlineComment(commentText);
- // Heuristic: returns-like lines (e.g., `{ok, Binary}` / `{error, Binary}` ...)
- const returnsLikeTuple = /^`?\{[^}]+\}`?/.test(cleaned);
- const returnsLikeAtom = /^`?(ok|error|not_found|true|false)\b/i.test(cleaned);
- const isTagParam = /^\s*@param\b/i.test(cleaned);
- const isTagReturns = /^\s*@returns?\b/i.test(cleaned);
+ const returnsLikeTuple = REGEX.RETURNS_LIKE_TUPLE.test(cleaned);
+ const returnsLikeAtom = REGEX.RETURNS_LIKE_ATOM.test(cleaned);
+ const isTagParam = cleaned.startsWith(STRINGS.PARAM_TAG);
+ const isTagReturns = cleaned.startsWith(STRINGS.RETURNS_TAG);
if (isTagParam || isTagReturns || returnsLikeTuple || returnsLikeAtom) {
- const lineAsTag = isTagParam || isTagReturns
+ const lineAsTag = (isTagParam || isTagReturns)
? cleaned.trim()
- : `@returns ${cleaned.trim()}`;
- // Accumulate tag lines
+ : `${STRINGS.RETURNS_TAG} ${cleaned.trim()}`;
pendingTagLines.push(lineAsTag);
- } else if (
- pendingTagLines.length > 0 &&
- (pendingTagLines[pendingTagLines.length - 1].startsWith('@returns') ||
- pendingTagLines[pendingTagLines.length - 1].startsWith('@param'))
- ) {
- // Continuation of the previous @returns/@param block; append line
- pendingTagLines.push(cleaned.trim());
+ } else if (pendingTagLines.length > 0) {
+ const lastTag = pendingTagLines[pendingTagLines.length - 1];
+ if (lastTag.startsWith(STRINGS.RETURNS_TAG) || lastTag.startsWith(STRINGS.PARAM_TAG)) {
+ pendingTagLines.push(cleaned.trim());
+ } else {
+ flushTags();
+ segments.push({ type: 'comment', content: cleaned });
+ }
} else {
- // Flush any pending tag block before emitting a normal comment
flushTags();
segments.push({ type: 'comment', content: cleaned });
}
} else {
- // Non-comment code line; flush any pending tags first, then add code
flushTags();
currentCode.push(line);
}
}
- // Flush any remaining tag or code blocks
flushTags();
flushCode();
-
return segments;
}
cleanInlineComment(text) {
- // Convert `thing' to `thing`
- return text.replace(/`([^']*?)'/g, '`$1`').trim();
+ return text.replace(REGEX.BACKTICK_QUOTE, `${STRINGS.BACKTICK}$1${STRINGS.BACKTICK}`).trim();
}
cleanDocumentation(text) {
- if (!text) return '';
+ if (!text) return STRINGS.EMPTY;
- // Handle tags with structured content
- text = text.replace(/([\s\S]*?)<\/pre>/g, (match, content) => {
- return this.formatPreContent(content);
- });
-
- // Convert Erlang doc syntax to Markdown
- let cleaned = text
- .replace(/`([^']*?)'/g, '`$1`') // Convert `code' to `code`
- .replace(/<</g, '<<') // Fix HTML entities
- .replace(/>>/g, '>>')
- .replace(/@doc\s*/g, '') // Remove @doc tags
- .replace(/\n\s*\n\s*\n/g, '\n\n') // Normalize multiple empty lines to double newlines
- .replace(/[ \t]+$/gm, '') // Trim trailing spaces per line
- .replace(/^\s+|\s+$/g, ''); // Final trim
+ text = text.replace(REGEX.PRE_TAG, (match, content) => this.formatPreContent(content));
- // Reflow numbered lists and ensure separation from following headings/labels
- cleaned = this.reflowNumberedLists(cleaned);
+ const cleaned = text
+ .replace(REGEX.BACKTICK_QUOTE, `${STRINGS.BACKTICK}$1${STRINGS.BACKTICK}`)
+ .replace(REGEX.HTML_ENTITIES_LT, '<<')
+ .replace(REGEX.HTML_ENTITIES_GT, '>>')
+ .replace(REGEX.REMOVE_DOC, STRINGS.EMPTY)
+ .replace(REGEX.MULTIPLE_NEWLINES, '\n\n')
+ .replace(REGEX.TRAILING_SPACES, STRINGS.EMPTY)
+ .replace(REGEX.TRIM, STRINGS.EMPTY);
- return cleaned;
+ return this.reflowNumberedLists(cleaned);
}
formatReturnsText(text) {
- if (!text) return '';
- // First, clean the documentation text
+ if (!text) return STRINGS.EMPTY;
let result = this.cleanDocumentation(text);
- // Wrap leading return token if it's a tuple/list or common atom
- const leadingMatch = result.match(/^(\s*)(\{[^}]+\}|\[[^\]]+\]|ok|error|not_found|true|false)(\b|\s|$)/i);
+ const leadingMatch = result.match(REGEX.LEADING_RETURN_TOKEN);
if (leadingMatch) {
- const [, leadSpace, token, trail] = leadingMatch;
- result = leadSpace + '`' + token + '`' + result.slice(leadSpace.length + token.length);
+ const [, leadSpace, token] = leadingMatch;
+ result = leadSpace + STRINGS.BACKTICK + token + STRINGS.BACKTICK +
+ result.slice(leadSpace.length + token.length);
}
- // Wrap any standalone tuple occurrences not already inside backticks
- result = result.replace(/(^|[^`])(\{[^}]+\})([^`]|$)/g, (m, pre, tuple, post) => {
- return `${pre}\`${tuple}\`${post}`;
- });
-
- return result;
+ return result.replace(REGEX.STANDALONE_TUPLE,
+ (m, pre, tuple, post) => `${pre}${STRINGS.BACKTICK}${tuple}${STRINGS.BACKTICK}${post}`);
}
splitReturnsIntoOutcomes(text) {
if (!text) return [];
const s = this.cleanDocumentation(text);
- const tokenRegex = /(\{[^}]+\}|\bok\b|\berror\b|\bnot_found\b|\btrue\b|\bfalse\b)/gi;
- const parts = [];
- let match;
const matches = [];
- while ((match = tokenRegex.exec(s)) !== null) {
+ let match;
+
+ // Reset regex lastIndex to avoid issues with global regex
+ REGEX.RETURNS_TOKENS.lastIndex = 0;
+ while ((match = REGEX.RETURNS_TOKENS.exec(s)) !== null) {
matches.push({ index: match.index, token: match[0] });
}
- // If no tokens or prose exists before the first token, don't split; keep as one descriptive line
- if (matches.length === 0 || (matches.length > 0 && matches[0].index > 0)) {
+
+ if (matches.length === 0 || matches[0].index > 0) {
return [s.trim()];
}
- for (let i = 0; i < matches.length; i++) {
+
+ const parts = [];
+ const matchesLength = matches.length;
+ for (let i = 0; i < matchesLength; i++) {
const start = matches[i].index;
- const nextStart = (i + 1 < matches.length) ? matches[i + 1].index : s.length;
- let segment = s.slice(start, nextStart).trim();
- // Remove leading commas that were used as separators
- segment = segment.replace(/^,\s*/, '');
- // If there's trailing comma before next token, trim it but keep sentence end
- segment = segment.replace(/,\s*$/, '');
+ const nextStart = (i + 1 < matchesLength) ? matches[i + 1].index : s.length;
+ let segment = s.slice(start, nextStart).trim()
+ .replace(REGEX.COMMA_START, STRINGS.EMPTY)
+ .replace(REGEX.COMMA_END, STRINGS.EMPTY);
if (segment) parts.push(segment.trim());
}
- // If we accidentally merged two outcomes without clear token boundaries, ensure uniqueness
- return parts.filter(p => p.length > 0);
+
+ return parts.filter(Boolean);
}
reflowNumberedLists(text) {
- if (!text) return '';
- const lines = text.split('\n');
+ if (!text) return STRINGS.EMPTY;
+ const lines = text.split(STRINGS.NEWLINE);
const out = [];
let inNumbered = false;
let lastNumIndex = -1;
- for (let i = 0; i < lines.length; i++) {
+
+ const linesLength = lines.length;
+ for (let i = 0; i < linesLength; i++) {
const raw = lines[i];
const trimmed = raw.trim();
- const isNumbered = /^\d+\.\s/.test(trimmed);
- const isBullet = /^[-*]\s/.test(trimmed);
- const isHeading = /^#{1,6}\s/.test(trimmed);
- const isCodeFence = /^```/.test(trimmed);
+ const isNumbered = REGEX.NUMBERED_LIST.test(trimmed);
+ const isBullet = REGEX.BULLET_LIST.test(trimmed);
+ const isHeading = REGEX.HEADING.test(trimmed);
+ const isCodeFence = REGEX.CODE_FENCE.test(trimmed);
+
+ // Ensure a blank line BEFORE any list (numbered or bullet) begins
+ if ((isNumbered || isBullet) && out.length > 0) {
+ const prev = out[out.length - 1];
+ if (prev.trim() !== STRINGS.EMPTY) {
+ out.push(STRINGS.EMPTY);
+ }
+ }
if (isNumbered) {
out.push(trimmed);
@@ -552,19 +561,23 @@ class ErlangLiterateParser {
continue;
}
+ // Pass through bullet list lines unchanged (no reflow of bullets for now)
+ if (isBullet) {
+ out.push(raw);
+ continue;
+ }
+
if (inNumbered) {
- if (trimmed === '') {
- out.push('');
+ if (!trimmed) {
+ out.push(STRINGS.EMPTY);
inNumbered = false;
lastNumIndex = -1;
continue;
}
if (!isNumbered && !isBullet && !isHeading && !isCodeFence) {
- // Continuation of previous numbered item; append
- out[lastNumIndex] = out[lastNumIndex] + ' ' + trimmed;
+ out[lastNumIndex] = out[lastNumIndex] + STRINGS.SPACE + trimmed;
continue;
}
- // Different kind of line; end numbered block and fall through
inNumbered = false;
lastNumIndex = -1;
}
@@ -572,32 +585,32 @@ class ErlangLiterateParser {
out.push(raw);
}
- // Ensure a blank line between last numbered item and a label/heading line like 'Config options ...:'
+ // Ensure blank line separation
const separated = [];
- for (let i = 0; i < out.length; i++) {
+ const outLength = out.length;
+ for (let i = 0; i < outLength; i++) {
const cur = out[i];
- const next = i + 1 < out.length ? out[i + 1] : '';
+ const next = i + 1 < outLength ? out[i + 1] : STRINGS.EMPTY;
separated.push(cur);
- if (/^\d+\.\s/.test(cur.trim()) && next && !/^\s*$/.test(next) && /:\s*$/.test(next.trim())) {
- // Insert a blank line if not already present
- if (separated[separated.length - 1] !== '') {
- separated.push('');
+ // If a paragraph ends with ':' and is immediately followed by a numbered list,
+ // insert a blank line between them to satisfy Markdown list rendering rules.
+ if (REGEX.COLON_END.test(cur.trim()) && next && REGEX.NUMBERED_LIST.test(next.trim())) {
+ if (separated[separated.length - 1] !== STRINGS.EMPTY) {
+ separated.push(STRINGS.EMPTY);
}
}
}
- return separated.join('\n');
+ return separated.join(STRINGS.NEWLINE);
}
formatPreContent(content) {
- // First, let's look at the actual structure of the content more carefully
- // The issue is that definitions span multiple lines with varying indentation
-
- const lines = content.trim().split('\n');
+ const lines = content.trim().split(STRINGS.NEWLINE);
const formatted = [];
let i = 0;
- while (i < lines.length) {
+ const linesLength = lines.length;
+ while (i < linesLength) {
const line = lines[i].trim();
if (!line) {
@@ -605,74 +618,54 @@ class ErlangLiterateParser {
continue;
}
- // Look for definition pattern: starts with word(s), colon, then description
- // Pattern: "DevMod:ExportedFunc : Description" or "info/exports : Description"
- const defMatch = line.match(/^(\S+(?:\s*:\s*\S+)?)\s*:\s*(.*)$/);
+ const defMatch = line.match(REGEX.DEFINITION);
if (defMatch) {
const [, term, initialDesc] = defMatch;
let fullDescription = initialDesc.trim();
- // Collect continuation lines for this definition
let j = i + 1;
- while (j < lines.length) {
+ while (j < linesLength) {
const nextLine = lines[j];
- // Empty line - check if there's more content
if (!nextLine.trim()) {
j++;
continue;
}
- // If it looks like a new definition, stop
- if (nextLine.trim().match(/^\S+(?:\s*:\s*\S+)?\s*:\s*/)) {
+ if (nextLine.trim().match(REGEX.DEFINITION)) {
break;
}
- // This is a continuation line - add it to the description
if (nextLine.trim()) {
- fullDescription += ' ' + nextLine.trim();
+ fullDescription += STRINGS.SPACE + nextLine.trim();
}
j++;
}
- // Format the definition
- formatted.push('');
- formatted.push(`**${term.trim()}**`);
- formatted.push('');
- formatted.push(fullDescription);
-
- i = j; // Move to the next unprocessed line
+ formatted.push(STRINGS.EMPTY, `**${term.trim()}**`, STRINGS.EMPTY, fullDescription);
+ i = j;
} else {
- // Not a definition - handle as regular content
if (line.toLowerCase().includes('hyperbeam') && line.includes('options')) {
- formatted.push('');
- formatted.push(`### ${line}`);
- formatted.push('');
- } else if (line.match(/^`[^`]+`\s*:/)) {
- // Special case for option definitions like `update_hashpath`:
- const optMatch = line.match(/^(`[^`]+`)\s*:\s*(.*)$/);
+ formatted.push(STRINGS.EMPTY, `### ${line}`, STRINGS.EMPTY);
+ } else {
+ const optMatch = line.match(REGEX.OPTION_DEF);
if (optMatch) {
const [, optName, optDesc] = optMatch;
- formatted.push('');
- formatted.push(`**${optName}**`);
- formatted.push('');
- formatted.push(optDesc);
+ formatted.push(STRINGS.EMPTY, `**${optName}**`, STRINGS.EMPTY, optDesc);
} else {
formatted.push(line);
}
- } else {
- formatted.push(line);
}
i++;
}
}
- return formatted.join('\n');
+ return formatted.join(STRINGS.NEWLINE);
}
parseDocumentation(docText) {
- const lines = docText.split('\n');
+ const lines = docText.split(STRINGS.NEWLINE);
const result = {
description: [],
params: [],
@@ -686,51 +679,47 @@ class ErlangLiterateParser {
for (const line of lines) {
const trimmed = line.trim();
- // Check for @param
- const paramMatch = trimmed.match(/^@param\s+(\S+)\s*(.*)/);
+ const paramMatch = trimmed.match(REGEX.PARAM);
if (paramMatch) {
if (currentParam) {
result.params.push(currentParam);
}
currentParam = {
name: paramMatch[1],
- description: paramMatch[2] || ''
+ description: paramMatch[2] || STRINGS.EMPTY
};
currentSection = 'param';
continue;
}
- // Check for @returns
- if (trimmed.match(/^@returns?\s/)) {
+ if (REGEX.RETURNS.test(trimmed)) {
if (currentParam) {
result.params.push(currentParam);
currentParam = null;
}
- const returnsText = trimmed.replace(/^@returns?\s*/, '');
+ const returnsText = trimmed.replace(REGEX.RETURNS, STRINGS.EMPTY);
result.returns.push(returnsText);
lastReturnIndex = result.returns.length - 1;
currentSection = 'returns';
continue;
}
- // Add to current section
if (currentSection === 'description') {
if (trimmed) {
result.description.push(trimmed);
} else {
- // Preserve a single blank line to break paragraphs/lists
const last = result.description[result.description.length - 1];
- if (last !== '') {
- result.description.push('');
+ if (last !== STRINGS.EMPTY) {
+ result.description.push(STRINGS.EMPTY);
}
}
} else if (currentSection === 'param' && currentParam && trimmed) {
- currentParam.description += ' ' + trimmed;
+ currentParam.description += STRINGS.SPACE + trimmed;
} else if (currentSection === 'returns' && trimmed) {
if (lastReturnIndex >= 0) {
- // Append continuation text to the last returns entry
result.returns[lastReturnIndex] =
- (result.returns[lastReturnIndex] + ' ' + trimmed).replace(/\s+/g, ' ').trim();
+ (result.returns[lastReturnIndex] + STRINGS.SPACE + trimmed)
+ .replace(REGEX.WHITESPACE_NORMALIZE, STRINGS.SPACE).trim();
} else {
result.returns.push(trimmed);
lastReturnIndex = result.returns.length - 1;
@@ -738,7 +727,6 @@ class ErlangLiterateParser {
}
}
- // Save final param if exists
if (currentParam) {
result.params.push(currentParam);
}
@@ -748,90 +736,80 @@ class ErlangLiterateParser {
generateMarkdown(fileName) {
const githubUrl = `${this.options.githubBase}/${fileName}`;
- let md = [];
+ const md = [];
// Header
- md.push(`# ${this.moduleInfo.name || fileName.replace('.erl', '')}`);
- md.push('');
+ md.push(`# ${this.moduleInfo.name || fileName.replace('.erl', STRINGS.EMPTY)}`);
+ md.push(STRINGS.EMPTY);
md.push(`[View source on GitHub](${githubUrl})`);
- md.push('');
+ md.push(STRINGS.EMPTY);
// Module documentation
if (this.moduleInfo.doc) {
md.push(this.moduleInfo.doc);
- md.push('');
- md.push('---');
- md.push('');
+ md.push(STRINGS.EMPTY);
+ md.push(STRINGS.SEPARATOR);
+ md.push(STRINGS.EMPTY);
}
// Exports
- if (this.moduleInfo.exports && this.moduleInfo.exports.length > 0) {
- md.push('## Exported Functions');
- md.push('');
- for (const exp of this.moduleInfo.exports) {
- md.push(`- \`${exp}\``);
- }
- md.push('');
- md.push('---');
- md.push('');
+ if (this.moduleInfo.exports?.length > 0) {
+ md.push(STRINGS.EXPORTED_FUNCTIONS);
+ md.push(STRINGS.EMPTY);
+ this.moduleInfo.exports.forEach(exp =>
+ md.push(`- ${STRINGS.BACKTICK}${exp}${STRINGS.BACKTICK}`));
+ md.push(STRINGS.EMPTY);
+ md.push(STRINGS.SEPARATOR);
+ md.push(STRINGS.EMPTY);
}
- // Group functions by name to merge overloaded functions
const groupedFunctions = this.groupFunctionsByName(this.functions);
- // Functions
for (const group of groupedFunctions) {
md.push(`## ${group.name}`);
- md.push('');
+ md.push(STRINGS.EMPTY);
- // Combine documentation from all functions in the group
const combinedDoc = this.combineFunctionDocs(group.functions);
if (combinedDoc) {
md.push(combinedDoc);
- md.push('');
+ md.push(STRINGS.EMPTY);
}
- // Add all specs and bodies for the function group
for (const func of group.functions) {
- // Spec
if (func.spec) {
- md.push('```erlang');
+ md.push(`\`\`\`${STRINGS.ERLANG}`);
md.push(func.spec.trim());
md.push('```');
- md.push('');
+ md.push(STRINGS.EMPTY);
}
- // Function body with inline comments
- if (func.body && func.body.length > 0) {
- md.push('');
-
+ if (func.body?.length > 0) {
+ md.push(STRINGS.EMPTY);
for (const segment of func.body) {
if (segment.type === 'comment') {
md.push(segment.content);
- md.push('');
+ md.push(STRINGS.EMPTY);
} else if (segment.type === 'doc') {
- // Insert structured params/returns adjacent to the preceding code
md.push(segment.content);
- md.push('');
+ md.push(STRINGS.EMPTY);
} else if (segment.type === 'code') {
- md.push('```erlang');
+ md.push(`\`\`\`${STRINGS.ERLANG}`);
md.push(segment.content.trim());
md.push('```');
- md.push('');
+ md.push(STRINGS.EMPTY);
}
}
}
}
- md.push('');
+ md.push(STRINGS.EMPTY);
}
- // Footer
- md.push('---');
- md.push('');
+ md.push(STRINGS.SEPARATOR);
+ md.push(STRINGS.EMPTY);
md.push(`*Generated from [${fileName}](${githubUrl})*`);
- return md.join('\n');
+ return md.join(STRINGS.NEWLINE);
}
groupFunctionsByName(functions) {
@@ -840,14 +818,9 @@ class ErlangLiterateParser {
for (const func of functions) {
if (!currentGroup || currentGroup.name !== func.name) {
- // Start a new group
- currentGroup = {
- name: func.name,
- functions: [func]
- };
+ currentGroup = { name: func.name, functions: [func] };
groups.push(currentGroup);
} else {
- // Add to current group
currentGroup.functions.push(func);
}
}
@@ -856,42 +829,35 @@ class ErlangLiterateParser {
}
combineFunctionDocs(functions) {
- // Use the documentation from the first function that has it
- // In practice, usually only the first clause of an overloaded function has detailed docs
for (const func of functions) {
if (func.doc) {
const parsed = this.parseDocumentation(func.doc);
- let combinedDoc = [];
+ const combinedDoc = [];
- // Description
if (parsed.description.length > 0) {
- combinedDoc.push(this.cleanDocumentation(parsed.description.join('\n')));
- combinedDoc.push('');
+ combinedDoc.push(this.cleanDocumentation(parsed.description.join(STRINGS.NEWLINE)));
+ combinedDoc.push(STRINGS.EMPTY);
}
- // Parameters
if (parsed.params.length > 0) {
- combinedDoc.push('### Parameters');
- combinedDoc.push('');
- for (const param of parsed.params) {
+ combinedDoc.push(STRINGS.PARAMETERS_HEADER);
+ combinedDoc.push(STRINGS.EMPTY);
+ parsed.params.forEach(param => {
const desc = this.cleanDocumentation(param.description);
- combinedDoc.push(`- \`${param.name}\` - ${desc}`);
- }
- combinedDoc.push('');
+ combinedDoc.push(`- ${STRINGS.BACKTICK}${param.name}${STRINGS.BACKTICK} - ${desc}`);
+ });
+ combinedDoc.push(STRINGS.EMPTY);
}
- // Returns
if (parsed.returns.length > 0) {
- combinedDoc.push('### Returns');
- combinedDoc.push('');
+ combinedDoc.push(STRINGS.RETURNS_HEADER);
+ combinedDoc.push(STRINGS.EMPTY);
const expanded = parsed.returns.flatMap(r => this.splitReturnsIntoOutcomes(r));
- for (const ret of expanded) {
- combinedDoc.push(`- ${this.formatReturnsText(ret)}`);
- }
- combinedDoc.push('');
+ expanded.forEach(ret => combinedDoc.push(`- ${this.formatReturnsText(ret)}`));
+ combinedDoc.push(STRINGS.EMPTY);
}
- return combinedDoc.join('\n');
+ return combinedDoc.join(STRINGS.NEWLINE);
}
}
return null;
@@ -903,16 +869,13 @@ function main() {
const args = process.argv.slice(2);
const verbose = args.includes('-v') || args.includes('--verbose');
- // Get source directory
const srcDir = process.env.SRC_DIR || path.join(process.cwd(), 'src');
const outputDir = process.env.OUTPUT_DIR || path.join(process.cwd(), 'docs/literate-erlang');
- // Ensure output directory exists
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
- // Process all .erl files
const files = fs.readdirSync(srcDir).filter(f => f.endsWith('.erl'));
const parser = new ErlangLiterateParser({ verbose });
@@ -923,11 +886,9 @@ function main() {
try {
const inputPath = path.join(srcDir, file);
- const outputPath = path.join(outputDir, file + '.md');
-
+ const outputPath = path.join(outputDir, `${file}.md`);
const markdown = parser.parseFile(inputPath);
fs.writeFileSync(outputPath, markdown);
-
} catch (error) {
console.error(`Error processing ${file}:`, error.message);
}
From bb2dac936cf15cce40ac1b83ab0418dee5da39fd Mon Sep 17 00:00:00 2001
From: Dylan Shade <63427984+dpshade@users.noreply.github.com>
Date: Wed, 24 Sep 2025 17:19:15 -0400
Subject: [PATCH 16/17] docs: Remove deprecated documentation build scripts
- Update outdated scripts: `build-all.sh`, `build-and-serve.sh`, `build-literate-erlang.sh`, and `build-literate-erlang-js.sh`.
- These scripts have been superseded by more efficient implementations, streamlining the documentation generation process.
- Ensure the codebase is cleaner and more maintainable by removing legacy files.
---
docs/book/book.toml | 5 +-
docs/{build-all.sh => build-docs.sh} | 0
docs/build-literate-erlang.sh | 666 --------------
docs/deploy-dry-run.sh | 94 ++
docs/erlang-literate-parser.js | 847 ++++++++++++++++--
...erlang-js.sh => generate-literate-docs.sh} | 90 +-
docs/{build-and-serve.sh => serve-book.sh} | 2 +-
7 files changed, 960 insertions(+), 744 deletions(-)
rename docs/{build-all.sh => build-docs.sh} (100%)
delete mode 100755 docs/build-literate-erlang.sh
create mode 100755 docs/deploy-dry-run.sh
rename docs/{build-literate-erlang-js.sh => generate-literate-docs.sh} (51%)
rename docs/{build-and-serve.sh => serve-book.sh} (96%)
diff --git a/docs/book/book.toml b/docs/book/book.toml
index ab213ea81..51f3cc0fa 100644
--- a/docs/book/book.toml
+++ b/docs/book/book.toml
@@ -8,9 +8,7 @@ description = "Literate programming documentation for the HyperBEAM decentralize
[build]
build-dir = "dist"
-[rust]
-edition = "2021"
-
+# Configure for non-Rust documentation - this prevents mdBook from trying to compile code blocks as Rust
[output.html]
default-theme = "rust"
additional-css = ["custom.css", "theme/highlight.css"]
@@ -41,6 +39,7 @@ copy-js = true
line-numbers = false
runnable = false
+
[output.html.search]
enable = true
limit-results = 30
diff --git a/docs/build-all.sh b/docs/build-docs.sh
similarity index 100%
rename from docs/build-all.sh
rename to docs/build-docs.sh
diff --git a/docs/build-literate-erlang.sh b/docs/build-literate-erlang.sh
deleted file mode 100755
index f1cf0262d..000000000
--- a/docs/build-literate-erlang.sh
+++ /dev/null
@@ -1,666 +0,0 @@
-#!/bin/bash
-
-# Script to generate literate Erlang documentation from HyperBEAM source files
-#
-# ⚠️ DEPRECATED: This bash parser has been superseded by the JavaScript version
-# For superior results with comprehensive comment parsing, use:
-# ./docs/build-literate-erlang-js.sh
-#
-# This creates .erl.md files that combine source code with documentation
-# in a format optimized for GitHub rendering with cleaner appearance
-#
-# Usage: ./docs/build-literate-erlang.sh [-v | --verbose]
-# -v, --verbose: Show detailed processing output
-
-# --- Color Definitions ---
-GREEN='\033[0;32m'
-RED='\033[0;31m'
-YELLOW='\033[0;33m'
-BLUE='\033[0;34m'
-BOLD='\033[1m'
-NC='\033[0m' # No Color
-
-# HyperBEAM Logo Colors
-NEON_GREEN='\033[38;5;46m'
-CYAN='\033[38;5;51m'
-BRIGHT_YELLOW='\033[38;5;226m'
-MAGENTA='\033[38;5;201m'
-BRIGHT_RED='\033[38;5;196m'
-BLACK='\033[38;5;0m'
-GRAY='\033[38;5;245m'
-
-# --- Helper Functions ---
-log_success() {
- echo -e "${GREEN}✓ $1${NC}"
-}
-
-log_info() {
- echo -e "${BLUE}→ $1${NC}"
-}
-
-log_step() {
- echo -e "\n${YELLOW}${BOLD}$1${NC}"
-}
-
-log_error() {
- echo -e "${RED}✗ $1${NC}"
-}
-
-log_verbose() {
- if [ "$VERBOSE" = true ]; then
- echo -e "${GRAY} $1${NC}"
- fi
-}
-
-# --- Variable Defaults ---
-VERBOSE=false
-
-# --- Parse Command Line Arguments ---
-while [[ $# -gt 0 ]]; do
- key="$1"
- case $key in
- -v|--verbose)
- VERBOSE=true
- log_info "Verbose mode enabled"
- shift
- ;;
- *)
- log_error "Unknown option: $1"
- echo "Usage: $0 [-v | --verbose]"
- exit 1
- ;;
- esac
-done
-
-# --- Display HyperBEAM ASCII Logo ---
-display_logo() {
- echo -e "
-${NEON_GREEN} ++ ${BLACK}${BOLD} ${NC}
-${NEON_GREEN} +++ ${BLACK}${BOLD} _ ${NC}
-${NEON_GREEN} ++++* ${BLACK}${BOLD}| |__ _ _ _ __ ___ _ __ ${NC}
-${NEON_GREEN} :+++*${BRIGHT_YELLOW}## ${BLACK}${BOLD} | '_ \\| | | | '_ \\ / _ \\ '__| ${NC}
-${NEON_GREEN} ++**${BRIGHT_YELLOW}#### ${BLACK}${BOLD} | | | | |_| | |_) | __/ | ${NC}
-${NEON_GREEN} +++${BRIGHT_YELLOW}####${NEON_GREEN}*** ${BLACK}${BOLD} |_| |_|\\__, | .__/ \\___|_| ${NC}
-${NEON_GREEN} +*${BRIGHT_YELLOW}##${NEON_GREEN}****${MAGENTA}+-- ${BLACK}${BOLD} |___/|_| ${NC}
-${MAGENTA} -**${BRIGHT_YELLOW}##${NEON_GREEN}**${MAGENTA}+------ ${BLACK}${BOLD} BEAM.${NC}
-${MAGENTA} -##${NEON_GREEN}*+${BRIGHT_RED}---:::::::
-${GRAY} =${GRAY}%%${NEON_GREEN}*+${BRIGHT_RED}=-:::::::::${GRAY} LITERATE ERLANG DOCUMENTATION${NC}
-"
-}
-
-# --- Script Start ---
-display_logo
-log_step "LITERATE ERLANG DOCUMENTATION GENERATION"
-
-# Display deprecation notice
-echo -e "${YELLOW}${BOLD}⚠️ DEPRECATION NOTICE${NC}"
-echo -e "${YELLOW}This bash parser has been superseded by a superior JavaScript implementation.${NC}"
-echo -e "${YELLOW}For comprehensive comment parsing and true literate programming, use:${NC}"
-echo -e "${GREEN}${BOLD} ./docs/build-literate-erlang-js.sh${NC}"
-echo -e "${GRAY}Continuing with legacy bash parser...${NC}"
-echo ""
-
-# Ensure we're in the root directory
-ROOT_DIR="$(dirname "$(realpath "$0")")/.."
-cd "$ROOT_DIR" || { log_error "Failed to change to root directory"; exit 1; }
-
-# GitHub repository base URL
-GITHUB_BASE_URL="https://github.com/permaweb/HyperBEAM/blob/edge/src"
-
-# Output directory for literate Erlang files
-OUTPUT_DIR="$ROOT_DIR/docs/literate-erlang"
-mkdir -p "$OUTPUT_DIR"
-
-# --- Function to extract module documentation ---
-extract_module_doc() {
- local file="$1"
- local in_doc=false
- local doc_content=""
-
- while IFS= read -r line; do
- if [[ "$line" =~ ^%%%[[:space:]]?(.*)$ ]]; then
- in_doc=true
- doc_content+="${BASH_REMATCH[1]}"$'\n'
- elif [[ "$line" =~ ^%%[[:space:]]?(@doc[[:space:]])?(.*)$ ]] && [ "$in_doc" = true ]; then
- # Extract content after @doc if present
- doc_content+="${BASH_REMATCH[2]}"$'\n'
- elif [[ ! "$line" =~ ^%% ]] && [ "$in_doc" = true ]; then
- break
- fi
- done < "$file"
-
- # Clean up @doc prefixes and convert edocs syntax to markdown, preserving paragraph breaks
- echo "$doc_content" | \
- sed 's/^@doc$//' | \
- sed 's/^@doc //' | \
- sed 's/^@end$//' | \
- sed 's/^@author /**Author:** /' | \
- sed 's/^@copyright /**Copyright:** /' | \
- sed 's/^---*$//' | \
- sed "s/\`\([^']*\)'/\`\1\`/g" | \
- awk '
- BEGIN { prev_empty = 0 }
- /^[[:space:]]*$/ {
- if (!prev_empty) {
- print ""
- prev_empty = 1
- }
- next
- }
- {
- print $0
- prev_empty = 0
- }'
-}
-
-# --- Function to extract function documentation ---
-extract_function_doc() {
- local content="$1"
-
- # Clean the content and separate into sections
- local cleaned_content=$(echo "$content" | \
- sed 's/^%% *//' | \
- sed 's/^% *//' | \
- sed 's/^@doc$//' | \
- sed 's/^@doc //' | \
- sed 's/^@end$//' | \
- sed "s/\`\([^']*\)'/\`\1\`/g")
-
- # Initialize variables for different sections
- local description=""
- local params=""
- local returns=""
- local in_params=false
- local in_returns=false
- local current_param=""
-
- while IFS= read -r line; do
- if [[ "$line" =~ ^@param[[:space:]]+([^[:space:]]+)[[:space:]]+(.*)$ ]]; then
- # Save any current param before starting new one
- if [ -n "$current_param" ]; then
- params+="- ${current_param}"$'\n'
- fi
- # Start new param with code-formatted name
- current_param="\`${BASH_REMATCH[1]}\` - ${BASH_REMATCH[2]}"
- in_params=true
- in_returns=false
- elif [[ "$line" =~ ^@returns?[[:space:]]+(.*)$ ]]; then
- # Save any current param before starting returns
- if [ -n "$current_param" ]; then
- params+="- ${current_param}"$'\n'
- current_param=""
- fi
- returns="${BASH_REMATCH[1]}"
- in_params=false
- in_returns=true
- elif [[ "$line" =~ ^@author[[:space:]]+(.*)$ ]]; then
- # Skip author lines for now
- continue
- elif [[ "$line" =~ ^@copyright[[:space:]]+(.*)$ ]]; then
- # Skip copyright lines for now
- continue
- elif [[ "$line" =~ ^[[:space:]]*$ ]]; then
- # Empty line - add to current section
- if [ "$in_params" = true ] && [ -n "$current_param" ]; then
- current_param+=" "
- elif [ "$in_returns" = true ]; then
- returns+=" "
- elif [ "$in_params" = false ] && [ "$in_returns" = false ]; then
- description+="$line"$'\n'
- fi
- else
- # Regular content line
- if [ "$in_params" = true ]; then
- # Continue current param description - clean up whitespace
- local cleaned_line=$(echo "$line" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//')
- current_param+=" $cleaned_line"
- elif [ "$in_returns" = true ]; then
- # Continue returns description - clean up whitespace
- local cleaned_line=$(echo "$line" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//')
- returns+=" $cleaned_line"
- else
- # Part of main description
- description+="$line"$'\n'
- fi
- fi
- done <<< "$cleaned_content"
-
- # Save any remaining param
- if [ -n "$current_param" ]; then
- params+="- ${current_param}"$'\n'
- fi
-
- # Build formatted output
- local output=""
-
- # Add description (clean up extra newlines)
- if [ -n "$description" ]; then
- output+=$(echo "$description" | awk '
- BEGIN { prev_empty = 0 }
- /^[[:space:]]*$/ {
- if (!prev_empty) {
- print ""
- prev_empty = 1
- }
- next
- }
- {
- print $0
- prev_empty = 0
- }')
- output+=$'\n'
- fi
-
- # Add parameters section
- if [ -n "$params" ]; then
- output+=$'\n'"#### Parameters"$'\n'$'\n'
- output+="$params"$'\n'
- fi
-
- # Add returns section
- if [ -n "$returns" ]; then
- output+=$'\n'"#### Returns"$'\n'$'\n'
- # Clean up returns text and parse return type vs description
- local cleaned_returns=$(echo "$returns" | sed 's/[[:space:]]\+/ /g' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//')
-
- # Try to extract and format return types - always use bullet format
- # Check for multiple return patterns first (most comprehensive)
- if [[ "$cleaned_returns" =~ , ]]; then
- # Try to format multiple return patterns - handle nested braces properly
- # Use a more robust approach for nested structures
- local formatted_returns=$(echo "$cleaned_returns" | \
- perl -pe 's/(\{(?:[^{}]++|(?1))*\})/`$1`/g' 2>/dev/null || \
- echo "$cleaned_returns" | sed -E 's/(\{[^{}]*(\{[^}]*\}[^{}]*)*\})/`\1`/g')
- formatted_returns=$(echo "$formatted_returns" | sed -E 's/([[:space:]]|^)(not_found|error|ok|true|false)([[:space:]]|$)/\1`\2`\3/g')
- output+="- $formatted_returns"$'\n'
- elif [[ "$cleaned_returns" =~ ^\{ ]]; then
- # Complex return type - use better pattern to handle nested braces
- # Extract the complete tuple including nested structures
- local temp_string="$cleaned_returns"
- local brace_count=0
- local char_pos=0
- local return_type=""
-
- # Parse character by character to find matching braces
- while [ $char_pos -lt ${#temp_string} ]; do
- local char="${temp_string:$char_pos:1}"
- return_type+="$char"
-
- if [ "$char" = "{" ]; then
- ((brace_count++))
- elif [ "$char" = "}" ]; then
- ((brace_count--))
- if [ $brace_count -eq 0 ]; then
- break
- fi
- fi
- ((char_pos++))
- done
-
- # Extract description after the return type
- local return_desc="${temp_string:$((char_pos + 1))}"
- return_desc=$(echo "$return_desc" | sed 's/^[[:space:]]*//')
-
- if [ -n "$return_desc" ]; then
- output+="- \`$return_type\` $return_desc"$'\n'
- else
- output+="- \`$return_type\`"$'\n'
- fi
- elif [[ "$cleaned_returns" =~ ^(true|false)[[:space:]]+(.*) ]]; then
- # Boolean return types
- local return_type="${BASH_REMATCH[1]}"
- local return_desc="${BASH_REMATCH[2]}"
- output+="- \`$return_type\` $return_desc"$'\n'
- elif [[ "$cleaned_returns" =~ ^(ok|error|not_found)[[:space:]]+(.*) ]]; then
- # Simple atom return types
- local return_type="${BASH_REMATCH[1]}"
- local return_desc="${BASH_REMATCH[2]}"
- output+="- \`$return_type\` $return_desc"$'\n'
- else
- output+="- $cleaned_returns"$'\n'
- fi
- fi
-
- echo "$output"
-}
-
-# --- Function to process a single Erlang file ---
-process_erlang_file() {
- local src_file="$1"
- local module_name=$(basename "$src_file" .erl)
- local output_file="$OUTPUT_DIR/${module_name}.erl.md"
-
- log_verbose "Processing $module_name"
-
- # Start the literate Erlang document with cleaner format
- cat > "$output_file" <> "$output_file"
- echo "" >> "$output_file"
- echo "---" >> "$output_file"
- echo "" >> "$output_file"
- fi
-
- # Add module exports in a clean format
- local exports=$(grep -E "^-export\(" "$src_file" | sed 's/-export(\[//' | sed 's/\]).*//' | tr ',' '\n' | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//' | sort -u)
-
- if [ -n "$exports" ]; then
- echo "## Exported Functions" >> "$output_file"
- echo "" >> "$output_file"
-
- # Create a proper bulleted list for exports
- while IFS= read -r export; do
- if [[ "$export" =~ ^[a-z] ]]; then
- echo "- \`$export\`" >> "$output_file"
- fi
- done <<< "$exports"
-
- echo "" >> "$output_file"
- echo "---" >> "$output_file"
- echo "" >> "$output_file"
- fi
-
- # Process functions
- local in_function=false
- local in_spec=false
- local in_doc_comment=false
- local current_function=""
- local function_content=""
- local spec_content=""
- local doc_content=""
- local previous_doc_content=""
- local functions_written=0
-
- while IFS= read -r line; do
- # Check for doc comments (before functions)
- if [[ "$line" =~ ^%%[[:space:]]?@doc[[:space:]](.*)$ ]] ||
- [[ "$line" =~ ^%%[[:space:]]?@doc$ ]]; then
- in_doc_comment=true
- if [[ "$line" =~ @doc[[:space:]](.*)$ ]]; then
- doc_content+="${BASH_REMATCH[1]}"$'\n'
- fi
- continue
- fi
-
- # Continue collecting doc comment lines
- if [ "$in_doc_comment" = true ] && [[ "$line" =~ ^%% ]]; then
- # Remove %% prefix and collect
- local cleaned_line=$(echo "$line" | sed 's/^%%[[:space:]]*//')
- doc_content+="$cleaned_line"$'\n'
- continue
- fi
-
- # Check for -spec
- if [[ "$line" =~ ^-spec[[:space:]] ]]; then
- in_spec=true
- spec_content="$line"$'\n'
- in_doc_comment=false
- continue
- fi
-
- # Continue collecting spec if in multi-line spec
- if [ "$in_spec" = true ]; then
- spec_content+="$line"$'\n'
- if [[ "$line" =~ \.[[:space:]]*$ ]]; then
- in_spec=false
- fi
- continue
- fi
-
- # Check for function definition
- if [[ "$line" =~ ^([a-z][a-z0-9_]*)[[:space:]]*\( ]]; then
- # If we were already in a function, write it out
- if [ -n "$current_function" ] && [ -n "$function_content" ]; then
- write_clean_function "$output_file" "$current_function" "$spec_content" "$previous_doc_content" "$function_content" "$functions_written"
- ((functions_written++))
- fi
-
- # Start new function - preserve current doc_content for this function
- current_function="${BASH_REMATCH[1]}"
- function_content="$line"$'\n'
- in_function=true
- in_doc_comment=false
- previous_doc_content="$doc_content"
- spec_content=""
- doc_content=""
- continue
- fi
-
- # Continue collecting function content
- if [ "$in_function" = true ]; then
- function_content+="$line"$'\n'
- # Check for function end (period at end of line, not in string or comment)
- if [[ "$line" =~ ^[[:space:]]*end\.[[:space:]]*$ ]] ||
- ([[ "$line" =~ \.[[:space:]]*$ ]] && ! [[ "$line" =~ \" ]] && ! [[ "$line" =~ ^[[:space:]]*% ]]); then
- in_function=false
- write_clean_function "$output_file" "$current_function" "$spec_content" "$previous_doc_content" "$function_content" "$functions_written"
- ((functions_written++))
- current_function=""
- function_content=""
- spec_content=""
- previous_doc_content=""
- fi
- else
- # Only reset doc content if we hit a non-comment, non-spec, non-function line
- if ! [[ "$line" =~ ^% ]] && ! [[ "$line" =~ ^-spec ]] && [ "$in_doc_comment" = false ]; then
- doc_content=""
- fi
- in_doc_comment=false
- fi
- done < "$src_file"
-
- # Write any remaining function
- if [ -n "$current_function" ] && [ -n "$function_content" ]; then
- write_clean_function "$output_file" "$current_function" "$spec_content" "$previous_doc_content" "$function_content" "$functions_written"
- fi
-
- # Add footer
- echo "" >> "$output_file"
- echo "---" >> "$output_file"
- echo "" >> "$output_file"
- echo "*Generated from [$module_name.erl]($GITHUB_BASE_URL/${module_name}.erl)*" >> "$output_file"
-}
-
-# --- Function to write a function section with cleaner format ---
-write_clean_function() {
- local output_file="$1"
- local func_name="$2"
- local spec="$3"
- local doc="$4"
- local code="$5"
- local func_num="$6"
-
- # Add section separator for better readability (except for first function)
- if [ "$func_num" -gt 0 ]; then
- echo "" >> "$output_file"
- fi
-
- echo "### $func_name" >> "$output_file"
- echo "" >> "$output_file"
-
- # Add documentation if present
- if [ -n "$doc" ]; then
- local cleaned_doc=$(extract_function_doc "$doc")
- if [ -n "$cleaned_doc" ]; then
- echo "$cleaned_doc" >> "$output_file"
- echo "" >> "$output_file"
- fi
- fi
-
- # Add spec if present (in a more compact format)
- if [ -n "$spec" ] && [ "$spec" != $'\n' ]; then
- echo '```erlang' >> "$output_file"
- echo -n "$spec" | sed '/^[[:space:]]*$/d' >> "$output_file"
- echo '```' >> "$output_file"
- echo "" >> "$output_file"
- fi
-
- # Add Function subheader before the implementation
- echo "#### Function" >> "$output_file"
- echo "" >> "$output_file"
-
- # Add implementation with inline comment processing
- write_code_with_inline_comments "$output_file" "$code"
-}
-
-# --- Function to write code blocks with inline comments breaking them ---
-write_code_with_inline_comments() {
- local output_file="$1"
- local code="$2"
-
- local in_code_block=false
- local in_comment_block=false
- local comment_lines=()
-
- while IFS= read -r line; do
- # Check if this is an inline comment
- # Match lines that have whitespace followed by a single % (not %% or %%%)
- if [[ "$line" =~ ^[[:space:]]*%([[:space:]]|$) ]] && ! [[ "$line" =~ ^[[:space:]]*%% ]]; then
- # If we were in a code block, close it
- if [ "$in_code_block" = true ]; then
- echo '```' >> "$output_file"
- echo "" >> "$output_file"
- in_code_block=false
- fi
-
- # Extract comment text - remove leading whitespace and %
- comment_text=$(echo "$line" | sed 's/^[[:space:]]*%//')
-
- # Remove leading space after % if present
- comment_text=$(echo "$comment_text" | sed 's/^[[:space:]]//')
-
- # Convert backtick-quote pairs to proper markdown backticks
- comment_text=$(echo "$comment_text" | sed "s/\`\([^']*\)'/\`\1\`/g")
-
- # Add to comment lines array
- comment_lines+=("$comment_text")
- in_comment_block=true
- else
- # Regular code line (including %% and %%% doc comments)
- # If we were collecting comments, write them out first
- if [ "$in_comment_block" = true ]; then
- # Write each comment line separately to preserve structure
- for comment_line in "${comment_lines[@]}"; do
- echo "$comment_line" >> "$output_file"
- done
- echo "" >> "$output_file"
- echo '```erlang' >> "$output_file"
- in_comment_block=false
- comment_lines=()
- in_code_block=true
- elif [ "$in_code_block" = false ]; then
- echo '```erlang' >> "$output_file"
- in_code_block=true
- fi
- echo "$line" >> "$output_file"
- fi
- done <<< "$code"
-
- # Handle any remaining comment lines
- if [ "$in_comment_block" = true ]; then
- for comment_line in "${comment_lines[@]}"; do
- echo "$comment_line" >> "$output_file"
- done
- echo "" >> "$output_file"
- fi
-
- # Close any remaining code block
- if [ "$in_code_block" = true ]; then
- echo '```' >> "$output_file"
- echo "" >> "$output_file"
- fi
-}
-
-# --- Main processing loop ---
-log_step "Processing Erlang source files"
-
-# Count total files
-total_files=$(find "$ROOT_DIR/src" -name "*.erl" -type f | wc -l)
-processed=0
-
-# Process each .erl file in src directory
-find "$ROOT_DIR/src" -name "*.erl" -type f | sort | while read -r erl_file; do
- ((processed++))
- module_name=$(basename "$erl_file" .erl)
- log_info "[$processed/$total_files] Processing $module_name.erl"
- process_erlang_file "$erl_file"
-done
-
-log_success "Processed $total_files Erlang files"
-
-# --- Generate index file ---
-log_step "Generating index file"
-
-cat > "$OUTPUT_DIR/README.md" <> "$OUTPUT_DIR/README.md"
-echo "|--------|-------------|" >> "$OUTPUT_DIR/README.md"
-
-find "$OUTPUT_DIR" -name "*.erl.md" -type f | sort | while read -r md_file; do
- module_name=$(basename "$md_file" .erl.md)
- # Try to extract first line of module doc as description
- first_line=$(grep -m 1 -A 1 "^# $module_name" "$md_file" | tail -1 | head -c 100)
- if [ "$first_line" = "[View source on GitHub]"* ] || [ -z "$first_line" ]; then
- first_line="Erlang module"
- fi
- echo "| [$module_name](./${module_name}.erl.md) | $first_line... |" >> "$OUTPUT_DIR/README.md"
-done
-
-cat >> "$OUTPUT_DIR/README.md" < e.trim())
- .filter(Boolean);
+ // Exports - handle multi-line exports
+ if (REGEX.EXPORT_PREFIX.test(trimmed)) {
+ const exportLines = this.collectMultiLineConstruct(i, '[', ']');
+ const fullExport = exportLines.join(' ');
+ const exportMatch = fullExport.match(REGEX.EXPORT);
+ if (exportMatch) {
+ const exports = exportMatch[1]
+ .split(',')
+ .map(e => e.trim())
+ .filter(Boolean);
+ this.moduleInfo.exports.push(...exports);
+ }
+ i += exportLines.length - 1;
+ continue;
+ }
+
+ // Includes
+ const includeMatch = trimmed.match(REGEX.INCLUDE);
+ if (includeMatch) {
+ this.moduleInfo.includes.push({
+ file: includeMatch[1],
+ line: trimmed
+ });
+ continue;
+ }
+
+ // Defines
+ const defineMatch = trimmed.match(REGEX.DEFINE);
+ if (defineMatch) {
+ this.moduleInfo.defines.push({
+ name: defineMatch[1],
+ value: defineMatch[2] || '',
+ line: trimmed
+ });
+ continue;
+ }
+
+ // Behaviours
+ const behaviourMatch = trimmed.match(REGEX.BEHAVIOUR);
+ if (behaviourMatch) {
+ this.moduleInfo.behaviours.push(behaviourMatch[1]);
+ continue;
+ }
+
+ // Records
+ if (REGEX.RECORD.test(trimmed)) {
+ const recordLines = this.collectMultiLineConstruct(i, '{', '}');
+ const recordMatch = trimmed.match(REGEX.RECORD);
+ if (recordMatch) {
+ this.moduleInfo.records.push({
+ name: recordMatch[1],
+ definition: recordLines.join('\n')
+ });
+ }
+ i += recordLines.length - 1;
+ continue;
+ }
+
+ // Types
+ const typeMatch = trimmed.match(REGEX.TYPE);
+ if (typeMatch) {
+ const typeLines = this.collectMultiLineConstruct(i, '(', ')');
+ this.moduleInfo.types.push({
+ name: typeMatch[1],
+ definition: typeLines.join('\n')
+ });
+ i += typeLines.length - 1;
+ continue;
+ }
+
+ // Specs (collect but don't process here)
+ if (REGEX.SPEC.test(trimmed)) {
+ const specLines = this.collectMultiLineConstruct(i, '(', ')');
+ const specMatch = trimmed.match(REGEX.SPEC);
+ if (specMatch) {
+ this.moduleInfo.specs.push({
+ function: specMatch[1],
+ definition: specLines.join('\n')
+ });
+ }
+ i += specLines.length - 1;
+ continue;
+ }
+
+ // Other attributes
+ const attrMatch = trimmed.match(REGEX.ATTRIBUTE);
+ if (attrMatch) {
+ this.moduleInfo.attributes.push({
+ name: attrMatch[1],
+ line: trimmed
+ });
+ continue;
}
- continue;
}
- // Module documentation - check for percent first
- if (firstChar === CHAR_CODES.PERCENT && trimmed.startsWith(STRINGS.TRIPLE_PERCENT)) {
+ // Module documentation - only collect at the beginning of the file
+ if (firstChar === CHAR_CODES.PERCENT && trimmed.startsWith(STRINGS.TRIPLE_PERCENT) && !moduleDocCollected) {
inModuleDoc = true;
let docLine = trimmed.substring(3).trim();
docLine = docLine.replace(REGEX.REMOVE_DOC, STRINGS.EMPTY);
+
+ // Check for termination pattern (%%% ''')
+ if (docLine === "'''") {
+ inModuleDoc = false; // End module documentation processing
+ moduleDocCollected = true; // Mark as collected
+ continue; // Continue processing rest of file
+ }
+
moduleDoc.push(docLine);
} else if (inModuleDoc && (firstChar === CHAR_CODES.PERCENT || firstChar === CHAR_CODES.DASH)) {
- break;
+ inModuleDoc = false;
+ moduleDocCollected = true; // Mark as collected
+ // Don't break - continue processing this line for module declarations
+ i--; // Re-process this line outside module doc context
}
}
- this.moduleInfo.doc = this.cleanDocumentation(moduleDoc.join(STRINGS.NEWLINE));
+ this.moduleInfo.doc = this.cleanDocumentation(this.fixModuleDocCodeBlocks(moduleDoc.join(STRINGS.NEWLINE)));
+ }
+
+ collectMultiLineConstruct(startIdx, openChar, closeChar) {
+ const lines = [];
+ let depth = 0;
+ let found = false;
+
+ for (let i = startIdx; i < this.lines.length; i++) {
+ const line = this.lines[i];
+ lines.push(line);
+
+ for (let j = 0; j < line.length; j++) {
+ const char = line[j];
+ if (char === openChar) {
+ depth++;
+ found = true;
+ } else if (char === closeChar && found) {
+ depth--;
+ if (depth === 0) {
+ return lines;
+ }
+ }
+ }
+
+ // Safety check for runaway constructs
+ if (i - startIdx > 100) break;
+ }
+
+ return lines;
}
processFunctions() {
const linesLength = this.lines.length;
+ const processedFunctions = new Set();
+ const commentedCodeBlocks = [];
+ let currentCommentedBlock = [];
+ let inCommentedBlock = false;
for (let i = 0; i < linesLength; i++) {
const line = this.lines[i];
const trimmed = line.trim();
- if (!trimmed) continue;
+ if (!trimmed) {
+ if (inCommentedBlock) {
+ currentCommentedBlock.push(line);
+ }
+ continue;
+ }
+
+ // Check for comment-style section headers
+ if (trimmed.startsWith('%%%') && trimmed.match(/^%%%-{10,}$/)) {
+ // This is a dash line, check if next line is a header and line after that is also dashes
+ if (i + 1 < linesLength && i + 2 < linesLength) {
+ const nextLine = this.lines[i + 1].trim();
+ const afterLine = this.lines[i + 2].trim();
+
+ if (nextLine.startsWith('%%%') && !nextLine.match(/^%%%-{10,}$/) &&
+ afterLine.match(/^%%%-{10,}$/)) {
+ // Extract header text
+ const headerText = nextLine.replace(/^%%%\s*/, '').trim();
+ if (headerText) {
+ this.sections.push({
+ type: 'section_header',
+ title: headerText,
+ lineNumber: i + 2 // Store line number for sorting later
+ });
+ }
+ // Skip the next two lines
+ i += 2;
+ continue;
+ }
+ }
+ }
+
+ // Check for commented-out code blocks (lines starting with % but containing code patterns)
+ if (trimmed.startsWith('%') && !trimmed.startsWith('%%')) {
+ const uncommented = trimmed.substring(1).trim();
+ if (this.looksLikeCode(uncommented)) {
+ if (!inCommentedBlock) {
+ inCommentedBlock = true;
+ currentCommentedBlock = [];
+ }
+ currentCommentedBlock.push(line);
+ continue;
+ } else if (inCommentedBlock) {
+ // End of commented code block
+ if (currentCommentedBlock.length > 0) {
+ commentedCodeBlocks.push({
+ type: 'commented_code',
+ lines: [...currentCommentedBlock],
+ startLine: i - currentCommentedBlock.length + 1
+ });
+ }
+ inCommentedBlock = false;
+ currentCommentedBlock = [];
+ }
+ } else if (inCommentedBlock) {
+ // End of commented code block
+ if (currentCommentedBlock.length > 0) {
+ commentedCodeBlocks.push({
+ type: 'commented_code',
+ lines: [...currentCommentedBlock],
+ startLine: i - currentCommentedBlock.length + 1
+ });
+ }
+ inCommentedBlock = false;
+ currentCommentedBlock = [];
+ }
// Check for start of function documentation block
if (REGEX.DOC_BLOCK_START.test(trimmed)) {
@@ -203,16 +419,31 @@ class ErlangLiterateParser {
continue;
}
+ // Check for conditional compilation directives
+ if (trimmed.startsWith('-ifdef(') || trimmed.startsWith('-ifndef(') ||
+ trimmed.startsWith('-else') || trimmed.startsWith('-endif')) {
+ this.addDirectiveToOutput(line, i);
+ continue;
+ }
+
// Check for function start
const funcMatch = trimmed.match(REGEX.FUNCTION);
if (funcMatch && !this.currentState.inFunction) {
+ const functionName = this.currentState.specFunctionName || funcMatch[1];
+ processedFunctions.add(functionName);
+
if (this.currentState.pendingDoc) {
this.currentState.functionDoc = this.currentState.pendingDoc;
this.currentState.pendingDoc = STRINGS.EMPTY;
+ this.startFunction(functionName, i);
+ } else {
+ // This is an undocumented function
+ this.startUndocumentedFunction(functionName, i);
+ // Skip ahead to avoid reprocessing
+ while (i < linesLength && !this.isFunctionEnd(this.lines[i])) {
+ i++;
+ }
}
-
- const functionName = this.currentState.specFunctionName || funcMatch[1];
- this.startFunction(functionName);
this.currentState.specFunctionName = STRINGS.EMPTY;
}
@@ -225,9 +456,21 @@ class ErlangLiterateParser {
}
}
+ // Handle remaining commented code block
+ if (inCommentedBlock && currentCommentedBlock.length > 0) {
+ commentedCodeBlocks.push({
+ type: 'commented_code',
+ lines: currentCommentedBlock,
+ startLine: linesLength - currentCommentedBlock.length + 1
+ });
+ }
+
if (this.currentState.inFunction) {
this.endFunction();
}
+
+ // Store commented code blocks for later inclusion
+ this.commentedCodeBlocks = commentedCodeBlocks;
}
collectFunctionDoc(startIdx) {
@@ -316,9 +559,10 @@ class ErlangLiterateParser {
this.currentState.functionSpec = specLines.join(STRINGS.NEWLINE);
}
- startFunction(name) {
+ startFunction(name, lineNumber = 0) {
this.currentState.inFunction = true;
this.currentState.functionName = name;
+ this.currentState.functionLineNumber = lineNumber;
this.currentState.functionLines.length = 0;
this.currentState.braceDepth = 0;
this.currentState.parenDepth = 0;
@@ -365,12 +609,15 @@ class ErlangLiterateParser {
name: this.currentState.functionName,
spec: this.currentState.functionSpec,
doc: this.currentState.functionDoc,
- body: processedBody
+ body: processedBody,
+ hasImplementation: true,
+ lineNumber: this.currentState.functionLineNumber
});
// Reset state efficiently
this.currentState.inFunction = false;
this.currentState.functionName = STRINGS.EMPTY;
+ this.currentState.functionLineNumber = 0;
this.currentState.functionSpec = STRINGS.EMPTY;
this.currentState.functionDoc = STRINGS.EMPTY;
this.currentState.functionLines.length = 0;
@@ -378,6 +625,104 @@ class ErlangLiterateParser {
this.currentState.inlineDocTags.length = 0;
}
+ startUndocumentedFunction(name, startLine) {
+ const functionLines = [];
+ let braceDepth = 0;
+ let parenDepth = 0;
+ let i = startLine;
+
+ // Collect the entire function
+ while (i < this.lines.length) {
+ const line = this.lines[i];
+ functionLines.push(line);
+
+ // Track depth
+ for (let j = 0; j < line.length; j++) {
+ const charCode = line.charCodeAt(j);
+ switch (charCode) {
+ case CHAR_CODES.OPEN_BRACE:
+ case CHAR_CODES.OPEN_BRACKET:
+ braceDepth++;
+ break;
+ case CHAR_CODES.CLOSE_BRACE:
+ case CHAR_CODES.CLOSE_BRACKET:
+ braceDepth--;
+ break;
+ case CHAR_CODES.OPEN_PAREN:
+ parenDepth++;
+ break;
+ case CHAR_CODES.CLOSE_PAREN:
+ parenDepth--;
+ break;
+ }
+ }
+
+ // Check if function ended
+ const trimmed = line.trim();
+ if (braceDepth === 0 && parenDepth === 0 &&
+ trimmed.charCodeAt(trimmed.length - 1) === CHAR_CODES.DOT &&
+ trimmed.charCodeAt(0) !== CHAR_CODES.PERCENT) {
+ break;
+ }
+
+ i++;
+ }
+
+ // Find corresponding spec
+ const spec = this.moduleInfo.specs.find(s => s.function === name);
+
+ this.undocumentedFunctions.push({
+ name,
+ spec: spec ? spec.definition : null,
+ body: this.processFunctionBody(functionLines),
+ lines: functionLines
+ });
+ }
+
+ // Helper method to detect if a line looks like code
+ looksLikeCode(line) {
+ if (!line || line.length === 0) return false;
+
+ // Check for common code patterns
+ const codePatterns = [
+ /^[a-z][a-z0-9_]*\s*\(/, // function calls: function(
+ /^[A-Z][a-zA-Z0-9_]*\s*=/, // variable assignments: Var =
+ /^\s*\{/, // tuples/records: {
+ /^\s*\[/, // lists: [
+ /^\s*case\s+/, // case statements
+ /^\s*if\s+/, // if statements
+ /^\s*catch\s+/, // catch blocks
+ /^\s*after\s+/, // after blocks
+ /^\s*end[,.]?\s*$/, // end keywords
+ /^\s*ok\s*$/, // ok atoms
+ /^\s*true\s*$/, // boolean atoms
+ /^\s*false\s*$/, // boolean atoms
+ /->\s*$/, // arrow operators
+ /^\s*\?[A-Z]/, // macro usage
+ /^\s*[a-z_][a-z0-9_]*\s*\(/, // function definitions
+ /^\s*\d+\s*$/, // numbers
+ /^\s*".*"\s*$/, // strings
+ /^\s*<<.*>>\s*$/, // binaries
+ /^\s*#\w+/, // record syntax
+ /^\s*receive\s+/, // receive blocks
+ /^\s*spawn/, // spawn calls
+ /^\s*gen_server:/, // gen_server calls
+ /^\s*supervisor:/, // supervisor calls
+ /\bmatch\b|\bguard\b|\btry\b|\bfun\b/, // erlang keywords
+ ];
+
+ return codePatterns.some(pattern => pattern.test(line));
+ }
+
+ // Helper method to add conditional compilation directives
+ addDirectiveToOutput(line, lineNumber) {
+ this.conditionalDirectives.push({
+ line: line.trim(),
+ lineNumber: lineNumber + 1,
+ type: 'conditional_compilation'
+ });
+ }
+
processFunctionBody(lines) {
const segments = [];
const currentCode = [];
@@ -399,7 +744,7 @@ class ErlangLiterateParser {
if (parsed.params.length > 0) {
docParts.push(STRINGS.PARAMETERS_HEADER, STRINGS.EMPTY);
parsed.params.forEach(p => {
- const desc = this.cleanDocumentation(p.description || STRINGS.EMPTY);
+ const desc = this.cleanDocumentation(p.description || STRINGS.EMPTY, true);
docParts.push(`- ${STRINGS.BACKTICK}${p.name}${STRINGS.BACKTICK} - ${desc}`);
});
docParts.push(STRINGS.EMPTY);
@@ -467,12 +812,139 @@ class ErlangLiterateParser {
return text.replace(REGEX.BACKTICK_QUOTE, `${STRINGS.BACKTICK}$1${STRINGS.BACKTICK}`).trim();
}
- cleanDocumentation(text) {
+ fixCodeBlocks(text) {
+ if (!text) return text;
+
+ const lines = text.split(STRINGS.NEWLINE);
+ const result = [];
+ let inCodeBlock = false;
+
+ for (let i = 0; i < lines.length; i++) {
+ const line = lines[i];
+ const trimmed = line.trim();
+
+ // Check if this is a code block delimiter
+ if (REGEX.CODE_FENCE.test(trimmed)) {
+ if (!inCodeBlock && trimmed === '```') {
+ // Start of unmarked code block - check if it's empty first
+ let blockContent = [];
+ let j = i + 1;
+
+ // Collect content until closing ```
+ while (j < lines.length) {
+ const contentLine = lines[j];
+ if (contentLine.trim() === '```') {
+ break;
+ }
+ blockContent.push(contentLine);
+ j++;
+ }
+
+ // If block is empty, skip it entirely
+ const hasContent = blockContent.some(line => line.trim() !== '');
+ if (!hasContent) {
+ // Skip the empty code block entirely
+ i = j; // Skip to after the closing ```
+ continue;
+ }
+
+ // Determine appropriate language based on content
+ const nextLine = blockContent.length > 0 ? blockContent[0].trim() : '';
+ let language = 'text'; // default
+
+ // Heuristics to determine language based on content
+ if (nextLine.startsWith('/') || nextLine.includes('Parameters:') ||
+ nextLine.includes('- `') || nextLine.includes('(optional)')) {
+ language = 'text';
+ } else if (nextLine.includes('#{') || nextLine.includes('<<') ||
+ nextLine.includes('->') || nextLine.match(/^[a-z_]+\(/)) {
+ language = 'erlang';
+ }
+
+ // For text blocks, add ignore attribute to prevent mdBook testing
+ if (language === 'text') {
+ result.push('```text,ignore');
+ } else {
+ result.push('```' + language);
+ }
+ inCodeBlock = true;
+ } else if (inCodeBlock && trimmed === '```') {
+ // End of code block
+ result.push(line);
+ inCodeBlock = false;
+ } else {
+ // Already has language specifier or other case
+ result.push(line);
+ if (trimmed.startsWith('```')) {
+ inCodeBlock = !inCodeBlock;
+ }
+ }
+ } else {
+ result.push(line);
+ }
+ }
+
+ return result.join(STRINGS.NEWLINE);
+ }
+
+ fixModuleDocCodeBlocks(text) {
+ if (!text) return STRINGS.EMPTY;
+
+ const lines = text.split(STRINGS.NEWLINE);
+ const result = [];
+ let inCodeBlock = false;
+ let codeBlockStart = -1;
+
+ for (let i = 0; i < lines.length; i++) {
+ const line = lines[i];
+ const trimmed = line.trim();
+
+ // Check for code block start
+ if (trimmed === '```' && !inCodeBlock) {
+ inCodeBlock = true;
+ codeBlockStart = i;
+ result.push(line);
+ continue;
+ }
+
+ // Check for code block end
+ if (trimmed === '```' && inCodeBlock) {
+ inCodeBlock = false;
+ result.push(line);
+ continue;
+ }
+
+ // Check for implicit code block end (new section starting with /)
+ if (inCodeBlock && trimmed.startsWith('/')) {
+ // Close the previous code block
+ result.push('```');
+ result.push('');
+ inCodeBlock = false;
+ }
+
+ result.push(line);
+ }
+
+ // Close any unclosed code block at the end
+ if (inCodeBlock) {
+ result.push('```');
+ }
+
+ return result.join(STRINGS.NEWLINE);
+ }
+
+ cleanDocumentation(text, skipCodeBlockFix = false) {
if (!text) return STRINGS.EMPTY;
text = text.replace(REGEX.PRE_TAG, (match, content) => this.formatPreContent(content));
- const cleaned = text
+ // Only fix unmarked code blocks for module-level documentation
+ // Skip for function documentation to avoid excessive text,ignore blocks
+ if (!skipCodeBlockFix) {
+ text = this.fixCodeBlocks(text);
+ }
+
+ let cleaned = text
.replace(REGEX.BACKTICK_QUOTE, `${STRINGS.BACKTICK}$1${STRINGS.BACKTICK}`)
.replace(REGEX.HTML_ENTITIES_LT, '<<')
.replace(REGEX.HTML_ENTITIES_GT, '>>')
@@ -484,9 +956,127 @@ class ErlangLiterateParser {
return this.reflowNumberedLists(cleaned);
}
+ convertCommentStyleHeaders(text) {
+ if (!text) return text;
+
+ const lines = text.split(STRINGS.NEWLINE);
+ const result = [];
+
+ for (let i = 0; i < lines.length; i++) {
+ const line = lines[i];
+ const trimmed = line.trim();
+
+ // Look for pattern: dashes followed by text followed by dashes
+ if (trimmed.match(/^-{10,}$/)) {
+ // This is a dash line, check if next line is a header
+ if (i + 1 < lines.length) {
+ const nextLine = lines[i + 1];
+ const nextTrimmed = nextLine.trim();
+
+ // Check if the line after is also dashes (closing the header)
+ if (i + 2 < lines.length && lines[i + 2].trim().match(/^-{10,}$/)) {
+ // This is a comment-style header: convert to markdown
+ if (nextTrimmed) {
+ result.push(`## ${nextTrimmed}`);
+ result.push(STRINGS.EMPTY);
+ }
+ // Skip the next two lines (header text and closing dashes)
+ i += 2;
+ continue;
+ }
+ }
+ }
+
+ result.push(line);
+ }
+
+ return result.join(STRINGS.NEWLINE);
+ }
+
+ generateInterleavedContent(md) {
+ // Create a combined list of sections and functions, sorted by line number
+ const contentItems = [];
+
+ // Add sections
+ for (const section of this.sections) {
+ if (section.type === 'section_header') {
+ contentItems.push({
+ type: 'section',
+ lineNumber: section.lineNumber,
+ title: section.title
+ });
+ }
+ }
+
+ // Add functions
+ const groupedFunctions = this.groupFunctionsByName(this.functions);
+ for (const group of groupedFunctions) {
+ // Use the line number of the first function in the group
+ const lineNumber = group.functions[0]?.lineNumber || 0;
+ contentItems.push({
+ type: 'function_group',
+ lineNumber: lineNumber,
+ group: group
+ });
+ }
+
+ // Sort by line number
+ contentItems.sort((a, b) => a.lineNumber - b.lineNumber);
+
+ // Generate markdown for each item
+ for (const item of contentItems) {
+ if (item.type === 'section') {
+ md.push(`## ${item.title}`);
+ md.push(STRINGS.EMPTY);
+ } else if (item.type === 'function_group') {
+ this.generateFunctionGroupMarkdown(md, item.group);
+ }
+ }
+ }
+
+ generateFunctionGroupMarkdown(md, group) {
+ md.push(`## ${group.name}`);
+ md.push(STRINGS.EMPTY);
+
+ const combinedDoc = this.combineFunctionDocs(group.functions);
+ if (combinedDoc) {
+ md.push(combinedDoc);
+ md.push(STRINGS.EMPTY);
+ }
+
+ for (const func of group.functions) {
+ if (func.spec) {
+ md.push(`\`\`\`${STRINGS.ERLANG}`);
+ md.push(func.spec.trim());
+ md.push('```');
+ md.push(STRINGS.EMPTY);
+ }
+
+ if (func.body?.length > 0) {
+ md.push(STRINGS.EMPTY);
+ for (const segment of func.body) {
+ if (segment.type === 'comment') {
+ md.push(segment.content);
+ md.push(STRINGS.EMPTY);
+ } else if (segment.type === 'doc') {
+ md.push(segment.content);
+ md.push(STRINGS.EMPTY);
+ } else if (segment.type === 'code') {
+ md.push(`\`\`\`${STRINGS.ERLANG}`);
+ md.push(segment.content.trim());
+ md.push('```');
+ md.push(STRINGS.EMPTY);
+ }
+ }
+ }
+ }
+
+ md.push(STRINGS.EMPTY);
+ }
+
formatReturnsText(text) {
if (!text) return STRINGS.EMPTY;
- let result = this.cleanDocumentation(text);
+ let result = this.cleanDocumentation(text, true);
const leadingMatch = result.match(REGEX.LEADING_RETURN_TOKEN);
if (leadingMatch) {
@@ -501,7 +1091,7 @@ class ErlangLiterateParser {
splitReturnsIntoOutcomes(text) {
if (!text) return [];
- const s = this.cleanDocumentation(text);
+ const s = this.cleanDocumentation(text, true);
const matches = [];
let match;
@@ -734,6 +1324,31 @@ class ErlangLiterateParser {
return result;
}
+ addSeparator(md) {
+ // Only add separator if the last entry is not empty and not already a separator
+ if (md.length > 0) {
+ const lastLine = md[md.length - 1];
+ const prevLine = md.length > 1 ? md[md.length - 2] : '';
+
+ // Don't add separator if last line is already empty and previous is separator
+ if (lastLine === STRINGS.EMPTY && prevLine === STRINGS.SEPARATOR) {
+ return;
+ }
+
+ // Don't add separator if last line is already a separator
+ if (lastLine === STRINGS.SEPARATOR) {
+ return;
+ }
+
+ // Add separator with proper spacing
+ if (lastLine !== STRINGS.EMPTY) {
+ md.push(STRINGS.EMPTY);
+ }
+ md.push(STRINGS.SEPARATOR);
+ md.push(STRINGS.EMPTY);
+ }
+ }
+
generateMarkdown(fileName) {
const githubUrl = `${this.options.githubBase}/${fileName}`;
const md = [];
@@ -744,38 +1359,61 @@ class ErlangLiterateParser {
md.push(`[View source on GitHub](${githubUrl})`);
md.push(STRINGS.EMPTY);
+ // Metadata section
+ this.generateMetadataSection(md);
+
// Module documentation
if (this.moduleInfo.doc) {
md.push(this.moduleInfo.doc);
+ this.addSeparator(md);
+ }
+
+ // Generate interleaved content (sections and functions) sorted by line number
+ this.generateInterleavedContent(md);
+
+ // Commented-out code blocks
+ if (this.commentedCodeBlocks && this.commentedCodeBlocks.length > 0) {
+ md.push('## Commented-Out Code');
md.push(STRINGS.EMPTY);
- md.push(STRINGS.SEPARATOR);
+ md.push('*The following code blocks are commented out but may contain useful examples:*');
md.push(STRINGS.EMPTY);
+
+ for (const block of this.commentedCodeBlocks) {
+ md.push('```erlang');
+ for (const line of block.lines) {
+ md.push(line);
+ }
+ md.push('```');
+ md.push(STRINGS.EMPTY);
+ }
}
- // Exports
- if (this.moduleInfo.exports?.length > 0) {
- md.push(STRINGS.EXPORTED_FUNCTIONS);
+ // Conditional compilation directives
+ if (this.conditionalDirectives && this.conditionalDirectives.length > 0) {
+ md.push('## Conditional Compilation');
md.push(STRINGS.EMPTY);
- this.moduleInfo.exports.forEach(exp =>
- md.push(`- ${STRINGS.BACKTICK}${exp}${STRINGS.BACKTICK}`));
+ md.push('*The following conditional compilation directives are used in this module:*');
md.push(STRINGS.EMPTY);
- md.push(STRINGS.SEPARATOR);
+
+ md.push('```erlang');
+ for (const directive of this.conditionalDirectives) {
+ md.push(directive.line);
+ }
+ md.push('```');
md.push(STRINGS.EMPTY);
}
- const groupedFunctions = this.groupFunctionsByName(this.functions);
-
- for (const group of groupedFunctions) {
- md.push(`## ${group.name}`);
+ // Undocumented functions section
+ if (this.undocumentedFunctions.length > 0) {
+ md.push('## Undocumented Functions');
+ md.push(STRINGS.EMPTY);
+ md.push('*The following functions lack documentation comments but are included for completeness:*');
md.push(STRINGS.EMPTY);
- const combinedDoc = this.combineFunctionDocs(group.functions);
- if (combinedDoc) {
- md.push(combinedDoc);
+ for (const func of this.undocumentedFunctions) {
+ md.push(`### ${func.name}`);
md.push(STRINGS.EMPTY);
- }
- for (const func of group.functions) {
if (func.spec) {
md.push(`\`\`\`${STRINGS.ERLANG}`);
md.push(func.spec.trim());
@@ -784,14 +1422,10 @@ class ErlangLiterateParser {
}
if (func.body?.length > 0) {
- md.push(STRINGS.EMPTY);
for (const segment of func.body) {
if (segment.type === 'comment') {
md.push(segment.content);
md.push(STRINGS.EMPTY);
- } else if (segment.type === 'doc') {
- md.push(segment.content);
- md.push(STRINGS.EMPTY);
} else if (segment.type === 'code') {
md.push(`\`\`\`${STRINGS.ERLANG}`);
md.push(segment.content.trim());
@@ -802,14 +1436,109 @@ class ErlangLiterateParser {
}
}
- md.push(STRINGS.EMPTY);
+ this.addSeparator(md);
}
- md.push(STRINGS.SEPARATOR);
- md.push(STRINGS.EMPTY);
+ this.addSeparator(md);
md.push(`*Generated from [${fileName}](${githubUrl})*`);
- return md.join(STRINGS.NEWLINE);
+ const finalMarkdown = md.join(STRINGS.NEWLINE);
+ return finalMarkdown;
+ }
+
+ generateMetadataSection(md) {
+ md.push('## Module Metadata');
+ md.push(STRINGS.EMPTY);
+
+ // Basic module information
+ md.push(`**Module:** \`${this.moduleInfo.name || 'unknown'}\``);
+ md.push(`**Exports:** ${this.moduleInfo.exports.length} functions`);
+
+ if (this.moduleInfo.behaviours.length > 0) {
+ md.push(`**Behaviours:** ${this.moduleInfo.behaviours.map(b => `\`${b}\``).join(', ')}`);
+ }
+
+ if (this.moduleInfo.includes.length > 0) {
+ md.push(`**Includes:** ${this.moduleInfo.includes.length} files`);
+ }
+
+ if (this.moduleInfo.defines.length > 0) {
+ md.push(`**Defines:** ${this.moduleInfo.defines.length} macros`);
+ }
+
+ if (this.moduleInfo.records.length > 0) {
+ md.push(`**Records:** ${this.moduleInfo.records.length} records`);
+ }
+
+ if (this.moduleInfo.types.length > 0) {
+ md.push(`**Types:** ${this.moduleInfo.types.length} type definitions`);
+ }
+
+ md.push(STRINGS.EMPTY);
+
+ // Exports section
+ if (this.moduleInfo.exports.length > 0) {
+ md.push('### Exported Functions');
+ md.push(STRINGS.EMPTY);
+ this.moduleInfo.exports.forEach(exp => {
+ md.push(`- \`${exp}\``);
+ });
+ md.push(STRINGS.EMPTY);
+ }
+
+ // Includes section
+ if (this.moduleInfo.includes.length > 0) {
+ md.push('### Includes');
+ md.push(STRINGS.EMPTY);
+ md.push('```erlang');
+ this.moduleInfo.includes.forEach(inc => {
+ md.push(inc.line);
+ });
+ md.push('```');
+ md.push(STRINGS.EMPTY);
+ }
+
+ // Defines section
+ if (this.moduleInfo.defines.length > 0) {
+ md.push('### Macro Definitions');
+ md.push(STRINGS.EMPTY);
+ md.push('```erlang');
+ this.moduleInfo.defines.forEach(def => {
+ md.push(def.line);
+ });
+ md.push('```');
+ md.push(STRINGS.EMPTY);
+ }
+
+ // Records section
+ if (this.moduleInfo.records.length > 0) {
+ md.push('### Record Definitions');
+ md.push(STRINGS.EMPTY);
+ this.moduleInfo.records.forEach(rec => {
+ md.push(`#### \`${rec.name}\``);
+ md.push(STRINGS.EMPTY);
+ md.push('```erlang');
+ md.push(rec.definition);
+ md.push('```');
+ md.push(STRINGS.EMPTY);
+ });
+ }
+
+ // Types section
+ if (this.moduleInfo.types.length > 0) {
+ md.push('### Type Definitions');
+ md.push(STRINGS.EMPTY);
+ this.moduleInfo.types.forEach(type => {
+ md.push(`#### \`${type.name}\``);
+ md.push(STRINGS.EMPTY);
+ md.push('```erlang');
+ md.push(type.definition);
+ md.push('```');
+ md.push(STRINGS.EMPTY);
+ });
+ }
+
+ this.addSeparator(md);
}
groupFunctionsByName(functions) {
@@ -835,7 +1564,7 @@ class ErlangLiterateParser {
const combinedDoc = [];
if (parsed.description.length > 0) {
- combinedDoc.push(this.cleanDocumentation(parsed.description.join(STRINGS.NEWLINE)));
+ combinedDoc.push(this.cleanDocumentation(parsed.description.join(STRINGS.NEWLINE), true));
combinedDoc.push(STRINGS.EMPTY);
}
@@ -843,7 +1572,7 @@ class ErlangLiterateParser {
combinedDoc.push(STRINGS.PARAMETERS_HEADER);
combinedDoc.push(STRINGS.EMPTY);
parsed.params.forEach(param => {
- const desc = this.cleanDocumentation(param.description);
+ const desc = this.cleanDocumentation(param.description, true);
combinedDoc.push(`- ${STRINGS.BACKTICK}${param.name}${STRINGS.BACKTICK} - ${desc}`);
});
combinedDoc.push(STRINGS.EMPTY);
@@ -870,7 +1599,7 @@ function main() {
const verbose = args.includes('-v') || args.includes('--verbose');
const srcDir = process.env.SRC_DIR || path.join(process.cwd(), 'src');
- const outputDir = process.env.OUTPUT_DIR || path.join(process.cwd(), 'docs/literate-erlang');
+ const outputDir = process.env.OUTPUT_DIR || path.join(process.cwd(), 'docs/book/src');
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
diff --git a/docs/build-literate-erlang-js.sh b/docs/generate-literate-docs.sh
similarity index 51%
rename from docs/build-literate-erlang-js.sh
rename to docs/generate-literate-docs.sh
index 91b5545ce..0c1328897 100755
--- a/docs/build-literate-erlang-js.sh
+++ b/docs/generate-literate-docs.sh
@@ -16,16 +16,54 @@ cd "$ROOT_DIR"
# Configuration
SRC_DIR="${SRC_DIR:-$ROOT_DIR/src}"
-OUTPUT_DIR="${OUTPUT_DIR:-$ROOT_DIR/docs/literate-erlang}"
+OUTPUT_DIR="${OUTPUT_DIR:-$ROOT_DIR/docs/book/src}"
PARSER_SCRIPT="$SCRIPT_DIR/erlang-literate-parser.js"
# Parse arguments
VERBOSE=false
+DRY_RUN=false
+SHOW_HELP=false
+
+if [[ "$@" == *"-h"* ]] || [[ "$@" == *"--help"* ]]; then
+ SHOW_HELP=true
+fi
if [[ "$@" == *"-v"* ]] || [[ "$@" == *"--verbose"* ]]; then
VERBOSE=true
fi
+if [[ "$@" == *"--dry-run"* ]] || [[ "$@" == *"--dryrun"* ]]; then
+ DRY_RUN=true
+fi
+
+if [ "$SHOW_HELP" = true ]; then
+ echo -e "${GREEN}HyperBEAM Literate Erlang Documentation Generator (JavaScript)${NC}"
+ echo "========================================================"
+ echo ""
+ echo "Usage: $0 [OPTIONS]"
+ echo ""
+ echo "Options:"
+ echo " -v, --verbose Enable verbose output"
+ echo " --dry-run Simulate deployment without actually deploying"
+ echo " -h, --help Show this help message"
+ echo ""
+ echo "Environment Variables:"
+ echo " SRC_DIR Source directory (default: ./src)"
+ echo " OUTPUT_DIR Output directory (default: ./docs/book/src)"
+ echo " DEPLOY_KEY Arweave wallet key for deployment"
+ echo " ANT_PROCESS ANT process ID for ArNS deployment"
+ echo ""
+ echo "Examples:"
+ echo " $0 # Generate documentation normally"
+ echo " $0 -v # Generate with verbose output"
+ echo " $0 --dry-run # Test deployment without deploying"
+ echo " $0 --dry-run -v # Dry run with verbose output"
+ echo ""
+ exit 0
+fi
echo -e "${GREEN}HyperBEAM Literate Erlang Documentation Generator (JavaScript)${NC}"
+if [ "$DRY_RUN" = true ]; then
+ echo -e "${YELLOW}[DRY RUN MODE] - No actual deployment will occur${NC}"
+fi
echo "========================================================"
# Check for Node.js
@@ -106,31 +144,53 @@ if [ $PARSER_EXIT_CODE -eq 0 ]; then
# Copy to mdBook if it exists
if [ -d "$ROOT_DIR/docs/book/src" ]; then
- echo -e "${GREEN}Copying documentation to mdBook...${NC}"
- cp "$OUTPUT_DIR"/*.md "$ROOT_DIR/docs/book/src/" 2>/dev/null
- if [ $? -eq 0 ]; then
- echo -e "${GREEN}✓ Documentation copied to mdBook${NC}"
+ if [ "$DRY_RUN" = true ]; then
+ echo -e "${YELLOW}[DRY RUN] Would copy documentation to mdBook...${NC}"
+ echo -e "${YELLOW}[DRY RUN] ✓ Documentation would be copied to mdBook${NC}"
else
- echo -e "${YELLOW}Warning: Could not copy to mdBook (no files generated?)${NC}"
+ echo -e "${GREEN}Copying documentation to mdBook...${NC}"
+ cp "$OUTPUT_DIR"/*.md "$ROOT_DIR/docs/book/src/" 2>/dev/null
+ if [ $? -eq 0 ]; then
+ echo -e "${GREEN}✓ Documentation copied to mdBook${NC}"
+ else
+ echo -e "${YELLOW}Warning: Could not copy to mdBook (no files generated?)${NC}"
+ fi
fi
fi
# Build mdBook if available
if command -v mdbook &> /dev/null && [ -f "$ROOT_DIR/docs/book/book.toml" ]; then
- echo -e "${GREEN}Building mdBook...${NC}"
- cd "$ROOT_DIR/docs/book"
- mdbook build
- if [ $? -eq 0 ]; then
- echo -e "${GREEN}✓ mdBook built successfully${NC}"
- echo "View at: file://$ROOT_DIR/docs/book/book/index.html"
+ if [ "$DRY_RUN" = true ]; then
+ echo -e "${YELLOW}[DRY RUN] Would build mdBook...${NC}"
+ echo -e "${YELLOW}[DRY RUN] ✓ mdBook would be built successfully${NC}"
+ echo -e "${YELLOW}[DRY RUN] Would be viewable at: file://$ROOT_DIR/docs/book/dist/index.html${NC}"
else
- echo -e "${YELLOW}Warning: mdBook build failed${NC}"
+ echo -e "${GREEN}Building mdBook...${NC}"
+ cd "$ROOT_DIR/docs/book"
+ mdbook build
+ if [ $? -eq 0 ]; then
+ echo -e "${GREEN}✓ mdBook built successfully${NC}"
+ echo "View at: file://$ROOT_DIR/docs/book/dist/index.html"
+ else
+ echo -e "${YELLOW}Warning: mdBook build failed${NC}"
+ fi
+ cd "$ROOT_DIR"
fi
- cd "$ROOT_DIR"
+ fi
+
+ # Run deployment dry run if requested
+ if [ "$DRY_RUN" = true ]; then
+ echo ""
+ echo -e "${YELLOW}Running deployment dry run...${NC}"
+ "$SCRIPT_DIR/deploy-dry-run.sh"
fi
echo ""
- echo -e "${GREEN}Documentation generation complete!${NC}"
+ if [ "$DRY_RUN" = true ]; then
+ echo -e "${YELLOW}Documentation generation and deployment dry run complete!${NC}"
+ else
+ echo -e "${GREEN}Documentation generation complete!${NC}"
+ fi
echo "Output directory: $OUTPUT_DIR"
else
diff --git a/docs/build-and-serve.sh b/docs/serve-book.sh
similarity index 96%
rename from docs/build-and-serve.sh
rename to docs/serve-book.sh
index c590e7c8d..ef7ddaea6 100755
--- a/docs/build-and-serve.sh
+++ b/docs/serve-book.sh
@@ -14,7 +14,7 @@ echo -e "${BLUE}🚀 Building and serving HyperBEAM documentation...${NC}"
# Generate literate documentation
echo -e "${GREEN}📚 Generating literate documentation...${NC}"
-./build-literate-erlang-js.sh
+./generate-literate-docs.sh
# Build and serve mdBook
echo -e "${GREEN}📖 Building mdBook...${NC}"
From 81e2a440bd854fafabde60d5789346bfcb5ad860 Mon Sep 17 00:00:00 2001
From: Dylan Shade <63427984+dpshade@users.noreply.github.com>
Date: Wed, 24 Sep 2025 17:24:52 -0400
Subject: [PATCH 17/17] docs: Update documentation build workflow to use new
script
- Replace deprecated `build-literate-erlang.sh` with `generate-literate-docs.sh` in the GitHub Actions workflow.
- Ensure the workflow reflects the latest script for generating literate documentation, maintaining consistency in the build process.
---
.github/workflows/build-deploy-mdbook.yml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/build-deploy-mdbook.yml b/.github/workflows/build-deploy-mdbook.yml
index 2780d2c22..717c493b9 100644
--- a/.github/workflows/build-deploy-mdbook.yml
+++ b/.github/workflows/build-deploy-mdbook.yml
@@ -10,7 +10,7 @@ on:
- "docs/book/book.toml"
- "docs/book/custom.css"
- "docs/book/custom.js"
- - "docs/build-literate-erlang.sh"
+ - "docs/generate-literate-docs.sh"
- ".github/workflows/build-deploy-mdbook.yml"
push:
branches:
@@ -21,7 +21,7 @@ on:
- "docs/book/book.toml"
- "docs/book/custom.css"
- "docs/book/custom.js"
- - "docs/build-literate-erlang.sh"
+ - "docs/generate-literate-docs.sh"
- ".github/workflows/build-deploy-mdbook.yml"
# Perform a release using a workflow dispatch
@@ -50,7 +50,7 @@ jobs:
- name: 📝 Generate Literate Erlang Documentation
run: |
- ./docs/build-literate-erlang.sh -v
+ ./docs/generate-literate-docs.sh -v
- name: 📚 Build mdBook Documentation
run: |
@@ -94,7 +94,7 @@ jobs:
- name: 📝 Generate Literate Erlang Documentation
run: |
- ./docs/build-literate-erlang.sh -v
+ ./docs/generate-literate-docs.sh -v
- name: 📚 Build mdBook Documentation
id: build_mdbook