From 380b0b00a97bea2b9c6413164e9bc7b8d3e2ea3a Mon Sep 17 00:00:00 2001 From: Sergey Prokhorov Date: Fri, 5 Oct 2012 19:31:32 +0400 Subject: [PATCH 1/6] Now can connect through ssl/not ssl proxy to *not ssl* host --- src/lhttpc_client.erl | 93 +++++++++++++++++++++++++++++++++++-------- src/lhttpc_lib.erl | 41 +++++++++++++++++++ 2 files changed, 117 insertions(+), 17 deletions(-) diff --git a/src/lhttpc_client.erl b/src/lhttpc_client.erl index f33f34d3..d465fa44 100644 --- a/src/lhttpc_client.erl +++ b/src/lhttpc_client.erl @@ -105,30 +105,26 @@ execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> PartialDownload = proplists:is_defined(partial_download, Options), PartialDownloadOptions = proplists:get_value(partial_download, Options, []), NormalizedMethod = lhttpc_lib:normalize_method(Method), - Proxy = case proplists:get_value(proxy, Options) of - undefined -> - undefined; - ProxyUrl when is_list(ProxyUrl), not Ssl -> - % The point of HTTP CONNECT proxying is to use TLS tunneled in - % a plain HTTP/1.1 connection to the proxy (RFC2817). - throw(origin_server_not_https); - ProxyUrl when is_list(ProxyUrl) -> - lhttpc_lib:parse_url(ProxyUrl) - end, - {ChunkedUpload, Request} = lhttpc_lib:format_request(Path, NormalizedMethod, - Hdrs, Host, Port, Body, PartialUpload), + + %% this can be target host's or proxy's host/port/ssl !?!? + %% How do we handle situation like + %% > request(host1, proxy1) + %% > request(host1, proxy2) + %% ??? SocketRequest = {socket, self(), Host, Port, Ssl}, Pool = proplists:get_value(pool, Options, whereis(lhttpc_manager)), Socket = case gen_server:call(Pool, SocketRequest, infinity) of {ok, S} -> S; % Re-using HTTP/1.1 connections no_socket -> undefined % Opening a new HTTP/1.1 connection end, + + State = #client_state{ host = Host, port = Port, ssl = Ssl, method = NormalizedMethod, - request = Request, + %% request = Request, requester = From, request_headers = Hdrs, socket = Socket, @@ -138,17 +134,21 @@ execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> attempts = 1 + proplists:get_value(send_retry, Options, 1), partial_upload = PartialUpload, upload_window = UploadWindowSize, - chunked_upload = ChunkedUpload, + %% chunked_upload = ChunkedUpload, partial_download = PartialDownload, download_window = proplists:get_value(window_size, PartialDownloadOptions, infinity), part_size = proplists:get_value(part_size, PartialDownloadOptions, infinity), - proxy = Proxy, - proxy_setup = (Socket =/= undefined), + %% proxy = Proxy, + %% proxy_setup = (Socket =/= undefined), proxy_ssl_options = proplists:get_value(proxy_ssl_options, Options, []) }, - Response = case send_request(State) of + + State2 = configure_proxy(State, Body, Path, Ssl, + proplists:get_value(proxy, Options)), + + Response = case send_request(State2) of {R, undefined} -> {ok, R}; {R, NewSocket} -> @@ -171,6 +171,65 @@ execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> end, {response, self(), Response}. +configure_proxy(State, Body, Path, _Ssl, undefined) -> + %% no proxy at all + set_request(#client_state{proxy=undefined}, Body, Path); +configure_proxy(State, Body, Path, true, ProxyUrl) when is_list(ProxyUrl) -> + %% connect through ssl/not ssl proxy to ssl host using CONNECT + Proxy = lhttpc_lib:parse_url(ProxyUrl), + set_request(State#client_state{ + proxy = Proxy, + proxy_setup = (State#client_state.socket =/= undefined)}, + Body, Path); +configure_proxy(State, Body, Path, false, ProxyUrl) -> + %% connect through ssl/not ssl proxy to not ssl host + %% just replace Path with full host URL + #client_state{request_headers=Hdrs, + host=DestHost, + port=DestPort} = State, + Proxy = lhttpc_lib:parse_url(ProxyUrl), + #lhttpc_url{ + %% host = Host, + %% port = Port, + user = User, + password = Passwd + %% is_ssl = SslProxy + } = Proxy, + NewPath = lhttpc_lib:format_url( + #lhttpc_url{host=DestHost, + port=DestPort, + path=Path, + is_ssl=false}), + Hdrs2 = case User of + "" -> + Hdrs; + User -> + AuthHdr = {"Proxy-Authorization", + "Basic " ++ base64:encode_to_string(User ++ ":" ++ Passwd)}, + [AuthHdr | Hdrs] + end, + set_request(State#client_state{ + %% ssl = SslProxy, % see request_first_destination + %% host = Host, + %% port = Port, + request_headers = Hdrs2, + proxy = Proxy, + proxy_setup = true}, + NewPath, Body). + + +set_request(State, Body, Path) -> + #client_state{partial_upload=PartialUpload, + method=NormalizedMethod, + request_headers=Hdrs, + host=Host, + port=Port} = State, + {ChunkedUpload, Request} = lhttpc_lib:format_request( + Path, NormalizedMethod, Hdrs, Host, Port, Body, PartialUpload), + State#client_state{request=Request, + chunked_upload = ChunkedUpload}. + + send_request(#client_state{attempts = 0}) -> % Don't try again if the number of allowed attempts is 0. throw(connection_closed); diff --git a/src/lhttpc_lib.erl b/src/lhttpc_lib.erl index f02d2136..f7008bf9 100644 --- a/src/lhttpc_lib.erl +++ b/src/lhttpc_lib.erl @@ -32,6 +32,7 @@ -export([ parse_url/1, + format_url/1, format_request/7, header_value/2, header_value/3, @@ -96,6 +97,46 @@ maybe_atom_to_list(Atom) when is_atom(Atom) -> maybe_atom_to_list(List) -> List. +format_url(URL) -> + #lhttpc_url{ + host = Host, + port = Port, + path = Path, + user = User, + password = Passwd, + is_ssl = IsSSL + } = URL, + U1 = add_scheme("", IsSSL), + U2 = add_credentials(U1, User, Passwd), + U3 = add_host(U2, Host), + U4 = add_port(U3, Port, IsSSL), + add_path(U4, Path). + +add_scheme(_, true) -> + "https://"; +add_scheme(_, false) -> + "http://". + +add_credentials(Scheme, "", "") -> + Scheme; +add_credentials(Scheme, User, Passwd) -> + Scheme ++ User ++ ":" ++ Passwd ++ "@". + +add_host(SUP, Host) -> + SUP ++ Host. + +add_port(SUPH, 80, false) -> + SUPH; +add_port(SUPH, 443, true) -> + SUPH; +add_port(SUPH, Port, _IsSSL) -> + SUPH ++ ":" ++ integer_to_list(Port). + +add_path(SUPHP, Path) -> + SUPHP ++ Path. + + + %% @spec (URL) -> #lhttpc_url{} %% URL = string() %% @doc From 62267df51efe1e1f95731be2eabaf36537c0044f Mon Sep 17 00:00:00 2001 From: Sergey Prokhorov Date: Fri, 5 Oct 2012 19:38:48 +0400 Subject: [PATCH 2/6] Typo fix --- src/lhttpc_client.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lhttpc_client.erl b/src/lhttpc_client.erl index d465fa44..89e970d0 100644 --- a/src/lhttpc_client.erl +++ b/src/lhttpc_client.erl @@ -173,7 +173,7 @@ execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> configure_proxy(State, Body, Path, _Ssl, undefined) -> %% no proxy at all - set_request(#client_state{proxy=undefined}, Body, Path); + set_request(State#client_state{proxy=undefined}, Body, Path); configure_proxy(State, Body, Path, true, ProxyUrl) when is_list(ProxyUrl) -> %% connect through ssl/not ssl proxy to ssl host using CONNECT Proxy = lhttpc_lib:parse_url(ProxyUrl), From b83c79d22f77eccc27c3f91130ea4a78ce148241 Mon Sep 17 00:00:00 2001 From: Sergey Prokhorov Date: Fri, 5 Oct 2012 19:47:49 +0400 Subject: [PATCH 3/6] Another typo =) --- src/lhttpc_client.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lhttpc_client.erl b/src/lhttpc_client.erl index 89e970d0..07f5b996 100644 --- a/src/lhttpc_client.erl +++ b/src/lhttpc_client.erl @@ -215,7 +215,7 @@ configure_proxy(State, Body, Path, false, ProxyUrl) -> request_headers = Hdrs2, proxy = Proxy, proxy_setup = true}, - NewPath, Body). + Body, NewPath). set_request(State, Body, Path) -> From c460a66a661b77da92e1a8e9181cfc5aeda774a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9F=D1=80=D0=BE?= =?UTF-8?q?=D1=85=D0=BE=D1=80=D0=BE=D0=B2?= Date: Sat, 6 Oct 2012 04:07:53 +0400 Subject: [PATCH 4/6] format_url/1 improevements * ipv6 support * use iolist() + lists:flatten/1 for url construction --- src/lhttpc_lib.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lhttpc_lib.erl b/src/lhttpc_lib.erl index f7008bf9..6e8aab67 100644 --- a/src/lhttpc_lib.erl +++ b/src/lhttpc_lib.erl @@ -110,7 +110,7 @@ format_url(URL) -> U2 = add_credentials(U1, User, Passwd), U3 = add_host(U2, Host), U4 = add_port(U3, Port, IsSSL), - add_path(U4, Path). + lists:flatten(add_path(U4, Path)). add_scheme(_, true) -> "https://"; @@ -120,20 +120,21 @@ add_scheme(_, false) -> add_credentials(Scheme, "", "") -> Scheme; add_credentials(Scheme, User, Passwd) -> - Scheme ++ User ++ ":" ++ Passwd ++ "@". + [Scheme, User, ":", Passwd, "@"]. add_host(SUP, Host) -> - SUP ++ Host. + Host2 = maybe_ipv6_enclose(Host), + [SUP, Host2]. add_port(SUPH, 80, false) -> SUPH; add_port(SUPH, 443, true) -> SUPH; add_port(SUPH, Port, _IsSSL) -> - SUPH ++ ":" ++ integer_to_list(Port). + [SUPH, ":", integer_to_list(Port)]. add_path(SUPHP, Path) -> - SUPHP ++ Path. + [SUPHP, Path]. From 2b2ee128212bd0129c9ef639760d8ed8953e3f02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9F=D1=80=D0=BE?= =?UTF-8?q?=D1=85=D0=BE=D1=80=D0=BE=D0=B2?= Date: Sat, 6 Oct 2012 04:36:39 +0400 Subject: [PATCH 5/6] Tests for format_url + fix scheme://user@host case (no password) --- src/lhttpc_lib.erl | 2 + test/lhttpc_lib_tests.erl | 92 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/lhttpc_lib.erl b/src/lhttpc_lib.erl index 6e8aab67..6b7531b9 100644 --- a/src/lhttpc_lib.erl +++ b/src/lhttpc_lib.erl @@ -119,6 +119,8 @@ add_scheme(_, false) -> add_credentials(Scheme, "", "") -> Scheme; +add_credentials(Scheme, User, "") -> + [Scheme, User, "@"]; add_credentials(Scheme, User, Passwd) -> [Scheme, User, ":", Passwd, "@"]. diff --git a/test/lhttpc_lib_tests.erl b/test/lhttpc_lib_tests.erl index cb688127..acaa0243 100644 --- a/test/lhttpc_lib_tests.erl +++ b/test/lhttpc_lib_tests.erl @@ -201,5 +201,95 @@ parse_url_test_() -> user = "joe", password = "erlang" }, - lhttpc_lib:parse_url("http://joe:erlang@[1080:0:0:0:8:800:200C:417A]:180/foo/bar")) + lhttpc_lib:parse_url("http://joe:erlang@[1080:0:0:0:8:800:200C:417A]:180/foo/bar")), + + %% Serialisation + ?_assertEqual("http://host/", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 80, + path = "/", + is_ssl = false, + user = "", + password = "" + })), + + ?_assertEqual("https://host/", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 443, + path = "/", + is_ssl = true, + user = "", + password = "" + })), + + ?_assertEqual("http://host:180/", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/", + is_ssl = false, + user = "", + password = "" + })), + + ?_assertEqual("https://host:180/", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/", + is_ssl = true, + user = "", + password = "" + })), + + ?_assertEqual("http://host:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/path", + is_ssl = false, + user = "", + password = "" + })), + + ?_assertEqual("http://user@host:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/path", + is_ssl = false, + user = "user", + password = "" + })), + + ?_assertEqual("http://user:pass@host:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/path", + is_ssl = false, + user = "user", + password = "pass" + })), + ?_assertEqual("http://:pass@host:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/path", + is_ssl = false, + user = "", + password = "pass" + })), + + ?_assertEqual("http://user:pass@[::1]:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "::1", + port = 180, + path = "/path", + is_ssl = false, + user = "user", + password = "pass" + })) ]. From 3b083b486cbaa878b1d1c8698bcf347b8336a073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9F=D1=80=D0=BE?= =?UTF-8?q?=D1=85=D0=BE=D1=80=D0=BE=D0=B2?= Date: Sun, 16 Oct 2016 19:32:59 +0300 Subject: [PATCH 6/6] Convert to utf-8 --- src/lhttpc.app.src | 2 +- src/lhttpc.erl | 2 +- src/lhttpc_client.erl | 2 +- src/lhttpc_lib.erl | 2 +- src/lhttpc_manager.erl | 2 +- src/lhttpc_sock.erl | 2 +- src/lhttpc_sup.erl | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/lhttpc.app.src b/src/lhttpc.app.src index e853322a..32c8d0b0 100644 --- a/src/lhttpc.app.src +++ b/src/lhttpc.app.src @@ -24,7 +24,7 @@ %%% ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %%% ---------------------------------------------------------------------------- -%%% @author Oscar Hellström +%%% @author Oscar Hellström %%% @doc This is the specification for the lhttpc application. %%% @end {application, lhttpc, diff --git a/src/lhttpc.erl b/src/lhttpc.erl index d269c579..86b5f071 100644 --- a/src/lhttpc.erl +++ b/src/lhttpc.erl @@ -24,7 +24,7 @@ %%% ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %%% ---------------------------------------------------------------------------- -%%% @author Oscar Hellström +%%% @author Oscar Hellström %%% @doc Main interface to the lightweight http client. %%% See {@link request/4}, {@link request/5} and {@link request/6} functions. -module(lhttpc). diff --git a/src/lhttpc_client.erl b/src/lhttpc_client.erl index 07f5b996..9dc92083 100644 --- a/src/lhttpc_client.erl +++ b/src/lhttpc_client.erl @@ -25,7 +25,7 @@ %%% ---------------------------------------------------------------------------- %%% @private -%%% @author Oscar Hellström +%%% @author Oscar Hellström %%% @doc %%% This module implements the HTTP request handling. This should normally %%% not be called directly since it should be spawned by the lhttpc module. diff --git a/src/lhttpc_lib.erl b/src/lhttpc_lib.erl index 6b7531b9..edd471fd 100644 --- a/src/lhttpc_lib.erl +++ b/src/lhttpc_lib.erl @@ -25,7 +25,7 @@ %%% ---------------------------------------------------------------------------- %%% @private -%%% @author Oscar Hellström +%%% @author Oscar Hellström %%% @doc %%% This module implements various library functions used in lhttpc. -module(lhttpc_lib). diff --git a/src/lhttpc_manager.erl b/src/lhttpc_manager.erl index 65a76976..9ce27805 100644 --- a/src/lhttpc_manager.erl +++ b/src/lhttpc_manager.erl @@ -24,7 +24,7 @@ %%% ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %%% ---------------------------------------------------------------------------- -%%% @author Oscar Hellström +%%% @author Oscar Hellström %%% @author Filipe David Manana %%% @doc Connection manager for the HTTP client. %%% This gen_server is responsible for keeping track of persistent diff --git a/src/lhttpc_sock.erl b/src/lhttpc_sock.erl index 507f74a2..abf5d524 100644 --- a/src/lhttpc_sock.erl +++ b/src/lhttpc_sock.erl @@ -25,7 +25,7 @@ %%% ---------------------------------------------------------------------------- %%% @private -%%% @author Oscar Hellström +%%% @author Oscar Hellström %%% @doc %%% This module implements wrappers for socket operations. %%% Makes it possible to have the same interface to ssl and tcp sockets. diff --git a/src/lhttpc_sup.erl b/src/lhttpc_sup.erl index 3dca9582..5663a3e2 100644 --- a/src/lhttpc_sup.erl +++ b/src/lhttpc_sup.erl @@ -24,7 +24,7 @@ %%% ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %%% ---------------------------------------------------------------------------- -%%% @author Oscar Hellström +%%% @author Oscar Hellström %%% @doc Top supervisor for the lhttpc application. %%% This is normally started by the application behaviour implemented in %%% {@link lhttpc}.