From bc6b13634c8a21bf4f2418fcaf6e70fe2f5cf258 Mon Sep 17 00:00:00 2001 From: JD Babac Date: Fri, 9 May 2025 23:43:16 +0800 Subject: [PATCH 1/5] IDEV-2050: Add e2e tests for proxy and ssl. --- tests/e2e/scripts/test_e2e_runner.sh | 41 +++++++++++++++++++++++ tests/e2e/test_proxy.py | 49 ++++++++++++++++++++++++++++ tests/e2e/test_ssl.py | 42 ++++++++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100755 tests/e2e/scripts/test_e2e_runner.sh create mode 100644 tests/e2e/test_proxy.py create mode 100644 tests/e2e/test_ssl.py diff --git a/tests/e2e/scripts/test_e2e_runner.sh b/tests/e2e/scripts/test_e2e_runner.sh new file mode 100755 index 0000000..6370ee8 --- /dev/null +++ b/tests/e2e/scripts/test_e2e_runner.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Script Requirements +# docker + +MITM_BASIC_AUTH_CONTAINER_NAME="e2e_test_mitm_basic_auth" +MITM_CUSTOM_CERT_CONTAINER_NAME="e2e_test_mitm_custom_cert" +DOCKER_NETWORK_NAME="e2e_test_docker_network" + +echo "Starting e2e tests..." + +echo "Create a bridge network for the containers to communicate" +docker network create ${DOCKER_NETWORK_NAME} + +echo "Spinning ${MITM_BASIC_AUTH_CONTAINER_NAME}" +docker run --rm -d \ + --name ${MITM_BASIC_AUTH_CONTAINER_NAME} \ + --network ${DOCKER_NETWORK_NAME} \ + -p 8080:8080 mitmproxy/mitmproxy mitmdump \ + --set proxyauth="username:pass" + +echo "Spinning ${MITM_CUSTOM_CERT_CONTAINER_NAME}" +docker run --rm -d -v ~/.test_mitmproxy:/home/mitmproxy/.mitmproxy \ + --name ${MITM_CUSTOM_CERT_CONTAINER_NAME} \ + --network ${DOCKER_NETWORK_NAME} \ + -p 8090:8090 mitmproxy/mitmproxy mitmdump \ + --set listen_port=8090 + +cp ~/.test_mitmproxy/mitmproxy-ca.pem tests/e2e/mitmproxy-ca.pem +python -m pytest -s --capture=sys -v --cov=domaintools tests/e2e + +# Clean up from the run +echo "Bringing down containers..." +docker stop ${MITM_BASIC_AUTH_CONTAINER_NAME} || true +docker stop ${MITM_CUSTOM_CERT_CONTAINER_NAME} || true +docker network rm ${DOCKER_NETWORK_NAME} || true + +# Clean up custom certs +echo "Removing custom certs..." +rm -rf tests/e2e/mitmproxy-ca.pem +rm -rf ~/.test_mitmproxy:/home/mitmproxy/.mitmproxy diff --git a/tests/e2e/test_proxy.py b/tests/e2e/test_proxy.py new file mode 100644 index 0000000..b96170a --- /dev/null +++ b/tests/e2e/test_proxy.py @@ -0,0 +1,49 @@ +"""Tests proxy support for DomainTools APIs""" + +import pytest + +from httpx import ProxyError +from tests.settings import api + + +class TestProxy: + + def test_account_information_using_proxy_with_basic_auth_should_succeed_if_correct_username_and_password_were_used(self): + api.verify_ssl = False + api.proxy_url = "http://username:pass@localhost:8080" + with api.account_information() as account_information: + assert "products" in account_information + for product in account_information: + assert "id" in product + assert "per_month_limit" in product + assert "absolute_limit" in product + assert "usage" in product + assert "expiration_date" in product + + def test_account_information_using_proxy_with_basic_auth_should_fail_if_no_username_and_password_was_used(self): + api.verify_ssl = False + api.proxy_url = "http://localhost:8080" + with pytest.raises(ProxyError): + account_information = api.account_information() + account_information.response() + + def test_account_information_using_proxy_with_basic_auth_should_fail_if_wrong_username_was_used(self): + api.verify_ssl = False + api.proxy_url = "http://wrongusername:pass@localhost:8080" + with pytest.raises(ProxyError): + account_information = api.account_information() + account_information.response() + + def test_account_information_using_proxy_with_basic_auth_should_fail_if_wrong_password_was_used(self): + api.verify_ssl = False + api.proxy_url = "http://username:wrongpass@localhost:8080" + with pytest.raises(ProxyError): + account_information = api.account_information() + account_information.response() + + def test_account_information_using_proxy_with_basic_auth_should_fail_if_wrong_username_and_password_were_used(self): + api.verify_ssl = False + api.proxy_url = "http://wrongusername:wrongpass@localhost:8080" + with pytest.raises(ProxyError): + account_information = api.account_information() + account_information.response() diff --git a/tests/e2e/test_ssl.py b/tests/e2e/test_ssl.py new file mode 100644 index 0000000..2c61713 --- /dev/null +++ b/tests/e2e/test_ssl.py @@ -0,0 +1,42 @@ +"""Tests proxy support for DomainTools APIs""" + +import pytest + +from httpx import ConnectError +from tests.settings import api + + +class TestSsl: + + def test_account_information_using_proxy_with_custom_cert_should_succeed_if_using_the_valid_custom_cert(self): + api.verify_ssl = "tests/e2e/mitmproxy-ca.pem" + api.proxy_url = "http://localhost:8090" + with api.account_information() as account_information: + assert "products" in account_information + for product in account_information: + assert "id" in product + assert "per_month_limit" in product + assert "absolute_limit" in product + assert "usage" in product + assert "expiration_date" in product + + def test_account_information_using_proxy_with_custom_cert_should_be_successful_when_verify_ssl_is_disabled(self): + api.verify_ssl = False + api.proxy_url = "http://localhost:8090" + with api.account_information() as account_information: + assert "products" in account_information + for product in account_information: + assert "id" in product + assert "per_month_limit" in product + assert "absolute_limit" in product + assert "usage" in product + assert "expiration_date" in product + + def test_account_information_using_proxy_with_custom_cert_should_fail_when_verify_ssl_is_enabled_but_not_using_the_installed_custom_cert( + self, + ): + api.verify_ssl = True + api.proxy_url = "http://localhost:8090" + with pytest.raises(ConnectError): + account_information = api.account_information() + account_information.response() From 66b3b365005fbab36fbb27b4398d218c73e88018 Mon Sep 17 00:00:00 2001 From: JD Babac Date: Fri, 9 May 2025 23:43:48 +0800 Subject: [PATCH 2/5] IDEV-2050: Modify tox.ini to ignore e2e tests. --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e7c06b0..0c998b8 100644 --- a/tox.ini +++ b/tox.ini @@ -14,5 +14,7 @@ deps= rich typer . -commands=py.test -s --capture=sys --cov=domaintools --cov=domaintools_async tests +commands = + py.test -s --capture=sys --cov=domaintools --cov=domaintools_async tests \ + --ignore=tests/e2e coverage html From d80c396f46700029d83d30ecff3e434ab1adba52 Mon Sep 17 00:00:00 2001 From: JD Babac Date: Sat, 10 May 2025 00:49:27 +0800 Subject: [PATCH 3/5] IDEV-2050: Fix dir in cleaning custom cert. --- tests/e2e/scripts/test_e2e_runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/scripts/test_e2e_runner.sh b/tests/e2e/scripts/test_e2e_runner.sh index 6370ee8..60ddb9c 100755 --- a/tests/e2e/scripts/test_e2e_runner.sh +++ b/tests/e2e/scripts/test_e2e_runner.sh @@ -38,4 +38,4 @@ docker network rm ${DOCKER_NETWORK_NAME} || true # Clean up custom certs echo "Removing custom certs..." rm -rf tests/e2e/mitmproxy-ca.pem -rm -rf ~/.test_mitmproxy:/home/mitmproxy/.mitmproxy +rm -rf ~/.test_mitmproxy From b702b0c24b5de555ec7420ac4472413a0137182f Mon Sep 17 00:00:00 2001 From: JD Babac Date: Sat, 10 May 2025 03:17:42 +0800 Subject: [PATCH 4/5] IDEV-2050: Use ssl.create_default_context for verify because `verify=` is deprecated in httpx. --- domaintools/api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/domaintools/api.py b/domaintools/api.py index 4e6586f..d20823b 100644 --- a/domaintools/api.py +++ b/domaintools/api.py @@ -1,8 +1,10 @@ from datetime import datetime, timedelta, timezone from hashlib import sha1, sha256, md5 from hmac import new as hmac +from typing import Union import re +import ssl from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, FEEDS_PRODUCTS_LIST, OutputFormat from domaintools._version import current as version @@ -76,7 +78,7 @@ def __init__( self.username = username self.key = key self.https = https - self.verify_ssl = verify_ssl + self.verify_ssl = self._get_ssl_default_context(verify_ssl) self.rate_limit = rate_limit self.proxy_url = proxy_url self.extra_request_params = {} @@ -92,6 +94,9 @@ def __init__( if proxy_url and not isinstance(proxy_url, str): raise Exception("Proxy URL must be a string. For example: '127.0.0.1:8888'") + def _get_ssl_default_context(self, verify_ssl: Union[str, bool]): + return ssl.create_default_context(cafile=verify_ssl) if isinstance(verify_ssl, str) else verify_ssl + def _build_api_url(self, api_url=None, api_port=None): """Build the API url based on the given url and port. Defaults to `https://api.domaintools.com`""" rest_api_url = "https://api.domaintools.com" From 1f918f62f0f00a1352625a79091aad39814391c6 Mon Sep 17 00:00:00 2001 From: JD Babac Date: Sat, 10 May 2025 03:18:31 +0800 Subject: [PATCH 5/5] IDEV-2050: Improve test e2e runner script. --- tests/e2e/scripts/test_e2e_runner.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/e2e/scripts/test_e2e_runner.sh b/tests/e2e/scripts/test_e2e_runner.sh index 60ddb9c..48c8a37 100755 --- a/tests/e2e/scripts/test_e2e_runner.sh +++ b/tests/e2e/scripts/test_e2e_runner.sh @@ -26,10 +26,21 @@ docker run --rm -d -v ~/.test_mitmproxy:/home/mitmproxy/.mitmproxy \ -p 8090:8090 mitmproxy/mitmproxy mitmdump \ --set listen_port=8090 +# Check until custom cert from mitmproxy container is copied locally +echo "Checking for valid custom cert..." +while [ ! -f ~/.test_mitmproxy/mitmproxy-ca.pem ] ; +do + sleep 2 +done +echo "Valid custom cert found!" + +# Copy valid custom cert to target dir cp ~/.test_mitmproxy/mitmproxy-ca.pem tests/e2e/mitmproxy-ca.pem + +echo "E2E processing..." python -m pytest -s --capture=sys -v --cov=domaintools tests/e2e -# Clean up from the run +# Clean up containers echo "Bringing down containers..." docker stop ${MITM_BASIC_AUTH_CONTAINER_NAME} || true docker stop ${MITM_CUSTOM_CERT_CONTAINER_NAME} || true