Introduce ProgramNamePlugin plugin (#904)

abhinavsingh · pre-commit-ci[bot] · web-flow · commit 13c64b535735 · 2021-12-25T12:40:25.000+05:30
* Add `ProgramNamePlugin` * Update readme * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove `_compat.py` * Add suggestions coming from #659 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
diff --git a/.flake8 b/.flake8
@@ -148,7 +148,6 @@ extend-ignore =
   WPS420  # FIXME: pointless keyword like `pass`
   WPS421  # FIXME: call to `print()`
   WPS425  # FIXME: bool non-keyword arg
-  WPS427  # FIXME: unreachable code
   WPS428  # FIXME: pointless statement
   WPS430  # FIXME: nested func
   WPS431  # FIXME: nested class
diff --git a/.gitignore b/.gitignore
@@ -22,6 +22,8 @@ proxy.py.iml
 *.crt
 *.key
 *.pem
+
+.venv*
 venv*
 
 cover
diff --git a/.pylintrc b/.pylintrc
@@ -127,7 +127,6 @@ disable=raw-checker-failed,
         too-many-return-statements,
         too-many-statements,
         unnecessary-pass,
-        unreachable,
         unused-argument,
         useless-return,
         useless-super-delegation,
diff --git a/README.md b/README.md
@@ -54,11 +54,12 @@
     - [Cache Responses Plugin](#cacheresponsesplugin)
     - [Man-In-The-Middle Plugin](#maninthemiddleplugin)
     - [Proxy Pool Plugin](#proxypoolplugin)
-    - [FilterByClientIpPlugin](#filterbyclientipplugin)
-    - [ModifyChunkResponsePlugin](#modifychunkresponseplugin)
-    - [CloudflareDnsResolverPlugin](#cloudflarednsresolverplugin)
-    - [CustomDnsResolverPlugin](#customdnsresolverplugin)
-    - [CustomNetworkInterface](#customnetworkinterface)
+    - [Filter By Client IP Plugin](#filterbyclientipplugin)
+    - [Modify Chunk Response Plugin](#modifychunkresponseplugin)
+    - [Cloudflare DNS Resolver Plugin](#cloudflarednsresolverplugin)
+    - [Custom DNS Resolver Plugin](#customdnsresolverplugin)
+    - [Custom Network Interface](#customnetworkinterface)
+    - [Program Name Plugin](#programnameplugin)
   - [HTTP Web Server Plugins](#http-web-server-plugins)
     - [Reverse Proxy](#reverse-proxy)
     - [Web Server Route](#web-server-route)
@@ -578,7 +579,7 @@ Verify mock API response using `curl -x localhost:8899 http://api.example.com/v1
 Verify the same by inspecting `proxy.py` logs:
 
 ```console
-2019-09-27 12:44:02,212 - INFO - pid:7077 - access_log:1210 - ::1:64792 - GET None:None/v1/users/ - None None - 0 byte
+... [redacted] ... - access_log:1210 - ::1:64792 - GET None:None/v1/users/ - None None - 0 byte
 ```
 
 Access log shows `None:None` as server `ip:port`. `None` simply means that
@@ -618,8 +619,8 @@ Verify the same by inspecting the logs for `proxy.py`.
 Along with the proxy request log, you must also see a http web server request log.
 
 ```
-2019-09-24 19:09:33,602 - INFO - pid:49996 - access_log:1241 - ::1:49525 - GET /
-2019-09-24 19:09:33,603 - INFO - pid:49995 - access_log:1157 - ::1:49524 - GET localhost:8899/ - 404 NOT FOUND - 70 bytes
+... [redacted] ... - access_log:1241 - ::1:49525 - GET /
+... [redacted] ... - access_log:1157 - ::1:49524 - GET localhost:8899/ - 404 NOT FOUND - 70 bytes
 ```
 
 ### FilterByUpstreamHostPlugin
@@ -650,10 +651,10 @@ Above `418 I'm a tea pot` is sent by our plugin.
 Verify the same by inspecting logs for `proxy.py`:
 
 ```console
-2019-09-24 19:21:37,893 - ERROR - pid:50074 - handle_readables:1347 - HttpProtocolException type raised
+... [redacted] ... - handle_readables:1347 - HttpProtocolException type raised
 Traceback (most recent call last):
 ... [redacted] ...
-2019-09-24 19:21:37,897 - INFO - pid:50074 - access_log:1157 - ::1:49911 - GET None:None/ - None None - 0 bytes
+... [redacted] ... - access_log:1157 - ::1:49911 - GET None:None/ - None None - 0 bytes
 ```
 
 ### CacheResponsesPlugin
@@ -887,6 +888,34 @@ for more details.
 PS: There is no plugin named, but [CustomDnsResolverPlugin](#customdnsresolverplugin)
 can be easily customized according to your needs.
 
+### ProgramNamePlugin
+
+Attempts to resolve program `(application)` name for proxy requests originating from the local machine.
+If identified, client IP in the access logs is replaced with program name.
+
+Start `proxy.py` as:
+
+```console
+❯ proxy \
+    --plugins proxy.plugin.ProgramNamePlugin
+```
+
+Make a request using `curl`:
+
+```console
+❯ curl -v -x localhost:8899 https://httpbin.org/get
+```
+
+You must see log lines like this:
+
+```console
+... [redacted] ... - [I] server.access_log:419 - curl:58096 - CONNECT httpbin.org:443 - 6010 bytes - 1824.62ms
+```
+
+Notice `curl` in-place of `::1` or `127.0.0.1` as client IP.
+
+[![WARNING](https://img.shields.io/static/v1?label=Compatibility&message=warning&color=red)](#programnameplugin) If `ProgramNamePlugin` does not work reliably on your operating system, kindly contribute by sending a pull request and/or open an issue.  Thank you!!!
+
 ## HTTP Web Server Plugins
 
 ### Reverse Proxy
diff --git a/proxy/common/_compat.py b/proxy/common/_compat.py
diff --git a/proxy/common/constants.py b/proxy/common/constants.py
@@ -13,14 +13,17 @@
 import time
 import secrets
 import pathlib
+import platform
 import sysconfig
 import ipaddress
 
 from typing import Any, List
 
-from ._compat import IS_WINDOWS  # noqa: WPS436
 from .version import __version__
 
+SYS_PLATFORM = platform.system()
+IS_WINDOWS = SYS_PLATFORM == 'Windows'
+
 
 def _env_threadless_compliant() -> bool:
     """Returns true for Python 3.8+ across all platforms
@@ -96,12 +99,13 @@ def _env_threadless_compliant() -> bool:
 DEFAULT_LOG_FILE = None
 DEFAULT_LOG_FORMAT = '%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(module)s.%(funcName)s:%(lineno)d - %(message)s'
 DEFAULT_LOG_LEVEL = 'INFO'
-DEFAULT_WEB_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - {request_method} {request_path} - {connection_time_ms}ms'
-DEFAULT_HTTP_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
+DEFAULT_WEB_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' \
+    '{request_method} {request_path} - {request_ua} - {connection_time_ms}ms'
+DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
     '{request_method} {server_host}:{server_port}{request_path} - ' + \
     '{response_code} {response_reason} - {response_bytes} bytes - ' + \
     '{connection_time_ms}ms'
-DEFAULT_HTTPS_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
+DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
     '{request_method} {server_host}:{server_port} - ' + \
     '{response_bytes} bytes - {connection_time_ms}ms'
 DEFAULT_NUM_ACCEPTORS = 0
diff --git a/proxy/common/flag.py b/proxy/common/flag.py
@@ -19,15 +19,14 @@
 
 from typing import Optional, List, Any, cast
 
-from ._compat import IS_WINDOWS  # noqa: WPS436
 from .plugins import Plugins
 from .types import IpAddress
 from .utils import bytes_, is_py2, is_threadless, set_open_file_limit
 from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_ACCEPTORS, DEFAULT_NUM_WORKERS
 from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE
 from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, DEFAULT_MIN_COMPRESSION_LIMIT
 from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE
-from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH
+from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH, IS_WINDOWS
 from .logger import Logger
 
 from .version import __version__
diff --git a/proxy/common/types.py b/proxy/common/types.py
@@ -22,6 +22,7 @@
 else:
     from typing_extensions import Protocol
 
+
 if TYPE_CHECKING:
     DictQueueType = queue.Queue[Dict[str, Any]]    # pragma: no cover
 else:
diff --git a/proxy/common/utils.py b/proxy/common/utils.py
@@ -23,8 +23,10 @@
 from types import TracebackType
 from typing import Optional, Dict, Any, List, Tuple, Type, Callable
 
-from ._compat import IS_WINDOWS  # noqa: WPS436
-from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT, DEFAULT_THREADLESS
+from .constants import (
+    HTTP_1_1, COLON, WHITESPACE, CRLF,
+    DEFAULT_TIMEOUT, DEFAULT_THREADLESS, IS_WINDOWS,
+)
 
 if not IS_WINDOWS:
     import resource
diff --git a/proxy/core/event/dispatcher.py b/proxy/core/event/dispatcher.py
@@ -83,11 +83,7 @@ def run(self) -> None:
                     self.run_once()
                 except queue.Empty:
                     pass
-        except BrokenPipeError:
-            pass
-        except EOFError:
-            pass
-        except KeyboardInterrupt:
+        except (BrokenPipeError, EOFError, KeyboardInterrupt):
             pass
         except Exception as e:
             logger.exception('Dispatcher exception', exc_info=e)
diff --git a/proxy/core/event/subscriber.py b/proxy/core/event/subscriber.py
@@ -103,9 +103,7 @@ def unsubscribe(self) -> None:
             return
         try:
             self.event_queue.unsubscribe(self.relay_sub_id)
-        except BrokenPipeError:
-            pass
-        except EOFError:
+        except (BrokenPipeError, EOFError):
             pass
         finally:
             # self.relay_sub_id = None
diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py
@@ -38,7 +38,7 @@
 from ...common.constants import DEFAULT_CA_KEY_FILE, DEFAULT_CA_SIGNING_KEY_FILE
 from ...common.constants import COMMA, DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CERT_FILE
 from ...common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_DISABLE_HEADERS
-from ...common.constants import DEFAULT_HTTP_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_ACCESS_LOG_FORMAT
+from ...common.constants import DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT, DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT
 from ...common.constants import DEFAULT_DISABLE_HTTP_PROXY, PLUGIN_PROXY_AUTH
 from ...common.utils import text_
 from ...common.pki import gen_public_key, gen_csr, sign_csr
@@ -413,9 +413,9 @@ def on_client_connection_close(self) -> None:
             )
 
     def access_log(self, log_attrs: Dict[str, Any]) -> None:
-        access_log_format = DEFAULT_HTTPS_ACCESS_LOG_FORMAT
+        access_log_format = DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT
         if not self.request.is_https_tunnel:
-            access_log_format = DEFAULT_HTTP_ACCESS_LOG_FORMAT
+            access_log_format = DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT
         logger.info(access_log_format.format_map(log_attrs))
 
     def on_response_chunk(self, chunk: List[memoryview]) -> List[memoryview]:
diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py
@@ -272,10 +272,10 @@ def on_client_connection_close(self) -> None:
             'request_method': text_(self.request.method),
             'request_path': text_(self.request.path),
             'request_bytes': self.request.total_size,
-            'request_ua': self.request.header(b'user-agent')
+            'request_ua': text_(self.request.header(b'user-agent'))
             if self.request.has_header(b'user-agent')
             else None,
-            'request_version': self.request.version,
+            'request_version': None if not self.request.version else text_(self.request.version),
             # Response
             #
             # TODO: Track and inject web server specific response attributes
diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py
@@ -27,6 +27,7 @@
 from .modify_chunk_response import ModifyChunkResponsePlugin
 from .custom_dns_resolver import CustomDnsResolverPlugin
 from .cloudflare_dns import CloudflareDnsResolverPlugin
+from .program_name import ProgramNamePlugin
 
 __all__ = [
     'CacheResponsesPlugin',
@@ -45,4 +46,5 @@
     'FilterByURLRegexPlugin',
     'CustomDnsResolverPlugin',
     'CloudflareDnsResolverPlugin',
+    'ProgramNamePlugin',
 ]
diff --git a/proxy/plugin/filter_by_url_regex.py b/proxy/plugin/filter_by_url_regex.py
@@ -72,8 +72,7 @@ def handle_client_request(
             request.path,
         )
         # check URL against list
-        rule_number = 1
-        for blocked_entry in self.filters:
+        for rule_number, blocked_entry in enumerate(self.filters, start=1):
             # if regex matches on URL
             if re.search(text_(blocked_entry['regex']), text_(url)):
                 # log that the request has been filtered
@@ -90,8 +89,4 @@ def handle_client_request(
                     status_code=httpStatusCodes.NOT_FOUND,
                     reason=b'Blocked',
                 )
-                # stop looping through filter list
-                break
-            # increment rule number
-            rule_number += 1
         return request
diff --git a/proxy/plugin/man_in_the_middle.py b/proxy/plugin/man_in_the_middle.py
@@ -15,5 +15,5 @@
 class ManInTheMiddlePlugin(HttpProxyBasePlugin):
     """Modifies upstream server responses."""
 
-    def handle_upstream_chunk(self, chunk: memoryview) -> memoryview:
+    def handle_upstream_chunk(self, _chunk: memoryview) -> memoryview:
         return okResponse(content=b'Hello from man in the middle')
diff --git a/proxy/plugin/program_name.py b/proxy/plugin/program_name.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import os
+import subprocess
+
+from typing import Any, Dict, Optional
+
+from ..common.utils import text_
+from ..common.constants import IS_WINDOWS
+
+from ..http.parser import HttpParser
+from ..http.proxy import HttpProxyBasePlugin
+
+
+class ProgramNamePlugin(HttpProxyBasePlugin):
+    """Tries to identify the application connecting to the
+    proxy instance.  This is only possible when connection
+    itself originates from the same machine where the proxy
+    instance is running."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.program_name: Optional[str] = None
+
+    def before_upstream_connection(
+            self, request: HttpParser,
+    ) -> Optional[HttpParser]:
+        if IS_WINDOWS:
+            raise NotImplementedError()
+        assert self.client.addr
+        if self.client.addr[0] in ('::1', '127.0.0.1'):
+            assert self.client.addr
+            port = self.client.addr[1]
+            ls = subprocess.Popen(
+                ('lsof', '-i', '-P', '-n'),
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            try:
+                output = subprocess.check_output(
+                    ('grep', '{0}'.format(port)),
+                    stdin=ls.stdout,
+                )
+                port_programs = output.splitlines()
+                for program in port_programs:
+                    parts = program.split()
+                    if int(parts[1]) != os.getpid():
+                        self.program_name = text_(parts[0])
+                        break
+            except subprocess.CalledProcessError:
+                pass
+            finally:
+                ls.wait(timeout=1)
+        if self.program_name is None:
+            self.program_name = self.client.addr[0]
+        return request
+
+    def on_access_log(self, context: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        context.update({'client_ip': self.program_name})
+        return context
diff --git a/proxy/plugin/web_server_route.py b/proxy/plugin/web_server_route.py
@@ -13,7 +13,6 @@
 
 from ..http.responses import okResponse
 from ..http.parser import HttpParser
-from ..http.websocket import WebsocketFrame
 from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes
 
 logger = logging.getLogger(__name__)
@@ -37,12 +36,3 @@ def handle_request(self, request: HttpParser) -> None:
             self.client.queue(HTTP_RESPONSE)
         elif request.path == b'/https-route-example':
             self.client.queue(HTTPS_RESPONSE)
-
-    def on_websocket_open(self) -> None:
-        logger.info('Websocket open')
-
-    def on_websocket_message(self, frame: WebsocketFrame) -> None:
-        logger.info(frame.data)
-
-    def on_client_connection_close(self) -> None:
-        logger.debug('Client connection close')
diff --git a/tests/core/test_listener.py b/tests/core/test_listener.py
diff --git a/tests/http/test_http2.py b/tests/http/test_http2.py
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
diff --git a/tests/test_set_open_file_limit.py b/tests/test_set_open_file_limit.py
diff --git a/tests/testing/test_embed.py b/tests/testing/test_embed.py

-Original file line number
+Diff line change
 *.crt
 *.key
 *.pem
++
 +.venv*
 venv*
 cover