Add a skeleton app to demonstrate how to use proxy.py for standalone projects (abhinavsingh#1029)

* Add a skeleton app structure

* Update `README.md` for skeleton app

* Add `skeleton-app` to pre commit

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update readme

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: abhinavsingh

.pre-commit-config.yaml

@@ -169,6 +169,7 @@ repos:
     - --strict-optional
     - benchmark/
     - examples/
+    - skeleton/
     - proxy/
     - tests/
     pass_filenames: false

README.md

@@ -1784,7 +1784,9 @@ Listed below are a few strategies for using `proxy.py` in your private/productio
 
 > You MUST `avoid forking` the repository *"just"* to put your plugin code in `proxy/plugin` directory.  Forking is recommended workflow for project contributors, NOT for project users.
 
-Instead, use one of the suggested approaches from below.  Then load your plugins using `--plugin`, `--plugins` flags or `plugin` kwargs.
+- Instead, use one of the suggested approaches from below.
+- Then load your plugins using `--plugin`, `--plugins` flags or `plugin` kwargs.
+- See [skeleton](https://github.com/abhinavsingh/proxy.py/tree/develop/skeleton) app for example standalone project using `proxy.py`.
 
 ### Via Requirements
 
@@ -2243,33 +2245,33 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE]
           [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--enable-events]
           [--threadless] [--threaded] [--num-workers NUM_WORKERS]
-          [--backlog BACKLOG] [--hostname HOSTNAME] [--port PORT]
-          [--port-file PORT_FILE] [--unix-socket-path UNIX_SOCKET_PATH]
-          [--local-executor LOCAL_EXECUTOR] [--num-acceptors NUM_ACCEPTORS]
-          [--version] [--log-level LOG_LEVEL] [--log-file LOG_FILE]
-          [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT]
+          [--local-executor LOCAL_EXECUTOR] [--backlog BACKLOG]
+          [--hostname HOSTNAME] [--port PORT] [--port-file PORT_FILE]
+          [--unix-socket-path UNIX_SOCKET_PATH]
+          [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL]
+          [--log-file LOG_FILE] [--log-format LOG_FORMAT]
+          [--open-file-limit OPEN_FILE_LIMIT]
           [--plugins PLUGINS [PLUGINS ...]] [--enable-dashboard]
-          [--enable-ssh-tunnel] [--work-klass WORK_KLASS]
-          [--pid-file PID_FILE] [--enable-conn-pool] [--key-file KEY_FILE]
+          [--basic-auth BASIC_AUTH] [--enable-ssh-tunnel]
+          [--work-klass WORK_KLASS] [--pid-file PID_FILE]
+          [--enable-proxy-protocol] [--enable-conn-pool] [--key-file KEY_FILE]
           [--cert-file CERT_FILE] [--client-recvbuf-size CLIENT_RECVBUF_SIZE]
           [--server-recvbuf-size SERVER_RECVBUF_SIZE] [--timeout TIMEOUT]
-          [--enable-proxy-protocol] [--disable-http-proxy]
-          [--disable-headers DISABLE_HEADERS] [--ca-key-file CA_KEY_FILE]
-          [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE]
-          [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE]
-          [--auth-plugin AUTH_PLUGIN] [--basic-auth BASIC_AUTH]
-          [--cache-dir CACHE_DIR]
-          [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS]
-          [--enable-web-server] [--enable-static-server]
-          [--static-server-dir STATIC_SERVER_DIR]
+          [--disable-http-proxy] [--disable-headers DISABLE_HEADERS]
+          [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR]
+          [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE]
+          [--ca-signing-key-file CA_SIGNING_KEY_FILE]
+          [--auth-plugin AUTH_PLUGIN] [--cache-dir CACHE_DIR]
+          [--proxy-pool PROXY_POOL] [--enable-web-server]
+          [--enable-static-server] [--static-server-dir STATIC_SERVER_DIR]
           [--min-compression-length MIN_COMPRESSION_LENGTH]
           [--pac-file PAC_FILE] [--pac-file-url-path PAC_FILE_URL_PATH]
-          [--proxy-pool PROXY_POOL]
+          [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
+          [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS]
           [--filtered-client-ips FILTERED_CLIENT_IPS]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
-          [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
 
-proxy.py v2.4.0rc7.dev12+gd234339.d20220116
+proxy.py v2.4.0rc7.dev28+gfbd7b46.d20220120
 
 options:
   -h, --help            show this help message and exit
@@ -2299,6 +2301,14 @@ options:
                         handle each client connection.
   --num-workers NUM_WORKERS
                         Defaults to number of CPU cores.
+  --local-executor LOCAL_EXECUTOR
+                        Default: 1. Enabled by default. Use 0 to disable. When
+                        enabled acceptors will make use of local (same
+                        process) executor instead of distributing load across
+                        remote (other process) executors. Enable this option
+                        to achieve CPU affinity between acceptors and
+                        executors, instead of using underlying OS kernel
+                        scheduling algorithm.
   --backlog BACKLOG     Default: 100. Maximum number of pending connections to
                         proxy server
   --hostname HOSTNAME   Default: 127.0.0.1. Server IP address.
@@ -2309,14 +2319,6 @@ options:
   --unix-socket-path UNIX_SOCKET_PATH
                         Default: None. Unix socket path to use. When provided
                         --host and --port flags are ignored
-  --local-executor LOCAL_EXECUTOR
-                        Default: 1. Enabled by default. Use 0 to disable. When
-                        enabled acceptors will make use of local (same
-                        process) executor instead of distributing load across
-                        remote (other process) executors. Enable this option
-                        to achieve CPU affinity between acceptors and
-                        executors, instead of using underlying OS kernel
-                        scheduling algorithm.
   --num-acceptors NUM_ACCEPTORS
                         Defaults to number of CPU cores.
   --version, -v         Prints proxy.py version.
@@ -2335,11 +2337,17 @@ options:
                         Comma separated plugins. You may use --plugins flag
                         multiple times.
   --enable-dashboard    Default: False. Enables proxy.py dashboard.
+  --basic-auth BASIC_AUTH
+                        Default: No authentication. Specify colon separated
+                        user:password to enable basic authentication.
   --enable-ssh-tunnel   Default: False. Enable SSH tunnel.
   --work-klass WORK_KLASS
                         Default: proxy.http.HttpProtocolHandler. Work klass to
                         use for work execution.
   --pid-file PID_FILE   Default: None. Save "parent" process ID to a file.
+  --enable-proxy-protocol
+                        Default: False. If used, will enable proxy protocol.
+                        Only version 1 is currently supported.
   --enable-conn-pool    Default: False. (WIP) Enable upstream connection
                         pooling.
   --key-file KEY_FILE   Default: None. Server key file to enable end-to-end
@@ -2358,9 +2366,6 @@ options:
   --timeout TIMEOUT     Default: 10.0. Number of seconds after which an
                         inactive connection must be dropped. Inactivity is
                         defined by no data sent or received by the client.
-  --enable-proxy-protocol
-                        Default: False. If used, will enable proxy protocol.
-                        Only version 1 is currently supported.
   --disable-http-proxy  Default: False. Whether to disable
                         proxy.HttpProxyPlugin.
   --disable-headers DISABLE_HEADERS
@@ -2388,17 +2393,13 @@ options:
                         generation of HTTPS certificates. If used, must also
                         pass --ca-key-file and --ca-cert-file
   --auth-plugin AUTH_PLUGIN
-                        Default: proxy.http.proxy.AuthPlugin. Auth plugin to
-                        use instead of default basic auth plugin.
-  --basic-auth BASIC_AUTH
-                        Default: No authentication. Specify colon separated
-                        user:password to enable basic authentication.
+                        Default: proxy.http.proxy.auth.AuthPlugin. Auth plugin
+                        to use instead of default basic auth plugin.
   --cache-dir CACHE_DIR
                         Default: A temporary directory. Flag only applicable
                         when cache plugin is used with on-disk storage.
-  --filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS
-                        Default: Blocks Facebook. Comma separated list of IPv4
-                        and IPv6 addresses.
+  --proxy-pool PROXY_POOL
+                        List of upstream proxies to use in the pool
   --enable-web-server   Default: False. Whether to enable
                         proxy.HttpWebServerPlugin.
   --enable-static-server
@@ -2419,18 +2420,19 @@ options:
                         this option enables proxy.HttpWebServerPlugin.
   --pac-file-url-path PAC_FILE_URL_PATH
                         Default: /. Web server path to serve the PAC file.
-  --proxy-pool PROXY_POOL
-                        List of upstream proxies to use in the pool
+  --cloudflare-dns-mode CLOUDFLARE_DNS_MODE
+                        Default: security. Either "security" (for malware
+                        protection) or "family" (for malware and adult content
+                        protection)
+  --filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS
+                        Default: Blocks Facebook. Comma separated list of IPv4
+                        and IPv6 addresses.
   --filtered-client-ips FILTERED_CLIENT_IPS
                         Default: 127.0.0.1,::1. Comma separated list of IPv4
                         and IPv6 addresses.
   --filtered-url-regex-config FILTERED_URL_REGEX_CONFIG
                         Default: No config. Comma separated list of IPv4 and
                         IPv6 addresses.
-  --cloudflare-dns-mode CLOUDFLARE_DNS_MODE
-                        Default: security. Either "security" (for malware
-                        protection) or "family" (for malware and adult content
-                        protection)
 
 Proxy.py not working? Report at:
 https://github.com/abhinavsingh/proxy.py/issues/new

check.py

@@ -37,6 +37,7 @@
     list(REPO_ROOT.glob('*.py')) +
     list((REPO_ROOT / 'proxy').rglob('*.py')) +
     list((REPO_ROOT / 'examples').rglob('*.py')) +
+    list((REPO_ROOT / 'skeleton').rglob('*.py')) +
     list((REPO_ROOT / 'benchmark').rglob('*.py')) +
     list((REPO_ROOT / 'tests').rglob('*.py'))
 )

skeleton/README.md

@@ -0,0 +1,37 @@
+# Skeleton App
+
+This directory contains a sample standalone application structure which uses `proxy.py`
+via `requirements.txt` file.
+
+## Setup
+
+```console
+$ git clone https://github.com/abhinavsingh/proxy.py.git
+$ cd proxy.py/skeleton
+$ python3 -m venv .venv
+$ source .venv/bin/activate
+$ pip install -r requirements.txt
+```
+
+## Run It
+
+Start your app and make a web request to `/` and a proxy request via the instance. You will
+see log lines like this:
+
+```console
+$ python -m app
+...[redacted]... - Loaded plugin proxy.http.proxy.HttpProxyPlugin
+...[redacted]... - Loaded plugin proxy.http.server.HttpWebServerPlugin
+...[redacted]... - Loaded plugin app.plugins.MyWebServerPlugin
+...[redacted]... - Loaded plugin app.plugins.MyProxyPlugin
+...[redacted]... - Listening on 127.0.0.1:9000
+...[redacted]... - Started 16 acceptors in threadless (local) mode
+...[redacted]... - HttpProtocolException: HttpRequestRejected b"I'm a tea pot"
+...[redacted]... - 127.0.0.1:64601 - GET None:None/get - None None - 0 bytes - 0.64ms
+...[redacted]... - 127.0.0.1:64622 - GET / - curl/7.77.0 - 0.95ms
+```
+
+Voila!!!
+
+That is your custom app skeleton structure built on top of `proxy.py`.  Now copy the `app` directory
+outside of `proxy.py` repo and create your own git repo.  Customize the `app` for your project needs

skeleton/app/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+from .app import entry_point
+
+
+__all__ = [
+    'entry_point',
+]

skeleton/app/__main__.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+from app import entry_point
+
+
+if __name__ == '__main__':
+    entry_point()

skeleton/app/app.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import proxy
+
+
+def entry_point() -> None:
+    with proxy.Proxy(
+            enable_web_server=True,
+            port=9000,
+            # NOTE: Pass plugins via *args if you define custom flags.
+            # Currently plugins passed via **kwargs are not discovered for
+            # custom flags by proxy.py
+            #
+            # See https://github.com/abhinavsingh/proxy.py/issues/871
+            plugins=[
+                'app.plugins.MyWebServerPlugin',
+                'app.plugins.MyProxyPlugin',
+            ],
+    ) as _:
+        proxy.sleep_loop()

skeleton/app/plugins/__init__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+from .my_web_plugin import MyWebServerPlugin
+from .my_proxy_plugin import MyProxyPlugin
+
+
+__all__ = [
+    'MyWebServerPlugin',
+    'MyProxyPlugin',
+]

skeleton/app/plugins/my_proxy_plugin.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+
+    .. spelling::
+
+       ip
+"""
+from typing import Optional
+
+from proxy.http import httpStatusCodes
+from proxy.http.proxy import HttpProxyBasePlugin
+from proxy.http.parser import HttpParser
+from proxy.http.exception import HttpRequestRejected
+
+
+class MyProxyPlugin(HttpProxyBasePlugin):
+    """Drop traffic by inspecting incoming client IP address."""
+
+    def before_upstream_connection(
+            self, request: HttpParser,
+    ) -> Optional[HttpParser]:
+        assert not self.flags.unix_socket_path and self.client.addr
+        if self.client.addr[0] in '127.0.0.1,::1'.split(','):
+            raise HttpRequestRejected(
+                status_code=httpStatusCodes.I_AM_A_TEAPOT,
+                reason=b'I\'m a tea pot',
+            )
+        return request

skeleton/app/plugins/my_web_plugin.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import logging
+from typing import List, Tuple
+
+from proxy.http.parser import HttpParser
+from proxy.http.server import HttpWebServerBasePlugin, httpProtocolTypes
+from proxy.http.responses import okResponse
+
+
+logger = logging.getLogger(__name__)
+
+
+class MyWebServerPlugin(HttpWebServerBasePlugin):
+    """Demonstrates inbuilt web server routing using plugin."""
+
+    def routes(self) -> List[Tuple[int, str]]:
+        return [
+            (httpProtocolTypes.HTTP, r'/$'),
+        ]
+
+    def handle_request(self, request: HttpParser) -> None:
+        self.client.queue(okResponse(content=b'Hello World'))