Handle SIGINFO. Try kill -s INFO <pid>. (abhinavsingh#1024)

* Handle `SIGINFO`.  Try `kill -s INFO <pid>`.

Also remove dirty hack added in flags to incorporate `--basic-auth`
flag.  Add `__pycache__` to ignore list.  Disable http proxy
during acceptor benchmark.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* not on windows

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* no cover

* # pylint: disable=E1101

* Curl retry on error and check_output 2 minute timeout

* check output timeout None

* Update `faulthandler_timeout` to 2 minutes

* Disable `test_circular_imports`, `isort` integration now works

* Fix curl flags

* Revert back to older flags

* SIGINFO attribute might not even exist

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: abhinavsingh

.gitignore

@@ -10,6 +10,8 @@
 .tox
 .python-version
 
+__pycache__
+
 coverage.xml
 proxy.py.iml
 

benchmark/_proxy.py

@@ -22,7 +22,7 @@
             backlog=65536,
             open_file_limit=65536,
             enable_web_server=True,
-            disable_proxy_server=False,
+            disable_proxy_server=True,
             num_acceptors=10,
             local_executor=1,
             log_file='/dev/null',

proxy/common/flag.py

@@ -98,17 +98,13 @@ def initialize(
             print(PY2_DEPRECATION_MESSAGE)
             sys.exit(1)
 
-        # Dirty hack to always discover --basic-auth flag
-        # defined by proxy auth plugin.
-        in_args = input_args + ['--plugin', PLUGIN_PROXY_AUTH]
-
         # Discover flags from requested plugin.
         # This will also surface external plugin flags
         # under --help.
-        Plugins.discover(in_args)
+        Plugins.discover(input_args)
 
         # Parse flags
-        args = flags.parse_args(in_args)
+        args = flags.parse_args(input_args)
 
         # Print version and exit
         if args.version:

proxy/http/proxy/auth.py

@@ -18,18 +18,7 @@
 from ...http import httpHeaders
 from ..exception import ProxyAuthenticationFailed
 from ...http.proxy import HttpProxyBasePlugin
-from ...common.flag import flags
 from ...http.parser import HttpParser
-from ...common.constants import DEFAULT_BASIC_AUTH
-
-
-flags.add_argument(
-    '--basic-auth',
-    type=str,
-    default=DEFAULT_BASIC_AUTH,
-    help='Default: No authentication. Specify colon separated user:password '
-    'to enable basic authentication.',
-)
 
 
 class AuthPlugin(HttpProxyBasePlugin):

proxy/proxy.py

@@ -11,6 +11,7 @@
 import os
 import sys
 import time
+import pprint
 import signal
 import logging
 from typing import Any, List, Optional
@@ -23,9 +24,9 @@
 from .core.acceptor import Listener, AcceptorPool
 from .common.constants import (
     IS_WINDOWS, DEFAULT_PLUGINS, DEFAULT_VERSION, DEFAULT_LOG_FILE,
-    DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT,
-    DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_ENABLE_DASHBOARD,
-    DEFAULT_ENABLE_SSH_TUNNEL,
+    DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_BASIC_AUTH,
+    DEFAULT_LOG_FORMAT, DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT,
+    DEFAULT_ENABLE_DASHBOARD, DEFAULT_ENABLE_SSH_TUNNEL,
 )
 
 
@@ -40,7 +41,6 @@
     help='Prints proxy.py version.',
 )
 
-# TODO: Convert me into 1-letter choices
 # TODO: Add --verbose option which also
 # starts to log traffic flowing between
 # clients and upstream servers.
@@ -98,6 +98,16 @@
     help='Default: False.  Enables proxy.py dashboard.',
 )
 
+# NOTE: Same reason as mention above.
+# Ideally this flag belongs to proxy auth plugin.
+flags.add_argument(
+    '--basic-auth',
+    type=str,
+    default=DEFAULT_BASIC_AUTH,
+    help='Default: No authentication. Specify colon separated user:password '
+    'to enable basic authentication.',
+)
+
 flags.add_argument(
     '--enable-ssh-tunnel',
     action='store_true',
@@ -133,6 +143,10 @@ class Proxy:
     Executor pool receives newly accepted work by :class:`~proxy.core.acceptor.Acceptor`
     and creates an instance of work class for processing the received work.
 
+    In ``--threadless`` mode and with ``--local-executor 0``,
+    acceptors will start a companion thread to handle accepted
+    client connections.
+
     Optionally, Proxy class also initializes the EventManager.
     A multi-process safe pubsub system which can be used to build various
     patterns for message sharing and/or signaling.
@@ -156,18 +170,17 @@ def __exit__(self, *args: Any) -> None:
 
     def setup(self) -> None:
         # TODO: Introduce cron feature
-        # https://github.com/abhinavsingh/proxy.py/issues/392
+        # https://github.com/abhinavsingh/proxy.py/discussions/808
+        #
+        # TODO: Introduce ability to change flags dynamically
+        # https://github.com/abhinavsingh/proxy.py/discussions/1020
         #
-        # TODO: Introduce ability to publish
-        # adhoc events which can modify behaviour of server
-        # at runtime.  Example, updating flags, plugin
-        # configuration etc.
+        # TODO: Python shell within running proxy.py environment
+        # https://github.com/abhinavsingh/proxy.py/discussions/1021
         #
-        # TODO: Python shell within running proxy.py environment?
+        # TODO: Near realtime resource / stats monitoring
+        # https://github.com/abhinavsingh/proxy.py/discussions/1023
         #
-        # TODO: Pid watcher which watches for processes started
-        # by proxy.py core.  May be alert or restart those processes
-        # on failure.
         self._write_pid_file()
         # We setup listeners first because of flags.port override
         # in case of ephemeral port being used
@@ -263,10 +276,15 @@ def _delete_port_file(self) -> None:
             os.remove(self.flags.port_file)
 
     def _register_signals(self) -> None:
-        # TODO: Handle SIGINFO, SIGUSR1, SIGUSR2
+        # TODO: Define SIGUSR1, SIGUSR2
         signal.signal(signal.SIGINT, self._handle_exit_signal)
         signal.signal(signal.SIGTERM, self._handle_exit_signal)
         if not IS_WINDOWS:
+            if hasattr(signal, 'SIGINFO'):
+                signal.signal(      # pragma: no cover
+                    signal.SIGINFO,       # pylint: disable=E1101
+                    self._handle_siginfo,
+                )
             signal.signal(signal.SIGHUP, self._handle_exit_signal)
             # TODO: SIGQUIT is ideally meant to terminate with core dumps
             signal.signal(signal.SIGQUIT, self._handle_exit_signal)
@@ -276,6 +294,9 @@ def _handle_exit_signal(signum: int, _frame: Any) -> None:
         logger.info('Received signal %d' % signum)
         sys.exit(0)
 
+    def _handle_siginfo(self, _signum: int, _frame: Any) -> None:
+        pprint.pprint(self.flags.__dict__)  # pragma: no cover
+
 
 def sleep_loop() -> None:
     while True:

pytest.ini

@@ -51,7 +51,7 @@ doctest_optionflags = ALLOW_UNICODE ELLIPSIS
 # Marks tests with an empty parameterset as xfail(run=False)
 empty_parameter_set_mark = xfail
 
-faulthandler_timeout = 30
+faulthandler_timeout = 120
 
 filterwarnings =
   error

tests/integration/test_integration.py

@@ -23,7 +23,7 @@
 from proxy.common.constants import IS_WINDOWS
 
 
-def check_output(args: List[Any]) -> bytes:
+def check_output(args: List[Any]) -> bytes:     # pragma: no cover
     args = args if not IS_WINDOWS else ['powershell'] + args
     return _check_output(args)
 

tests/integration/test_integration.sh

@@ -23,6 +23,8 @@ if [[ -z "$PROXY_PY_PORT" ]]; then
   exit 1
 fi
 
+CURL="curl -v --connect-timeout 20 --max-time 120 --retry-connrefused --retry-delay 5 --retry 3"
+
 PROXY_URL="http://localhost:$PROXY_PY_PORT"
 TEST_URL="$PROXY_URL/http-route-example"
 CURL_EXTRA_FLAGS=""
@@ -50,7 +52,7 @@ while true; do
 done
 
 # Wait for http proxy and web server to start
-CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL $TEST_URL"
+CMD="$CURL $CURL_EXTRA_FLAGS -x $PROXY_URL $TEST_URL"
 while true; do
     RESPONSE=$($CMD 2> /dev/null)
     if [[ $? == 0 ]]; then
@@ -90,13 +92,13 @@ Disallow: /deny
 EOM
 
 echo "[Test HTTP Request via Proxy]"
-CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL http://httpbin.org/robots.txt"
+CMD="$CURL $CURL_EXTRA_FLAGS -x $PROXY_URL http://httpbin.org/robots.txt"
 RESPONSE=$($CMD 2> /dev/null)
 verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
 VERIFIED1=$?
 
 echo "[Test HTTPS Request via Proxy]"
-CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL https://httpbin.org/robots.txt"
+CMD="$CURL $CURL_EXTRA_FLAGS -x $PROXY_URL https://httpbin.org/robots.txt"
 RESPONSE=$($CMD 2> /dev/null)
 verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
 VERIFIED2=$?
@@ -105,7 +107,7 @@ if $USE_HTTPS; then
     VERIFIED3=0
 else
     echo "[Test Internal Web Server via Proxy]"
-    curl -v \
+    $CURL \
         $CURL_EXTRA_FLAGS \
         -x $PROXY_URL \
         "$PROXY_URL"
@@ -121,7 +123,7 @@ fi
 echo "[Test Download File Hash Verifies 1]"
 touch downloaded.hash
 echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc  -" > downloaded.hash
-curl -vL \
+$CURL -L \
     $CURL_EXTRA_FLAGS \
     -o downloaded.whl \
     -x $PROXY_URL \
@@ -133,7 +135,7 @@ rm downloaded.whl downloaded.hash
 echo "[Test Download File Hash Verifies 2]"
 touch downloaded.hash
 echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8  -" > downloaded.hash
-curl -vL \
+$CURL -L \
     $CURL_EXTRA_FLAGS \
     -o downloaded.whl \
     -x $PROXY_URL \

tests/test_circular_imports.py

@@ -85,7 +85,15 @@ def _discover_path_importables(
     'import_path',
     _find_all_importables(proxy),
 )
-def test_no_warnings(import_path: str) -> None:
+# Marked as disabled_ because:
+# 1. This test case was added when isort integration was problematic
+# 2. This test case never found a real circular import scenario
+# 3. This test case consumes 60% of test suite runtime
+# 4. Kept in the repo because we might still want to enable
+#    this in future, conditionally.  Example, we can run
+#    this only on a single OS and Python version combination
+#    instead of running it across entire matrix.
+def disabled_test_no_warnings(import_path: str) -> None:
     """Verify that exploding importables doesn't explode.
 
     This is seeking for any import errors including ones caused