lots of changes

Author: pirate

.github/workflows/test-parallel.yml

@@ -0,0 +1,105 @@
+name: Parallel Tests
+
+on:
+  pull_request:
+    branches: [dev, main, master]
+  push:
+    branches: [dev]
+
+env:
+  PYTHONIOENCODING: utf-8
+  PYTHONLEGACYWINDOWSSTDIO: utf-8
+  USE_COLOR: False
+
+jobs:
+  discover-tests:
+    name: Discover test files
+    runs-on: ubuntu-22.04
+    outputs:
+      test-files: ${{ steps.set-matrix.outputs.test-files }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Discover test files
+        id: set-matrix
+        run: |
+          plugin_tests=$(find abx_plugins/plugins -path "*/tests/test_*.py" -type f | sort)
+
+          json_array="["
+          first=true
+          for test_file in $plugin_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+="," 
+            fi
+
+            plugin=$(echo $test_file | sed 's|abx_plugins/plugins/\([^/]*\)/.*|\1|')
+            test_name=$(basename $test_file .py | sed 's/^test_//')
+            name="plugin/$plugin/$test_name"
+
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+          done
+          json_array+="]"
+
+          echo "test-files=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $plugin_tests | wc -w) test files"
+          echo "$json_array" | jq '.'
+
+  run-tests:
+    name: ${{ matrix.test.name }}
+    runs-on: ubuntu-22.04
+    needs: discover-tests
+
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
+        python: ["3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
+
+      - name: Install dependencies with uv
+        run: |
+          uv pip install -e ".[dev]"
+
+      - name: Run test - ${{ matrix.test.name }}
+        run: |
+          uv run pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out

.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+
+data/
+*.sqlite3*
+__pycache__/
+*.pyc

README.md

@@ -16,3 +16,71 @@ plugins_dir = get_plugins_dir()
 
 Tools like `abx-dl` and ArchiveBox can discover plugins from this package
 without symlinks or environment-variable tricks.
+
+## Plugin Contract
+
+### Directory layout
+
+Each plugin lives under `plugins/<name>/` and may include:
+
+- `config.json` (optional) - config schema
+- `binaries.jsonl` (optional) - binary manifests
+- `on_*` hook scripts (required to do work)
+
+Hooks run with:
+
+- **SNAP_DIR** = base snapshot directory (default: `.`)
+- **CRAWL_DIR** = base crawl directory (default: `.`)
+- **Snapshot hook output** = `SNAP_DIR/<plugin>/...`
+- **Crawl hook output** = `CRAWL_DIR/<plugin>/...`
+- **Other plugin outputs** can be read via `../<other-plugin>/...` from your own output dir
+
+### Key environment variables
+
+- `SNAP_DIR` - base snapshot directory (default: `.`)
+- `CRAWL_DIR` - base crawl directory (default: `.`)
+- `LIB_DIR` - binaries/tools root (default: `~/.config/abx/lib`)
+- `PERSONAS_DIR` - persona profiles root (default: `~/.config/abx/personas`)
+- `ACTIVE_PERSONA` - persona name (default: `Default`)
+
+### Event JSONL interface (bbus-style, no dependency)
+
+Hooks emit JSONL events to stdout. They do **not** need to import `bbus`.
+The event envelope matches the bbus style so higher layers can stream/replay.
+
+Minimal envelope:
+
+```json
+{
+  "event_id": "uuidv7",
+  "event_type": "SnapshotCreated",
+  "event_created_at": "2026-02-01T20:10:22Z",
+  "event_parent_id": "uuidv7-or-null",
+  "event_schema": "abx.events.v1",
+  "event_path": "abx-plugins",
+  "data": { "...": "event-specific fields" }
+}
+```
+
+Conventions:
+
+- Active verb names are **requests** (e.g. `BinaryInstall`, `ProcessLaunch`).
+- Past tense names are **facts** (e.g. `BinaryInstalled`, `ProcessExited`).
+- Plugins can emit additional fields inside `data` without coordination.
+
+Common event types emitted by hooks:
+
+- `ArchiveResultCreated` (status + output files)
+- `Binary` records (dependency detection/install)
+- `ProcessStarted` / `ProcessExited`
+
+Higher-level tools (abx-dl / ArchiveBox) can:
+
+- Parse these events from stdout
+- Persist or project them (SQLite/JSONL/Django) without plugins knowing
+
+Legacy note:
+
+Some hooks still emit a lightweight JSONL record with a top-level `type` field
+(e.g., `{"type": "ArchiveResult", ...}`). Runtimes should accept those and
+optionally translate them into the event envelope above.

abx_plugins/plugins/__init__.py

@@ -0,0 +1 @@
+"""Plugin suite root package for abx-plugins."""

abx_plugins/plugins/accessibility/__init__.py

@@ -23,7 +23,13 @@ const puppeteer = require('puppeteer-core');
 
 // Extractor metadata
 const PLUGIN_NAME = 'accessibility';
-const OUTPUT_DIR = '.';
+const PLUGIN_DIR = path.basename(__dirname);
+const SNAP_DIR = path.resolve((process.env.SNAP_DIR || '.').trim());
+const OUTPUT_DIR = path.join(SNAP_DIR, PLUGIN_DIR);
+if (!fs.existsSync(OUTPUT_DIR)) {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+}
+process.chdir(OUTPUT_DIR);
 const OUTPUT_FILE = 'accessibility.json';
 const CHROME_SESSION_DIR = '../chrome';
 const CHROME_SESSION_REQUIRED_ERROR = 'No Chrome session found (chrome plugin must run first)';

abx_plugins/plugins/accessibility/on_Snapshot__39_accessibility.js

@@ -8,20 +8,17 @@
 import json
 import shutil
 import subprocess
-import sys
 import tempfile
 from pathlib import Path
 
 import pytest
-from django.test import TestCase
 
-# Import chrome test helpers
-sys.path.insert(0, str(Path(__file__).parent.parent.parent / 'chrome' / 'tests'))
-from chrome_test_helpers import (
+from abx_plugins.plugins.chrome.tests.chrome_test_helpers import (
     chrome_session,
     get_test_env,
     get_plugin_dir,
     get_hook_script,
+    chrome_test_url,
 )
 
 
@@ -38,29 +35,31 @@ def chrome_available() -> bool:
 ACCESSIBILITY_HOOK = get_hook_script(PLUGIN_DIR, 'on_Snapshot__*_accessibility.*')
 
 
-class TestAccessibilityPlugin(TestCase):
+class TestAccessibilityPlugin:
     """Test the accessibility plugin."""
 
     def test_accessibility_hook_exists(self):
         """Accessibility hook script should exist."""
-        self.assertIsNotNone(ACCESSIBILITY_HOOK, "Accessibility hook not found in plugin directory")
-        self.assertTrue(ACCESSIBILITY_HOOK.exists(), f"Hook not found: {ACCESSIBILITY_HOOK}")
+        assert ACCESSIBILITY_HOOK is not None, "Accessibility hook not found in plugin directory"
+        assert ACCESSIBILITY_HOOK.exists(), f"Hook not found: {ACCESSIBILITY_HOOK}"
 
 
-class TestAccessibilityWithChrome(TestCase):
+class TestAccessibilityWithChrome:
     """Integration tests for accessibility plugin with Chrome."""
 
-    def setUp(self):
+    def setup_method(self, _method=None):
         """Set up test environment."""
         self.temp_dir = Path(tempfile.mkdtemp())
+        self.snap_dir = self.temp_dir / 'snap'
+        self.snap_dir.mkdir(parents=True, exist_ok=True)
 
-    def tearDown(self):
+    def teardown_method(self, _method=None):
         """Clean up."""
         shutil.rmtree(self.temp_dir, ignore_errors=True)
 
-    def test_accessibility_extracts_page_outline(self):
+    def test_accessibility_extracts_page_outline(self, chrome_test_url):
         """Accessibility hook should extract headings and accessibility tree."""
-        test_url = 'https://example.com'
+        test_url = chrome_test_url
         snapshot_id = 'test-accessibility-snapshot'
 
         try:
@@ -85,7 +84,7 @@ def test_accessibility_extracts_page_outline(self):
                 )
 
                 # Check for output file
-                accessibility_output = snapshot_chrome_dir / 'accessibility.json'
+                accessibility_output = Path(env['SNAP_DIR']) / 'accessibility' / 'accessibility.json'
 
                 accessibility_data = None
 
@@ -98,25 +97,25 @@ def test_accessibility_extracts_page_outline(self):
                             pass
 
                 # Verify hook ran successfully
-                self.assertEqual(result.returncode, 0, f"Hook failed: {result.stderr}")
-                self.assertNotIn('Traceback', result.stderr)
+                assert result.returncode == 0, f"Hook failed: {result.stderr}"
+                assert 'Traceback' not in result.stderr
 
                 # example.com has headings, so we should get accessibility data
-                self.assertIsNotNone(accessibility_data, "No accessibility data was generated")
+                assert accessibility_data is not None, "No accessibility data was generated"
 
                 # Verify we got page outline data
-                self.assertIn('headings', accessibility_data, f"Missing headings: {accessibility_data}")
-                self.assertIn('url', accessibility_data, f"Missing url: {accessibility_data}")
+                assert 'headings' in accessibility_data, f"Missing headings: {accessibility_data}"
+                assert 'url' in accessibility_data, f"Missing url: {accessibility_data}"
 
         except RuntimeError:
             raise
 
-    def test_accessibility_disabled_skips(self):
+    def test_accessibility_disabled_skips(self, chrome_test_url):
         """Test that ACCESSIBILITY_ENABLED=False skips without error."""
-        test_url = 'https://example.com'
+        test_url = chrome_test_url
         snapshot_id = 'test-disabled'
 
-        env = get_test_env()
+        env = get_test_env() | {'SNAP_DIR': str(self.snap_dir)}
         env['ACCESSIBILITY_ENABLED'] = 'False'
 
         result = subprocess.run(
@@ -129,11 +128,11 @@ def test_accessibility_disabled_skips(self):
         )
 
         # Should exit 0 even when disabled
-        self.assertEqual(result.returncode, 0, f"Should succeed when disabled: {result.stderr}")
+        assert result.returncode == 0, f"Should succeed when disabled: {result.stderr}"
 
         # Should NOT create output file when disabled
-        accessibility_output = self.temp_dir / 'accessibility.json'
-        self.assertFalse(accessibility_output.exists(), "Should not create file when disabled")
+        accessibility_output = self.snap_dir / 'accessibility' / 'accessibility.json'
+        assert not accessibility_output.exists(), "Should not create file when disabled"
 
     def test_accessibility_missing_url_argument(self):
         """Test that missing --url argument causes error."""
@@ -145,31 +144,31 @@ def test_accessibility_missing_url_argument(self):
             capture_output=True,
             text=True,
             timeout=30,
-            env=get_test_env()
+            env=get_test_env() | {'SNAP_DIR': str(self.snap_dir)}
         )
 
         # Should fail with non-zero exit code
-        self.assertNotEqual(result.returncode, 0, "Should fail when URL missing")
+        assert result.returncode != 0, "Should fail when URL missing"
 
-    def test_accessibility_missing_snapshot_id_argument(self):
+    def test_accessibility_missing_snapshot_id_argument(self, chrome_test_url):
         """Test that missing --snapshot-id argument causes error."""
-        test_url = 'https://example.com'
+        test_url = chrome_test_url
 
         result = subprocess.run(
             ['node', str(ACCESSIBILITY_HOOK), f'--url={test_url}'],
             cwd=str(self.temp_dir),
             capture_output=True,
             text=True,
             timeout=30,
-            env=get_test_env()
+            env=get_test_env() | {'SNAP_DIR': str(self.snap_dir)}
         )
 
         # Should fail with non-zero exit code
-        self.assertNotEqual(result.returncode, 0, "Should fail when snapshot-id missing")
+        assert result.returncode != 0, "Should fail when snapshot-id missing"
 
-    def test_accessibility_with_no_chrome_session(self):
+    def test_accessibility_with_no_chrome_session(self, chrome_test_url):
         """Test that hook fails gracefully when no Chrome session exists."""
-        test_url = 'https://example.com'
+        test_url = chrome_test_url
         snapshot_id = 'test-no-chrome'
 
         result = subprocess.run(
@@ -182,13 +181,12 @@ def test_accessibility_with_no_chrome_session(self):
         )
 
         # Should fail when no Chrome session
-        self.assertNotEqual(result.returncode, 0, "Should fail when no Chrome session exists")
+        assert result.returncode != 0, "Should fail when no Chrome session exists"
         # Error should mention CDP or Chrome
         err_lower = result.stderr.lower()
-        self.assertTrue(
-            any(x in err_lower for x in ['chrome', 'cdp', 'cannot find', 'puppeteer']),
-            f"Should mention Chrome/CDP in error: {result.stderr}"
-        )
+        assert any(
+            x in err_lower for x in ['chrome', 'cdp', 'cannot find', 'puppeteer']
+        ), f"Should mention Chrome/CDP in error: {result.stderr}"
 
 
 if __name__ == '__main__':