Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions .github/workflows/headless-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -186,3 +186,80 @@ jobs:
- name: Check test result
if: steps.test-ext.outcome == 'failure'
run: exit 1

chrome-extension-bridge:
runs-on: ubuntu-22.04
env:
MIDSCENE_MODEL_API_KEY: ${{ secrets.MIDSCENE_MODEL_API_KEY }}
MIDSCENE_MODEL_BASE_URL: ${{ vars.MIDSCENE_MODEL_BASE_URL }}
MIDSCENE_MODEL_NAME: ${{ vars.MIDSCENE_MODEL_NAME }}
MIDSCENE_MODEL_FAMILY: ${{ vars.MIDSCENE_MODEL_FAMILY }}
MIDSCENE_MODEL_RETRY_COUNT: "2"
MIDSCENE_MODEL_RETRY_INTERVAL: "60000"
MIDSCENE_REPORT_QUIET: "true"
MIDSCENE_COMPUTER_HEADLESS_LINUX: "true"
CI: "1"
MIDSCENE_OPENAI_INIT_CONFIG_JSON: ${{ secrets.MIDSCENE_OPENAI_INIT_CONFIG_JSON }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
MIDSCENE_USE_QWEN3_VL: "1"

steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.branch || github.ref }}

- name: Setup pnpm
uses: pnpm/action-setup@v2
with:
version: 9.3.0

- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '24.13.0'
cache: 'pnpm'

- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y \
xvfb \
x11-xserver-utils \
imagemagick \
libxtst6 \
libxinerama1 \
libx11-6 \
libxkbcommon-x11-0 \
libpng16-16 \
libnss3 \
libatk-bridge2.0-0 \
libdrm2 \
libgbm1 \
libasound2
wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
sudo dpkg -i google-chrome-stable_current_amd64.deb || sudo apt-get -f install -y

- name: Install dependencies
run: pnpm install --frozen-lockfile

- name: Build project
run: pnpm run build:skip-cache

- name: Run Bridge mode start/stop test
run: AI_TEST_TYPE=computer npx nx test @midscene/computer -- tests/ai/chrome-extension-bridge.test.ts
timeout-minutes: 25
id: test-bridge
continue-on-error: true

- name: Upload bridge test report
if: always()
uses: actions/upload-artifact@v4
with:
name: chrome-extension-bridge-report
path: packages/computer/midscene_run/report
if-no-files-found: ignore

- name: Check test result
if: steps.test-bridge.outcome == 'failure'
run: exit 1
13 changes: 13 additions & 0 deletions apps/chrome-extension/src/extension/bridge/index.less
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,19 @@
align-items: center;
justify-content: center;
}

.bridge-toggle-btn {
margin-left: 8px;
font-size: 13px;
color: rgba(0, 0, 0, 0.65);
display: flex;
align-items: center;
gap: 4px;

&:hover {
color: #1890ff;
}
}
}

@keyframes hue-shift {
Expand Down
59 changes: 54 additions & 5 deletions apps/chrome-extension/src/extension/bridge/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import {
ClearOutlined,
DownOutlined,
LoadingOutlined,
PauseCircleOutlined,
PlayCircleOutlined,
} from '@ant-design/icons';
import { Button, Input, List, Spin } from 'antd';
import dayjs from 'dayjs';
Expand Down Expand Up @@ -223,6 +225,42 @@ export default function Bridge() {
});
};

const isBridgeActive =
bridgeStatus === 'listening' ||
bridgeStatus === 'connected' ||
bridgeStatus === 'disconnected';

const handleToggleBridge = () => {
if (isBridgeActive) {
// Stop bridge
chrome.runtime.sendMessage(
{ type: workerMessageTypes.BRIDGE_STOP },
Comment on lines +236 to +237
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Persist the stopped state before advertising a Stop control

This handler makes Stop look permanent, but BRIDGE_STOP only tears down the current worker instance. The extension is declared as an MV3 background.service_worker (apps/chrome-extension/static/manifest.json:5-17), and worker.ts still unconditionally runs initBackgroundBridge() plus safeSetupKeepalive({ shouldEnable: true }) at module load (apps/chrome-extension/src/scripts/worker.ts:352-379). After any normal service-worker restart, the bridge starts listening again, so users who stopped bridge mode can be silently re-exposed to incoming connections.

Useful? React with 👍 / 👎.

(response) => {
if (response?.success) {
setBridgeStatus('closed');
// Reset connection status message so next session gets a new message group
connectionStatusMessageId.current = null;
}
},
);
} else {
// Start bridge with optional custom server URL
const endpoint =
serverUrl && serverUrl !== DEFAULT_SERVER_URL ? serverUrl : undefined;
chrome.runtime.sendMessage(
{
type: workerMessageTypes.BRIDGE_START,
payload: { serverEndpoint: endpoint },
},
(response) => {
if (response?.success) {
setBridgeStatus(response.status || 'listening');
}
},
);
}
};

// check if scrolled to bottom
const checkIfScrolledToBottom = () => {
if (messageListRef.current) {
Expand Down Expand Up @@ -286,11 +324,10 @@ export default function Bridge() {

let statusIcon;
let statusTip: string;
if (
bridgeStatus === 'listening' ||
bridgeStatus === 'disconnected' ||
bridgeStatus === 'closed'
) {
if (bridgeStatus === 'closed') {
statusIcon = iconForStatus('failed');
statusTip = 'Stopped';
} else if (bridgeStatus === 'listening' || bridgeStatus === 'disconnected') {
statusIcon = (
<Spin
className="status-loading-icon"
Expand Down Expand Up @@ -441,6 +478,18 @@ export default function Bridge() {
<span className="bottom-status-icon">{statusIcon}</span>
<span className="bottom-status-tip">{statusTip}</span>
</div>
<div className="bottom-status-divider" />
<Button
type="text"
size="small"
className="bridge-toggle-btn"
icon={
isBridgeActive ? <PauseCircleOutlined /> : <PlayCircleOutlined />
}
onClick={handleToggleBridge}
>
{isBridgeActive ? 'Stop' : 'Start'}
</Button>
</div>
</div>
</div>
Expand Down
18 changes: 17 additions & 1 deletion apps/chrome-extension/src/scripts/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ self.addEventListener('unhandledrejection', (event) => {

// Background Bridge for MCP connection
const BRIDGE_PERMISSION_KEY = 'midscene_bridge_permission';
const BRIDGE_STOPPED_KEY = 'midscene_bridge_stopped';
let backgroundBridge: BridgeConnector | null = null;
let currentBridgeStatus: BridgeStatus = 'closed';

Expand Down Expand Up @@ -330,6 +331,8 @@ async function startBackgroundBridge(serverEndpoint?: string): Promise<void> {
if (backgroundBridge) {
await backgroundBridge.disconnect();
}
// Clear the user-stopped flag so service worker restarts will auto-connect
chrome.storage.local.remove(BRIDGE_STOPPED_KEY).catch(() => {});
backgroundBridge = createBackgroundBridge(serverEndpoint);
await backgroundBridge.connect();
console.log('[BackgroundBridge] Started');
Expand All @@ -341,6 +344,8 @@ async function stopBackgroundBridge(): Promise<void> {
backgroundBridge = null;
currentBridgeStatus = 'closed';
console.log('[BackgroundBridge] Stopped');
// Persist the user's intent to stop so service worker restarts don't auto-connect
chrome.storage.local.set({ [BRIDGE_STOPPED_KEY]: true }).catch(() => {});
// Update keepalive
safeSetupKeepalive({
storageKey: BRIDGE_PERMISSION_KEY,
Expand All @@ -358,6 +363,17 @@ async function initBackgroundBridge(): Promise<void> {
}

try {
// Respect the user's intent: if they explicitly stopped bridge, don't auto-start
const result = await chrome.storage.local.get(BRIDGE_STOPPED_KEY);
if (result[BRIDGE_STOPPED_KEY]) {
console.log(
'[BackgroundBridge] Bridge was stopped by user, skipping auto-start',
);
currentBridgeStatus = 'closed';
broadcastBridgeStatus('closed');
return;
}

console.log('[BackgroundBridge] Auto-starting background bridge...');
await startBackgroundBridge();
} catch (error) {
Expand All @@ -372,7 +388,7 @@ setTimeout(() => initBackgroundBridge(), 0);
// Register alarm listener for keepalive pings
registerAlarmListener();

// Setup keepalive on startup - always enable since we auto-start listening
// Setup keepalive on startup - will be updated after initBackgroundBridge checks stopped state
safeSetupKeepalive({
shouldEnable: true,
storageKey: BRIDGE_PERMISSION_KEY,
Expand Down
Loading
Loading