Add support for ChromeDriver, so we can configure mobile emulation

Author: delan

Cargo.lock

@@ -18,6 +18,7 @@ serde_with = "3.9.0"
 toml = "0.8.19"
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+webdriver_client = { version = "0.2.5", git = "https://github.com/delan/webdriver_client_rust.git", branch = "bump-log-to-0.4" }
 
 [profile.release]
 debug = true

Cargo.toml

@@ -0,0 +1,42 @@
+{ lib
+, stdenv
+, fetchurl
+, unzip
+, autoPatchelfHook
+, glib
+, nss
+, xorg
+}:
+
+stdenv.mkDerivation rec {
+  pname = "chromedriver";
+  version = "130.0.6723.91";
+
+  src = let
+    json = builtins.fromJSON (builtins.readFile (fetchurl {
+      url = "https://googlechromelabs.github.io/chrome-for-testing/${version}.json";
+      hash = "sha256-LS6n73mzL46AQtv7FvCRguGf090NyaPvotKxUueOIj0=";
+    }));
+    url = (lib.findFirst (d: d.platform == "linux64") null json.downloads.chromedriver).url;
+  in fetchurl {
+    url = url;
+    hash = "sha256-qMlM6ilsIqm8G5KLE4uGVb/s2bNyZSyQmxsq+EHKX/c=";
+  };
+
+  nativeBuildInputs = [
+    unzip
+    autoPatchelfHook
+  ];
+
+  buildInputs = [
+    glib
+    nss
+    xorg.libxcb
+  ];
+
+  sourceRoot = ".";
+
+  installPhase = ''
+    install -m755 -D chromedriver-linux64/chromedriver $out/bin/chromedriver
+  '';
+}

chromedriver.nix

@@ -0,0 +1,2 @@
+#!/bin/sh
+command -v "$1"

query-path.sh

@@ -0,0 +1,6 @@
+{ pkgs ? import (fetchTarball { url = "https://github.com/NixOS/nixpkgs/archive/dc460ec76cbff0e66e269457d7b728432263166c.tar.gz"; }) {} }:
+pkgs.mkShell {
+  buildInputs = [
+    (pkgs.callPackage ./chromedriver.nix {})
+  ];
+}

shell.nix

@@ -54,7 +54,7 @@ fn analyse_sample(
                 }
             }
         }
-        Engine::Chromium { .. } => {
+        Engine::Chromium { .. } | Engine::ChromeDriver { .. } => {
             let mut json_paths = vec![];
             let mut convert_jobs = vec![];
             for entry in std::fs::read_dir(&sample_dir)? {
@@ -99,7 +99,9 @@ fn analyse_sample(
 
     let summaries = match engine.engine {
         Engine::Servo { .. } => crate::servo::compute_summaries(args)?,
-        Engine::Chromium { .. } => crate::chromium::compute_summaries(args)?,
+        Engine::Chromium { .. } | Engine::ChromeDriver { .. } => {
+            crate::chromium::compute_summaries(args)?
+        }
     };
 
     File::create(sample_dir.join("summaries.json"))?.write_all(summaries.json().as_bytes())?;

src/analyse.rs

@@ -1,11 +1,20 @@
-use std::{fs::create_dir_all, path::Path, process::Command};
+use core::str;
+use std::{
+    collections::BTreeMap,
+    fs::{copy, create_dir_all, read_dir, File},
+    path::Path,
+    process::Command,
+    thread::sleep,
+};
 
 use jane_eyre::eyre::{self, bail, eyre, OptionExt};
-use tracing::info;
+use serde_json::json;
+use tracing::{debug, info};
+use webdriver_client::{chrome::ChromeDriver, messages::NewSessionCmd, Driver, LocationStrategy};
 
 use crate::{
     shell::SHELL,
-    study::{KeyedCpuConfig, KeyedEngine, KeyedSite, Study},
+    study::{Engine, KeyedCpuConfig, KeyedEngine, KeyedSite, Study},
 };
 
 pub fn main(args: Vec<String>) -> eyre::Result<()> {
@@ -60,6 +69,132 @@ fn create_sample(
         return Ok(());
     }
 
+    if let Engine::ChromeDriver { path } = engine.engine {
+        // Resolve path against PATH if needed. ChromeDriver or WebDriver seems to need this.
+        let query = SHELL
+            .lock()
+            .map_err(|e| eyre!("Mutex poisoned: {e:?}"))?
+            .run(include_str!("../query-path.sh"), [path])?
+            .output()?;
+        if !query.status.success() {
+            bail!("Process failed: {}", query.status);
+        }
+        let path = str::from_utf8(&query.stdout)?
+            .strip_suffix("\n")
+            .ok_or_eyre("Output has no trailing newline")?;
+
+        for i in 1..=study.sample_size {
+            info!("Starting ChromeDriver");
+            let driver =
+                ChromeDriver::spawn().map_err(|e| eyre!("Failed to spawn ChromeDriver: {e}"))?;
+
+            // Configure the browser with WebDriver capabilities. Note that ChromeDriver takes care
+            // of running Chromium with a clean profile (much like `--user-data-dir=$(mktemp -d)`)
+            // and in a way amenable to automation (e.g. `--no-first-run`).
+            // <https://www.w3.org/TR/webdriver/#capabilities>
+            // <https://developer.chrome.com/docs/chromedriver/capabilities>
+            let mut params = NewSessionCmd::default();
+            // Do not wait for page load to complete.
+            params.always_match("pageLoadStrategy", json!("none"));
+            // Allow the use of mitmproxy replay (see ../start-mitmproxy.sh).
+            params.always_match("acceptInsecureCerts", json!(true));
+
+            let mut mobile_emulation = BTreeMap::default();
+            if let Some(user_agent) = site.user_agent {
+                // ChromeDriver does not support the standard `userAgent` capability, which goes in
+                // the top level. Use `.goog:chromeOptions.mobileEmulation.userAgent` instead.
+                mobile_emulation.insert("userAgent", json!(user_agent));
+            }
+            if let Some((width, height)) = site.screen_size()? {
+                mobile_emulation
+                    .insert("deviceMetrics", json!({ "width": width, "height": height }));
+            }
+
+            let pftrace_temp_dir = mktemp::Temp::new_dir()?;
+            let attempted_pftrace_temp_path = pftrace_temp_dir.join("chrome.pftrace");
+            let attempted_pftrace_temp_path = attempted_pftrace_temp_path
+                .to_str()
+                .ok_or_eyre("Unsupported path")?;
+            let mut args = vec![
+                "--trace-startup".to_owned(),
+                format!("--trace-startup-file={attempted_pftrace_temp_path}"),
+            ];
+            args.extend(site.extra_engine_arguments(engine.key).to_owned());
+            params.always_match(
+                "goog:chromeOptions",
+                json!({
+                    // <https://developer.chrome.com/docs/chromedriver/capabilities>
+                    "mobileEmulation": mobile_emulation,
+                    "binary": path,
+                    "args": args,
+                }),
+            );
+
+            info!("Starting Chromium");
+            let session = driver.session(&params)?;
+
+            info!(site.url, "Navigating to site");
+            session.go(site.url)?;
+
+            info!(?site.browser_open_time, "Waiting for fixed amount of time");
+            sleep(site.browser_open_time);
+
+            info!(wait_for_selectors = ?site.wait_for_selectors().collect::<Vec<_>>(), "Checking for elements");
+            #[derive(Debug)]
+            struct ElementCounts {
+                expected: usize,
+                actual: usize,
+            }
+            let element_counts = site
+                .wait_for_selectors()
+                .map(
+                    |(selector, expected)| -> eyre::Result<(&String, ElementCounts)> {
+                        Ok((
+                            selector,
+                            ElementCounts {
+                                expected: *expected,
+                                actual: session
+                                    .find_elements(selector, LocationStrategy::Css)?
+                                    .len(),
+                            },
+                        ))
+                    },
+                )
+                .collect::<eyre::Result<BTreeMap<&String, ElementCounts>>>()?;
+            debug!(?element_counts, "Found elements");
+            for (selector, ElementCounts { expected, actual }) in element_counts {
+                assert_eq!(expected, actual, "Condition failed: wait_for_selectors.{selector:?}: expected {expected}, actual {actual}");
+            }
+
+            // When using ChromeDriver, for some reason, Chromium fails to rename the Perfetto trace
+            // to `--trace-startup-file`. Kill ChromeDriver and rename it ourselves.
+            drop(session);
+            let pftrace_path = sample_dir.join(format!(
+                "chrome{:0width$}.pftrace",
+                i,
+                width = study.sample_size.to_string().len()
+            ));
+            let pftrace_path = pftrace_path.to_str().ok_or_eyre("Unsupported path")?;
+            for entry in read_dir(&pftrace_temp_dir)? {
+                let pftrace_temp_path = entry?.path();
+                info!(
+                    ?pftrace_temp_path,
+                    ?pftrace_path,
+                    "Copying Perfetto trace to sample directory"
+                );
+                copy(pftrace_temp_path, pftrace_path)?;
+            }
+
+            // Extend the lifetime of `pftrace_temp_dir` to avoid premature deletion.
+            drop(pftrace_temp_dir);
+        }
+
+        info!("Marking sample as done");
+        File::create_new(sample_dir.join("done"))?;
+
+        return Ok(());
+    }
+
     let sample_dir = sample_dir.to_str().ok_or_eyre("Bad sample path")?;
     info!("Creating sample");
     let mut args = vec![

src/collect.rs

@@ -65,7 +65,12 @@ pub fn main(args: Vec<String>) -> eyre::Result<()> {
     // If there were any Chromium results, print sections for real Chromium events.
     if study
         .engines()
-        .find(|engine| matches!(engine.engine, Engine::Chromium { .. }))
+        .find(|engine| {
+            matches!(
+                engine.engine,
+                Engine::Chromium { .. } | Engine::ChromeDriver { .. },
+            )
+        })
         .is_some()
     {
         for summary_key in REAL_CHROMIUM_EVENTS.split(" ") {

src/report.rs

@@ -1,6 +1,6 @@
 use std::{collections::BTreeMap, fs::File, io::Read, path::Path, time::Duration};
 
-use jane_eyre::eyre;
+use jane_eyre::eyre::{self, bail};
 use serde::Deserialize;
 
 #[derive(Debug, Deserialize)]
@@ -29,6 +29,9 @@ enum Site {
     Full {
         url: String,
         browser_open_time: Option<u64>,
+        user_agent: Option<String>,
+        screen_size: Option<Vec<usize>>,
+        wait_for_selectors: Option<BTreeMap<String, usize>>,
         extra_engine_arguments: Option<BTreeMap<String, Vec<String>>>,
     },
 }
@@ -37,6 +40,9 @@ pub struct KeyedSite<'study> {
     pub key: &'study str,
     pub url: &'study str,
     pub browser_open_time: Duration,
+    pub user_agent: Option<&'study str>,
+    screen_size: Option<&'study [usize]>,
+    wait_for_selectors: Option<&'study BTreeMap<String, usize>>,
     extra_engine_arguments: Option<&'study BTreeMap<String, Vec<String>>>,
 }
 
@@ -45,6 +51,7 @@ pub struct KeyedSite<'study> {
 pub enum Engine {
     Servo { path: String },
     Chromium { path: String },
+    ChromeDriver { path: String },
 }
 #[derive(Clone, Copy, Debug)]
 pub struct KeyedEngine<'study> {
@@ -90,24 +97,52 @@ impl<'study> From<(&'study str, &'study Site)> for KeyedSite<'study> {
                 key,
                 url,
                 browser_open_time: default_browser_open_time,
+                user_agent: None,
+                screen_size: None,
+                wait_for_selectors: None,
                 extra_engine_arguments: None,
             },
             Site::Full {
                 url,
-                extra_engine_arguments,
                 browser_open_time,
+                user_agent,
+                screen_size,
+                wait_for_selectors,
+                extra_engine_arguments,
             } => Self {
                 key,
                 url,
                 browser_open_time: browser_open_time
                     .map_or(default_browser_open_time, Duration::from_secs),
+                user_agent: user_agent.as_deref(),
+                screen_size: screen_size.as_deref(),
+                wait_for_selectors: wait_for_selectors.as_ref(),
                 extra_engine_arguments: extra_engine_arguments.as_ref(),
             },
         }
     }
 }
 
 impl KeyedSite<'_> {
+    pub fn screen_size(&self) -> eyre::Result<Option<(usize, usize)>> {
+        self.screen_size
+            .map(|size| {
+                Ok(match size {
+                    [width, height] => (*width, *height),
+                    other => bail!("Bad screen_size: {other:?}"),
+                })
+            })
+            .transpose()
+    }
+
+    pub fn wait_for_selectors(&self) -> Box<dyn Iterator<Item = (&String, &usize)> + '_> {
+        if let Some(wait_for_selectors) = self.wait_for_selectors {
+            Box::new(wait_for_selectors.iter())
+        } else {
+            Box::new([].into_iter())
+        }
+    }
+
     pub fn extra_engine_arguments(&self, engine_key: &str) -> &[String] {
         self.extra_engine_arguments
             .and_then(|map| map.get(engine_key))
@@ -120,13 +155,17 @@ impl KeyedEngine<'_> {
         match self.engine {
             Engine::Servo { .. } => include_str!("../benchmark-servo.sh"),
             Engine::Chromium { .. } => include_str!("../benchmark-chromium.sh"),
+            Engine::ChromeDriver { .. } => {
+                panic!("BUG: Engine::ChromeDriver has no benchmark runner script")
+            }
         }
     }
 
     pub fn browser_path(&self) -> &str {
         match self.engine {
             Engine::Servo { path } => path,
             Engine::Chromium { path } => path,
+            Engine::ChromeDriver { path } => path,
         }
     }
 }

src/study.rs

@@ -27,18 +27,38 @@ isolate_cpu_command = ["sudo", "../../isolate-cpu-for-shell.sh"]  # on Linux
 # Sites can also have other settings, in the full table format.
 # - `url` has the same meaning as the string value above
 # - `browser_open_time` (optional) is in seconds
+# - `user_agent` (optional) overrides the browser’s default user agent
+#   - Currently supported for `ChromeDriver`-type engines only
+#   - For `Servo`-type engines, use `extra_engine_arguments.engine = ["--user-agent", "Android"]`
+#   - For `Chromium`-type engines, use `extra_engine_arguments.engine = ["--user-agent=Android"]`
+# - `screen_size` (optional) overrides the browser’s reported screen size (not the viewport size!)
+#   - Currently supported for `ChromeDriver`-type engines only
+#   - For `Servo`-type engines, use `extra_engine_arguments.engine = ["--screen-size", "320x568"]`
+#   - For `Chromium`-type engines, there is no way to do this
+# - `wait_for_selectors` (optional) is a map from CSS selectors to expected element counts
+#   - Currently supported for `ChromeDriver`-type engines only
+#   - For `Servo`-type engines, there is no way to do this
+#   - For `Chromium`-type engines, there is no way to do this
 # - `extra_engine_arguments` (optional) is keyed on the engine key
 # [sites."example.com"]
 # url = "http://example.com/"
 # browser_open_time = 20
+# user_agent = "Android"
+# screen_size = [320,568]
+# wait_for_selectors."nav a" = 3
 # extra_engine_arguments.servo1 = ["--pref", "dom.svg.enabled"]
 # extra_engine_arguments.servo2 = ["--pref", "dom.svg.enabled"]
 
 # Define your engines here.
 # - Syntax is `key = { type = "Servo|Chromium", path = "/path/to/browser" }`
 # - Dots in the key must be quoted
+# - `type` is one of the following:
+#   - `Servo` uses benchmark-servo.sh
+#   - `Chromium` uses benchmark-chromium.sh
+#   - `ChromeDriver` uses ChromeDriver, a WebDriver-based approach
 # - If `path` has no slashes, it represents a command in your PATH
 [engines]
 "servo1" = { type = "Servo", path = "/path/to/servo1/servo" }
 "servo2" = { type = "Servo", path = "/path/to/servo2/servo" }
-"chromium" = { type = "Chromium", path = "google-chrome-stable" }
+"chromium1" = { type = "Chromium", path = "google-chrome-stable" }
+"chromium2" = { type = "ChromeDriver", path = "google-chrome-stable" }