Merge pull request #102 from cryoem-uoft/release-v4.6

Release v4.6.1

Author: nfrasser

.github/workflows/python-package.yml

@@ -77,7 +77,7 @@ jobs:
       fail-fast: true
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.7", "3.12"]
+        python-version: ["3.7", "3.13"]
         exclude:
           # Latest macos runner does not support older Python versions
           # https://github.com/actions/setup-python/issues/852
@@ -125,7 +125,7 @@ jobs:
         uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64
-      - uses: pypa/cibuildwheel@v2.20.0
+      - uses: pypa/cibuildwheel@v2.21.3
         env:
           CIBW_SKIP: cp36-* pp*-win* pp*-macosx* *_i686
           CIBW_TEST_SKIP: "*-win_arm64"

.pre-commit-config.yaml

@@ -2,15 +2,15 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-toml
       - id: check-yaml
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.4
+    rev: v0.7.0
     hooks:
       - id: ruff
         name: ruff check
@@ -24,7 +24,7 @@ repos:
         name: ruff format
         alias: format
   - repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.379
+    rev: v1.1.386
     hooks:
       - id: pyright
-        additional_dependencies: [cython, httpretty, numpy, pytest]
+        additional_dependencies: [cython, httpretty, numpy, pytest, setuptools]

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v4.6.1
+
+- Added: Python 3.13 support
+- Fix: Disallow non-URL-encodable characters when creating external job inputs and outputs
+- Fix: Prevent queuing and killing external jobs (must use `job.start/job.stop()` or `with job.run()`)
+
 ## v4.6.0
 
 - Added: `Dataset.is_equivalent` method to check if two datasets have identical fields, but in a different order

cryosparc/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.6.0"
+__version__ = "4.6.1"
 
 
 def get_include():

cryosparc/dataset.py

@@ -844,7 +844,7 @@ def __init__(
         if allocate is None:  # Same as zero
             populate = [(("uid", "<u8"), n.ndarray(0, dtype=n.uint64))]
         elif isinstance(allocate, (int, n.integer)):
-            populate = [(("uid", "<u8"), generate_uids(allocate or 0))]
+            populate = [(("uid", "<u8"), generate_uids(int(allocate)))]
         elif isinstance(allocate, n.ndarray):  # record array
             for field in allocate.dtype.descr:
                 assert field[0], f"Cannot initialize with record array of dtype {allocate.dtype}"

cryosparc/errors.py

@@ -78,3 +78,11 @@ def __init__(self, caller: str, validation: SlotsValidation):
         )
 
         return super().__init__(msg)
+
+
+class ExternalJobError(Exception):
+    """
+    Raised during external job lifecycle failures
+    """
+
+    pass

cryosparc/job.py

@@ -4,6 +4,8 @@
 
 import json
 import math
+import re
+import urllib.parse
 from contextlib import contextmanager
 from io import BytesIO
 from pathlib import PurePath, PurePosixPath
@@ -15,7 +17,7 @@
 
 from .command import CommandError, make_json_request, make_request
 from .dataset import DEFAULT_FORMAT, Dataset
-from .errors import InvalidSlotsError
+from .errors import ExternalJobError, InvalidSlotsError
 from .spec import (
     ASSET_CONTENT_TYPES,
     IMAGE_CONTENT_TYPES,
@@ -39,6 +41,12 @@
     from .tools import CryoSPARC
 
 
+GROUP_NAME_PATTERN = r"^[A-Za-z][0-9A-Za-z_]*$"
+"""
+Input and output result groups may only contain, letters, numbers and underscores.
+"""
+
+
 class Job(MongoController[JobDocument]):
     """
     Accessor class to a job in CryoSPARC with ability to load inputs and
@@ -1233,6 +1241,11 @@ def add_input(
             ... )
             "input_micrographs"
         """
+        if name and not re.fullmatch(GROUP_NAME_PATTERN, name):
+            raise ValueError(
+                f'Invalid input name "{name}"; may only contain letters, numbers and underscores, '
+                "and must start with a letter"
+            )
         try:
             self.cs.vis.add_external_job_input(  # type: ignore
                 project_uid=self.project_uid,
@@ -1354,6 +1367,11 @@ def add_output(
             ... )
             "particle_alignments"
         """
+        if name and not re.fullmatch(GROUP_NAME_PATTERN, name):
+            raise ValueError(
+                f'Invalid output name "{name}"; may only contain letters, numbers and underscores, '
+                "and must start with a letter"
+            )
         try:
             self.cs.vis.add_external_job_output(  # type: ignore
                 project_uid=self.project_uid,
@@ -1519,7 +1537,8 @@ def save_output(self, name: str, dataset: Dataset, *, refresh: bool = True):
             >>> job.save_output("picked_particles", particles)
 
         """
-        url = f"/external/projects/{self.project_uid}/jobs/{self.uid}/outputs/{name}/dataset"
+
+        url = f"/external/projects/{self.project_uid}/jobs/{self.uid}/outputs/{urllib.parse.quote_plus(name)}/dataset"
         with make_request(self.cs.vis, url=url, data=dataset.stream(compression="lz4")) as res:
             result = res.read().decode()
             assert res.status >= 200 and res.status < 400, f"Save output failed with message: {result}"
@@ -1572,12 +1591,24 @@ def run(self):
         """
         error = False
         self.start("running")
-        self.refresh()
         try:
             yield self
         except Exception:
             error = True
             raise
         finally:
             self.stop(error)  # TODO: Write Error to job log, if possible
-            self.refresh()
+
+    def queue(
+        self,
+        lane: Optional[str] = None,
+        hostname: Optional[str] = None,
+        gpus: List[int] = [],
+        cluster_vars: Dict[str, Any] = {},
+    ):
+        raise ExternalJobError(
+            "Cannot queue an external job; use `job.start()`/`job.stop()` or `with job.run()` instead"
+        )
+
+    def kill(self):
+        raise ExternalJobError("Cannot kill an external job; use `job.stop()` instead")

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cryosparc-tools"
-version = "4.6.0"
+version = "4.6.1"
 description = "Toolkit for interfacing with CryoSPARC"
 readme = "README.md"
 requires-python = ">=3.7"
@@ -33,6 +33,7 @@ dev = [
     "pytest-xdist[psutil]",
     "pytest",
     "ruff",
+    "setuptools",
 ]
 
 build = ["build", "autodocsumm", "cython", "jupyter-book"]

setup.py

@@ -25,7 +25,7 @@
 
 setup(
     name="cryosparc_tools",
-    version="4.6.0",
+    version="4.6.1",
     description="Toolkit for interfacing with CryoSPARC",
     headers=["cryosparc/include/cryosparc-tools/dataset.h"],
     ext_modules=cythonize(

tests/conftest.py

@@ -98,9 +98,13 @@ def shuffle(self):
 # fmt: on
 
 
-def request_callback_core(request, uri, response_headers):
-    body = json.loads(request.body)
-    procs = {
+@pytest.fixture
+def mock_jsonrpc_procs_core() -> Dict[str, Any]:
+    """
+    Dictionary of JSON RPC method names and their return values. Can override
+    existing values in subfixtures.
+    """
+    return {
         "hello_world": {"hello": "world"},
         "get_running_version": "develop",
         "get_id_by_email_password": "6372a35e821ed2b71d9fe4e3",
@@ -253,17 +257,36 @@ def request_callback_core(request, uri, response_headers):
         "job_connect_group": True,
         "job_set_param": True,
     }
-    procs["system.describe"] = {"procs": [{"name": m} for m in procs]}
-    response_headers["content-type"] = "application/json"
-    return [200, response_headers, json.dumps({"result": procs[body["method"]]})]
 
 
-def request_callback_vis(request, uri, response_headers):
-    body = json.loads(request.body)
-    procs: Dict[str, Any] = {"hello_world": {"hello": "world"}}
-    procs["system.describe"] = {"procs": [{"name": m} for m in procs]}
-    response_headers["content-type"] = "application/json"
-    return [200, response_headers, json.dumps({"result": procs[body["method"]]})]
+@pytest.fixture
+def request_callback_core(mock_jsonrpc_procs_core):
+    def request_callback_core_fn(request, uri, response_headers):
+        body = json.loads(request.body)
+        mock_jsonrpc_procs_core["system.describe"] = {"procs": [{"name": m} for m in mock_jsonrpc_procs_core]}
+        response_headers["content-type"] = "application/json"
+        return [200, response_headers, json.dumps({"result": mock_jsonrpc_procs_core[body["method"]]})]
+
+    return request_callback_core_fn
+
+
+@pytest.fixture
+def mock_jsonrpc_procs_vis() -> Dict[str, Any]:
+    return {
+        "hello_world": {"hello": "world"},
+    }
+
+
+@pytest.fixture
+def request_callback_vis(mock_jsonrpc_procs_vis):
+    def request_callback_vis_fn(request, uri, response_headers):
+        body = json.loads(request.body)
+
+        mock_jsonrpc_procs_vis["system.describe"] = {"procs": [{"name": m} for m in mock_jsonrpc_procs_vis]}
+        response_headers["content-type"] = "application/json"
+        return [200, response_headers, json.dumps({"result": mock_jsonrpc_procs_vis[body["method"]]})]
+
+    return request_callback_vis_fn
 
 
 def request_callback_vis_get_project_file(request, uri, response_headers):
@@ -404,7 +427,7 @@ def t20s_particles_passthrough():
 
 
 @pytest.fixture
-def cs():
+def cs(request_callback_core, request_callback_vis):
     httpretty.enable(verbose=False, allow_net_connect=False)
     httpretty.register_uri(httpretty.POST, "http://localhost:39002/api", body=request_callback_core)  # type: ignore
     httpretty.register_uri(httpretty.POST, "http://localhost:39003/api", body=request_callback_vis)  # type: ignore