Store addons state as JSON (#19564)

Closes #19559

### Summary of the issue:
Addons state is saved in a pickle file.

### Description of user facing changes:
None

### Description of developer facing changes:
The add-ons state file is now in JSON.
The `fromPickledDict` method on `AddonsState` has been renamed to
`fromDict`, but a shim has been introduced.

### Description of development approach:
Switch to using lists instead of sets when populating AddonsState from
dict and serializing to dict, as json.dump doesn't support serializing
sets.

Added methods to read in a pickled add-ons state file and emit a good
JSON representation of that add-ons state. My approach here was to
discard invalid values rather than raising an error, so we have the
greatest chance of success. This does mean that the conversion may be
lossy.

When updating NVDA, convert from pickle to JSON for the system config's
add-ons state file. This is done because this is essentially the only
context in which we should write to the system config directory and
which we know will be executed. User copies perform the migration on
first run by migrating if and only if no addonsState.json exists, but
addonsState.pickle does. User config migration is done by the installed
copy as (a) this is the only time we can guarantee that the user's NVDA
config wil be present; and (b) to minimise the amount of pickles we load
while elevated. In both cases, a backup of the old state file is kept
(as `addonsState.pickle.bak`).

When running on secure desktops, the pickled state will not be read,
even if it is the only addonsState file in the system config. This
should never happen, as the installer is responsible for migrating the
system config's add-ons state file. In other cases, the fallback of
reading the pickled state will attempt to save the JSON state and back
up the pickled state, but this is not required in order for the load to
succede. This is so when running in secure mode or with access to the
user config restricted, add-ons continue to function properly. This does
remove the security benefit of migrating away from pickle, but there's
no other option other than ripping pickle support out entirely and
breaking everyone's add-ons state entirely and having them manually
migrate.

### Testing strategy:
Unit and system tests.

Manual testing as follows:

1. Installed 2025.3.3
2. Installed add-ons (combination of 2026.1 compatible and incompatible,
enabled and disabled) and restarted NVDA
3. Copied user config to system config
4. Upgraded using self-signed launcher
5. Ensured that the installer migrated the system addons state, and the
installed copy migrated the user addons state on first run

Also "upgraded" my 2026.2 alpha copy with this launcher to ensure that
the migration works as expected if updating within the 2026.x series.

### Known issues with pull request:

None known (see notes above for some caviats about the approach though)

Author: SaschaCowley

source/NVDAState.py

@@ -73,9 +73,15 @@ def nvdaConfigFile(self) -> str:
 
 	@property
 	def addonStateFile(self) -> str:
-		from addonHandler import stateFilename
+		from addonHandler import STATE_FILENAME
 
-		return os.path.join(self.configDir, stateFilename)
+		return os.path.join(self.configDir, STATE_FILENAME)
+
+	@property
+	def _oldAddonStateFile(self) -> str:
+		from addonHandler import _OLD_STATE_FILENAME
+
+		return os.path.join(self.configDir, _OLD_STATE_FILENAME)
 
 	@property
 	def profileTriggersFile(self) -> str:

source/addonHandler/__init__.py

@@ -7,7 +7,8 @@
 from __future__ import annotations  # Avoids quoting of forward references
 
 from abc import abstractmethod, ABC
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
+import json
 import sys
 import os.path
 import gettext
@@ -16,9 +17,10 @@
 import collections
 import shutil
 from io import StringIO
-import pickle
 from typing import (
 	IO,
+	Any,
+	Final,
 	Literal,
 	TYPE_CHECKING,
 )
@@ -29,6 +31,7 @@
 from config.registry import ADDON_BUNDLE_EXTENSION
 import languageHandler
 from logHandler import log
+from utils.security import isRunningOnSecureDesktop
 import winBindings.kernel32
 import addonAPIVersion
 import importlib
@@ -69,13 +72,32 @@
 		"NVDA_ADDON_PROG_ID",
 		"config.registry",
 	),
+	MovedSymbol(
+		"stateFilename",
+		"addonHandler",
+		"STATE_FILENAME",
+	),
 )
 
-MANIFEST_FILENAME = "manifest.ini"
-stateFilename = "addonsState.pickle"
-BUNDLE_MIMETYPE = "application/x-nvda-addon"
-ADDON_PENDINGINSTALL_SUFFIX = ".pendingInstall"
-DELETEDIR_SUFFIX = ".delete"
+MANIFEST_FILENAME: Final[str] = "manifest.ini"
+STATE_FILENAME: Final[str] = "addonsState.json"
+_OLD_STATE_FILENAME: Final[str] = "addonsState.pickle"
+# Decoding JSON can potentially consume a large amount of memory.
+# To avoid issues caused by this when reading add-on state,
+# limit the maximum size of the file.
+# The 2026.1 state JSON with no add-ons is 244 bytes.
+# As of 2026-02-24, there are 364 add-ons in the Add-on Store,
+# and the longest ID is 40 UTF-8 bytes (mean ~= median ~= 13 bytes).
+# Even if we take the empty state to be 300 bytes,
+# and assume that all add-on IDs are 100 characters long,
+# And those 100 characters are all 4 bytes wide,
+# and factor in the 4 extra bytes required for the enclosing quotes and ", " separator between IDs,
+# 1MiB is still enough space for a state file containing
+# (1024 * 1024) / (4 + 4 * 100) ~= 2594 add-ons.
+_MAX_STATE_FILESIZE_BYTES = 1024 * 1024  # 1MiB
+BUNDLE_MIMETYPE: Final[str] = "application/x-nvda-addon"
+ADDON_PENDINGINSTALL_SUFFIX: Final[str] = ".pendingInstall"
+DELETEDIR_SUFFIX: Final[str] = ".delete"
 
 
 # Allows add-ons to process additional command line arguments when NVDA starts.
@@ -100,6 +122,10 @@ class AddonsState(collections.UserDict[AddonStateCategory, CaseInsensitiveSet[st
 	Therefore add-on IDs should be treated as case insensitive.
 	"""
 
+	def __init__(self, *args, **kwargs) -> None:
+		super().__init__(*args, **kwargs)
+		self.setDefaultStateValues()
+
 	@staticmethod
 	def _generateDefaultStateContent() -> AddonStateDictT:
 		return {category: CaseInsensitiveSet() for category in AddonStateCategory}
@@ -120,30 +146,41 @@ def setDefaultStateValues(self) -> None:
 		# where the BACK_COMPAT_TO API version was 2023.1.0.
 		self.manualOverridesAPIVersion = MajorMinorPatch(2023, 1, 0)
 
+	def fromDict(
+		self,
+		stateDict: dict[str, Any],
+	) -> None:
+		if "backCompatToAPIVersion" in stateDict:
+			try:
+				self.manualOverridesAPIVersion = MajorMinorPatch(
+					*(int(num) for num in stateDict["backCompatToAPIVersion"]),
+				)
+			except Exception:
+				log.error("Unable to deserialise backward compatibility version.", exc_info=True)
+		for category in AddonStateCategory:
+			# Make the list of strings unique and case insensitive.
+			self[AddonStateCategory(category)] = CaseInsensitiveSet(stateDict.get(category, []))
+
 	def fromPickledDict(
 		self,
 		pickledState: dict[str, set[str] | addonAPIVersion.AddonApiVersionT | MajorMinorPatch],
 	) -> None:
-		# Load from pickledState
-		if "backCompatToAPIVersion" in pickledState:
-			self.manualOverridesAPIVersion = MajorMinorPatch(*pickledState["backCompatToAPIVersion"])
-		for category in AddonStateCategory:
-			# Make pickles case insensitive
-			self[AddonStateCategory(category)] = CaseInsensitiveSet(pickledState.get(category, set()))
-
-	def toDict(self) -> dict[str, set[str] | addonAPIVersion.AddonApiVersionT]:
-		# We cannot pickle instance of `AddonsState` directly
-		# since older versions of NVDA aren't aware about this class and they're expecting
-		# the state to be using inbuilt data types only.
-		picklableState: dict[str, set[str] | addonAPIVersion.AddonApiVersionT] = dict()
-		for category in self.data:
-			picklableState[category.value] = set(self.data[category])
-		picklableState["backCompatToAPIVersion"] = tuple(self.manualOverridesAPIVersion)
-		return picklableState
+		log.warning(
+			"addonHandler.AddonsState.fromPickledDict is deprecated. Use addonHandler.AddonsState.fromDict instead.",
+		)
+		return self.fromDict(pickledState)
+
+	def toDict(self) -> dict[str, list[str] | addonAPIVersion.AddonApiVersionT]:
+		"""Convert state to a dict that can be dumped to JSON."""
+		serializeableState: dict[str, list[str] | addonAPIVersion.AddonApiVersionT] = dict()
+		for category, addonIds in self.data.items():
+			serializeableState[category.value] = list(addonIds)
+		serializeableState["backCompatToAPIVersion"] = tuple(self.manualOverridesAPIVersion)
+		return serializeableState
 
 	def load(self) -> None:
 		"""Populates state with the default content and then loads values from the config."""
-		self._load(self.statePath)
+		self._loadWithFallback()
 		if self.manualOverridesAPIVersion != addonAPIVersion.BACK_COMPAT_TO:
 			log.debug(
 				"BACK_COMPAT_TO API version for manual compatibility overrides has changed. "
@@ -169,21 +206,51 @@ def _load(self, statePath: os.PathLike) -> None:
 
 		:param statePath: Path from which to load the addons state file.
 		"""
-		self.setDefaultStateValues()
 		try:
-			# #9038: Python 3 requires binary format when working with pickles.
-			with open(statePath, "rb") as f:
-				pickledState: dict[str, set[str] | addonAPIVersion.AddonApiVersionT] = pickle.load(f)
+			with open(statePath, "rt", encoding="utf-8") as file:
+				if (size := os.stat(file.fileno()).st_size) > _MAX_STATE_FILESIZE_BYTES:
+					log.error(f"Add-ons state file too large. {size=}B; {_MAX_STATE_FILESIZE_BYTES=}B")
+					return
+				stateDict = json.load(file)
 		except FileNotFoundError:
 			pass  # Clean config - no point logging in this case
-		except IOError:
-			log.debug("Error when reading state file", exc_info=True)
-		except pickle.UnpicklingError:
-			log.debugWarning("Failed to unpickle state", exc_info=True)
+		except OSError:
+			log.error("Error when reading state file", exc_info=True)
+		except json.JSONDecodeError:
+			log.error("Failed to deserialize add-ons state", exc_info=True)
 		except Exception:
 			log.exception()
 		else:
-			self.fromPickledDict(pickledState)
+			if not isinstance(stateDict, Mapping):
+				log.error(
+					f"Expected parsed state dictionary to be a mapping type; got {type(stateDict)} instead.",
+				)
+				return
+			self.fromDict(stateDict)
+
+	def _loadWithFallback(self) -> None:
+		if os.path.isfile(self.statePath):
+			self._load(self.statePath)
+		else:
+			if not isRunningOnSecureDesktop() and os.path.isfile(WritePaths._oldAddonStateFile):
+				# Only import if absolutely necessary.
+				from ._pickleToJsonMigration import _getAddonsStateDictFromPickle
+
+				log.warning("Loading add-ons state from pickle.")
+				try:
+					self.fromDict(_getAddonsStateDictFromPickle(WritePaths._oldAddonStateFile))
+				except Exception:
+					log.error("Failed to load pickled add-ons state.", exc_info=True)
+				else:
+					if NVDAState.shouldWriteToDisk():
+						self.save()
+				finally:
+					if NVDAState.shouldWriteToDisk():
+						log.debug("Backing up pickled add-ons state.")
+						try:
+							os.replace(WritePaths._oldAddonStateFile, WritePaths._oldAddonStateFile + ".bak")
+						except Exception:
+							log.debug("Unable to backup old add-ons state pickle file.", exc_info=True)
 
 	def removeStateFile(self) -> None:
 		if not NVDAState.shouldWriteToDisk():
@@ -223,10 +290,9 @@ def _save(self, statePath: os.PathLike) -> bool:
 			raise RuntimeError("Should not write to disk.")
 		if any(self.values()):
 			try:
-				# #9038: Python 3 requires binary format when working with pickles.
-				with open(statePath, "wb") as f:
-					pickle.dump(self.toDict(), f, protocol=0)
-			except (IOError, pickle.PicklingError):
+				with open(statePath, "wt", encoding="utf-8") as file:
+					json.dump(self.toDict(), file)
+			except (IOError, TypeError):
 				log.debugWarning("Error saving state", exc_info=True)
 			return True
 		return False

source/addonHandler/_pickleToJsonMigration.py

@@ -0,0 +1,97 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2026 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license.
+# For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt
+
+"""Utilities for migrating add-on state from the legacy pickle format to JSON.
+
+Prior to NVDA 2026.1, add-on state (which add-ons are disabled, pending install/removal, etc.) was persisted as a pickle file (``addonsState.pickle``).
+This module provides helpers that read such a pickle file, validate its contents, and return a JSON-serialisable dictionary so the state can be persisted as JSON instead (``addonsState.json``).
+
+.. warning:
+	This module is scheduled for removal in NVDA 2027.1 once all users have had sufficient opportunity to migrate.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import NVDAState
+from addonAPIVersion import BACK_COMPAT_TO
+from addonStore.models.status import AddonStateCategory
+from logHandler import log
+
+if TYPE_CHECKING:
+	import os
+
+
+# This module and its callers should be removed in 2027.1.
+# To ensure this isn't missed, log an error about using the module on 2027.1 if deprecated APIs are allowed, and fail if they're not.
+if BACK_COMPAT_TO >= (2027, 1, 0):
+	DEPRECATION_STRING = "addonHandler._pickleToJsonMigration is deprecated. All internal use of pickle should be removed in NVDA 2027.1."
+	if NVDAState._allowDeprecatedAPI():
+		log.error(DEPRECATION_STRING, stack_info=True)
+	else:
+		raise RuntimeError(DEPRECATION_STRING)
+
+
+def _pickledStateDictToJsonStateDict(pickledState: Any) -> dict[str, list[str] | tuple[int, int, int]]:
+	"""Convert an unpickled add-on state dictionary to a JSON-compatible form.
+
+	Keys and values are validated, and unrecognised or malformed entries are logged and silently discarded.
+	This allows partially corrupt pickles to be migrated.
+
+	:param pickledState: The raw object obtained from ``pickle.load``.
+		Expected to be a ``dict`` mapping :class:`AddonStateCategory` string keys to ``set[str]`` values.
+		May also include a ``"backCompatToAPIVersion"`` key with a 2- or 3-element integer tuple.
+	:return: A new dictionary suitable for JSON serialisation.
+		May be empty depending on the validity of ``pickledDict``.
+	"""
+	if not isinstance(pickledState, dict):
+		log.debug(f"Invalid pickled state: {pickledState!r}")
+		return {}
+	jsonState: dict[str, list[str] | tuple[int, int, int]] = {}
+	for key, value in pickledState.items():
+		if key == "backCompatToAPIVersion":
+			# backCompatToAPIVersion is an API version tuple, e.g. (2024, 1, 0).
+			# The patch version is optional and defaults to 0.
+			if (
+				isinstance(value, tuple)
+				and 2 <= len(value) <= 3
+				and all(isinstance(part, int) for part in value)
+			):
+				jsonState[key] = value
+			else:
+				log.debug(f"Invalid backCompatToAPIVersion: {value!r}. Discarding.")
+		elif key in AddonStateCategory:
+			# Add-on category values were stored as sets of add-on name strings.
+			# Since JSON doesn't have a set type, convert each to a list, filtering out any non-string values.
+			if not isinstance(value, set):
+				log.debug(f"Invalid category {key}: {value!r}. Discarding.")
+				continue
+			addons = jsonState[key] = []
+			for addon in value:
+				if isinstance(addon, str):
+					addons.append(addon)
+				else:
+					log.debug(f"Invalid add-on in category {key}: {addon!r}. Discarding")
+		else:
+			log.debug(f"Unrecognised key-value pair: {key!r}: {value!r}. Discarding")
+	return jsonState
+
+
+def _getAddonsStateDictFromPickle(picklePath: os.PathLike) -> dict[str, list[str] | tuple[int, int, int]]:
+	"""Load a legacy pickled add-on state file and return a JSON-compatible dict.
+
+	This is the main entry point used by :mod:`addonHandler` and :mod:`installer` when an ``addonsState.pickle`` file exists and hasn't yet been migrated to ``addonsState.json``.
+
+	:param picklePath: Path to the ``addonsState.pickle`` file.
+	:return: A validated, JSON-serialisable dictionary of add-on state.
+	:raises Exception: Any exception raised by :func:`open` or :func:`pickle.load`.
+		E.g. ``FileNotFoundError``, ``pickle.UnpicklingError``.
+	"""
+	# pickle is imported locally to avoid loading it at module level, since this migration path will eventually be removed entirely.
+	import pickle
+
+	with open(picklePath, "rb") as pickleFile:
+		return _pickledStateDictToJsonStateDict(pickle.load(pickleFile))