Merge remote-tracking branch 'upstream/main' into dev_version

Author: datapythonista

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.8
+    rev: v0.11.12
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -74,7 +74,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.19.1
+    rev: v3.20.0
     hooks:
     -   id: pyupgrade
         args: [--py310-plus]
@@ -95,14 +95,14 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v20.1.3
+    rev: v20.1.5
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: [-i]
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.8.0
+    rev: v1.8.1
     hooks:
     - id: meson-fmt
       args: ['--inplace']

asv_bench/asv.conf.json

@@ -42,7 +42,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "pip+build": [],
-        "Cython": ["3.0"],
+        "Cython": [],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

ci/deps/actions-310-minimum_versions.yaml

@@ -8,7 +8,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-310.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-311-downstream_compat.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-311-numpydev.yaml

@@ -8,7 +8,7 @@ dependencies:
   - versioneer
   - meson=1.2.1
   - meson-python=0.13.1
-  - cython>=0.29.33
+  - cython<4.0.0a0
 
   # test dependencies
   - pytest>=7.3.2

ci/deps/actions-311-pyarrownightly.yaml

@@ -7,7 +7,7 @@ dependencies:
   # build dependencies
   - versioneer
   - meson=1.2.1
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson-python=0.13.1
 
   # test dependencies

ci/deps/actions-311.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-312.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-313-freethreading.yaml

@@ -7,6 +7,7 @@ dependencies:
   # build dependencies
   - setuptools
   - versioneer
+  - cython<4.0.0a0
   - meson=1.8.0
   - meson-python=0.18.0
 
@@ -21,10 +22,8 @@ dependencies:
   # optional dependencies
   - hypothesis>=6.84.0
 
-  # Move Cython to build dependencies, once they release a version that supports freethreading
   - pip:
     # No free-threaded coveragepy (with the C-extension) on conda-forge yet
     - pytest-cov
     - "tzdata>=2022.7"
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-    - "cython"

ci/deps/actions-313.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

ci/deps/actions-pypy-39.yaml

@@ -9,7 +9,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython>=0.29.33
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

doc/cheatsheet/Pandas_Cheat_Sheet.pdf

@@ -1461,16 +1461,33 @@ Looking up values by index/column labels
 
 Sometimes you want to extract a set of values given a sequence of row labels
 and column labels, this can be achieved by ``pandas.factorize``  and NumPy indexing.
-For instance:
 
-.. ipython:: python
+For heterogeneous column types, we subset columns to avoid unnecessary NumPy conversions:
+
+.. code-block:: python
+
+   def pd_lookup_het(df, row_labels, col_labels):
+      rows = df.index.get_indexer(row_labels)
+      cols = df.columns.get_indexer(col_labels)
+      sub = df.take(np.unique(cols), axis=1)
+      sub = sub.take(np.unique(rows), axis=0)
+      rows = sub.index.get_indexer(row_labels)
+      values = sub.melt()["value"]
+      cols = sub.columns.get_indexer(col_labels)
+      flat_index = rows + cols * len(sub)
+      result = values[flat_index]
+      return result
+
+For homogeneous column types, it is fastest to skip column subsetting and go directly to NumPy:
+
+.. code-block:: python
 
-    df = pd.DataFrame({'col': ["A", "A", "B", "B"],
-                       'A': [80, 23, np.nan, 22],
-                       'B': [80, 55, 76, 67]})
-    df
-    idx, cols = pd.factorize(df['col'])
-    df.reindex(cols, axis=1).to_numpy()[np.arange(len(df)), idx]
+   def pd_lookup_hom(df, row_labels, col_labels):
+       rows = df.index.get_indexer(row_labels)
+       df = df.loc[:, sorted(set(col_labels))]
+       cols = df.columns.get_indexer(col_labels)
+       result = df.to_numpy()[rows, cols]
+       return result
 
 Formerly this could be achieved with the dedicated ``DataFrame.lookup`` method
 which was deprecated in version 1.2.0 and removed in version 2.0.0.

doc/cheatsheet/Pandas_Cheat_Sheet.pptx

@@ -32,6 +32,7 @@ enhancement1
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
 
+- :meth:`pandas.api.interchange.from_dataframe` now uses the `PyCapsule Interface <https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html>`_ if available, only falling back to the Dataframe Interchange Protocol if that fails (:issue:`60739`)
 - The semantics for the ``copy`` keyword in ``__array__`` methods (i.e. called
   when using ``np.array()`` or ``np.asarray()`` on pandas objects) has been
   updated to work correctly with NumPy >= 2 (:issue:`57739`)

doc/source/user_guide/indexing.rst

@@ -30,7 +30,6 @@ Other enhancements
 ^^^^^^^^^^^^^^^^^^
 - :class:`pandas.api.typing.FrozenList` is available for typing the outputs of :attr:`MultiIndex.names`, :attr:`MultiIndex.codes` and :attr:`MultiIndex.levels` (:issue:`58237`)
 - :class:`pandas.api.typing.SASReader` is available for typing the output of :func:`read_sas` (:issue:`55689`)
-- :meth:`pandas.api.interchange.from_dataframe` now uses the `PyCapsule Interface <https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html>`_ if available, only falling back to the Dataframe Interchange Protocol if that fails (:issue:`60739`)
 - Added :meth:`.Styler.to_typst` to write Styler objects to file, buffer or string in Typst format (:issue:`57617`)
 - Added missing :meth:`pandas.Series.info` to API reference (:issue:`60926`)
 - :class:`pandas.api.typing.NoDefault` is available for typing ``no_default``
@@ -891,6 +890,7 @@ Other
 - Bug in :meth:`DataFrame.query` which raised an exception when querying integer column names using backticks. (:issue:`60494`)
 - Bug in :meth:`DataFrame.shift` where passing a ``freq`` on a DataFrame with no columns did not shift the index correctly. (:issue:`60102`)
 - Bug in :meth:`DataFrame.sort_index` when passing ``axis="columns"`` and ``ignore_index=True`` and ``ascending=False`` not returning a :class:`RangeIndex` columns (:issue:`57293`)
+- Bug in :meth:`DataFrame.sort_values` where sorting by a column explicitly named ``None`` raised a ``KeyError`` instead of sorting by the column as expected. (:issue:`61512`)
 - Bug in :meth:`DataFrame.transform` that was returning the wrong order unless the index was monotonically increasing. (:issue:`57069`)
 - Bug in :meth:`DataFrame.where` where using a non-bool type array in the function would return a ``ValueError`` instead of a ``TypeError`` (:issue:`56330`)
 - Bug in :meth:`Index.sort_values` when passing a key function that turns values into tuples, e.g. ``key=natsort.natsort_key``, would raise ``TypeError`` (:issue:`56081`)

doc/source/whatsnew/v2.3.0.rst

@@ -8,7 +8,7 @@ dependencies:
 
   # build dependencies
   - versioneer
-  - cython~=3.0.5
+  - cython<4.0.0a0
   - meson=1.2.1
   - meson-python=0.13.1
 

doc/source/whatsnew/v3.0.0.rst

@@ -1645,11 +1645,7 @@ def _is_label_reference(self, key: Level, axis: Axis = 0) -> bool:
         axis_int = self._get_axis_number(axis)
         other_axes = (ax for ax in range(self._AXIS_LEN) if ax != axis_int)
 
-        return (
-            key is not None
-            and is_hashable(key)
-            and any(key in self.axes[ax] for ax in other_axes)
-        )
+        return is_hashable(key) and any(key in self.axes[ax] for ax in other_axes)
 
     @final
     def _is_label_or_level_reference(self, key: Level, axis: AxisInt = 0) -> bool:

environment.yml

@@ -633,6 +633,15 @@ def autocorrelation_plot(series: Series, ax: Axes | None = None, **kwargs) -> Ax
     """
     Autocorrelation plot for time series.
 
+    This method generates an autocorrelation plot for a given time series,
+    which helps to identify any periodic structure or correlation within the
+    data across various lags. It shows the correlation of a time series with a
+    delayed copy of itself as a function of delay. Autocorrelation plots are useful for
+    checking randomness in a data set. If the data are random, the autocorrelations
+    should be near zero for any and all time-lag separations. If the data are not
+    random, then one or more of the autocorrelations will be significantly
+    non-zero.
+
     Parameters
     ----------
     series : Series

pandas/core/generic.py

@@ -630,6 +630,13 @@ def test_sort_values_no_op_reset_index(self):
         expected = DataFrame({"A": [10, 20], "B": [1, 5]})
         tm.assert_frame_equal(result, expected)
 
+    def test_sort_by_column_named_none(self):
+        # GH#61512
+        df = DataFrame([[3, 1], [2, 2]], columns=[None, "C1"])
+        result = df.sort_values(by=None)
+        expected = DataFrame([[2, 2], [3, 1]], columns=[None, "C1"], index=[1, 0])
+        tm.assert_frame_equal(result, expected)
+
 
 class TestDataFrameSortKey:  # test key sorting (issue 27237)
     def test_sort_values_inplace_key(self, sort_by_key):

pandas/plotting/_misc.py

@@ -671,7 +671,7 @@ def test_groupby_raises_category_on_category(
         "nunique": (None, ""),
         "pct_change": (TypeError, "unsupported operand type"),
         "prod": (TypeError, "category type does not support prod operations"),
-        "quantile": (TypeError, ""),
+        "quantile": (TypeError, "No matching signature found"),
         "rank": (None, ""),
         "sem": (
             TypeError,

pandas/tests/frame/methods/test_sort_values.py

@@ -5,7 +5,7 @@ requires = [
     "meson-python>=0.13.1",
     "meson>=1.2.1,<2",
     "wheel",
-    "Cython==3.1.0rc1",  # Note: sync with setup.py, environment.yml and asv.conf.json
+    "Cython<4.0.0a0",  # Note: sync with setup.py, environment.yml and asv.conf.json
     # Force numpy higher than 2.0rc1, so that built wheels are compatible
     # with both numpy 1 and 2
     "numpy>=2.0.0rc1",

pandas/tests/groupby/test_raises.py

@@ -3,7 +3,7 @@
 
 pip
 versioneer[toml]
-cython~=3.0.5
+cython<4.0.0a0
 meson[ninja]==1.2.1
 meson-python==0.13.1
 pytest>=7.3.2