GH-48808: [Python] Drop support for Pandas < 2.0.3 (#50444)

### Rationale for this change

[Pandas 2.0.0](https://pypi.org/project/pandas/2.0.0/) was released on April 3rd 2023.
The last version of Pandas 1 was [Pandas 1.5.3](https://pypi.org/project/pandas/1.5.3/) and it was released on January 19th 2023. That version only supports up to Python 3.11.

We are currently supporting up to Pandas 1.5.2 (first version supporting 3.11).

On the issue we discussed to bump our Pandas support to 2.0.0 but given 2.0.3 is the latest in the 2.0.x, was released on June 2023, only supports up to Python 3.11 and bumping to it allows use to improve even further some of the conditionals we are bumping to Pandas >= 2.0.3 

### What changes are included in this PR?

Update minimal support check to Version(2.0.3) and remove all test skips, conditionals and scenarios where we are checking for Pandas < 2.0.3.

We found a couple of minor things

### Are these changes tested?

Yes via CI

### Are there any user-facing changes?

Yes, Pandas < 2 won't be supported.

* GitHub Issue: #48808

Authored-by: Raúl Cumplido <raulcumplido@gmail.com>
Signed-off-by: Antoine Pitrou <antoine@python.org>

Author: raulcd

.github/workflows/python.yml

@@ -67,7 +67,7 @@ jobs:
         name:
           - conda-python-docs
           - conda-python-3.12-nopandas
-          - conda-python-3.11-pandas-1.5.2
+          - conda-python-3.11-pandas-2.0.3
           - conda-python-3.14-pandas-latest
           - conda-python-3.13-no-numpy
         include:
@@ -79,11 +79,11 @@ jobs:
             image: conda-python
             title: AMD64 Conda Python 3.12 Without Pandas
             python: "3.12"
-          - name: conda-python-3.11-pandas-1.5.2
+          - name: conda-python-3.11-pandas-2.0.3
             image: conda-python-pandas
-            title: AMD64 Conda Python 3.11 Pandas 1.5.2
+            title: AMD64 Conda Python 3.11 Pandas 2.0.3
             python: "3.11"
-            pandas: "1.5.2"
+            pandas: "2.0.3"
             numpy: "1.23.2"
           - name: conda-python-3.14-pandas-latest
             image: conda-python-pandas

dev/tasks/tasks.yml

@@ -727,7 +727,7 @@ tasks:
 
   ############################## Integration tests ############################
 
-{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "1.5.2", "1.23.2", True),
+{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "2.0.3", "1.23.2", True),
                                                                      ("3.12", "latest", "latest", False),
                                                                      ("3.13", "latest", "1.26.2", False),
                                                                      ("3.13", "latest", "latest", False),

docs/source/python/install.rst

@@ -74,7 +74,7 @@ Dependencies
 Optional dependencies
 
 * **NumPy 1.23.2** or higher.
-* **pandas 1.5.2** or higher,
+* **pandas 2.0.3** or higher,
 * **cffi**.
 
 Additional packages PyArrow is compatible with are :ref:`fsspec <filesystem-fsspec>`

python/pyarrow/array.pxi

@@ -2383,10 +2383,6 @@ cdef _array_like_to_pandas(obj, options, types_mapper):
         arr = dtype.__from_arrow__(obj)
         return pandas_api.series(arr, name=name, copy=False)
 
-    if pandas_api.is_v1():
-        # ARROW-3789: Coerce date/timestamp types to datetime64[ns]
-        c_options.coerce_temporal_nanoseconds = True
-
     if isinstance(obj, Array):
         with nogil:
             check_status(ConvertArrayToPandas(c_options,

python/pyarrow/pandas-shim.pxi

@@ -38,7 +38,7 @@ cdef class _PandasAPIShim(object):
         object _array_like_types, _is_extension_array_dtype, _lock
         bint has_sparse
         bint _pd024
-        bint _is_v1, _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict
+        bint _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict
 
     def __init__(self):
         self._lock = Lock()
@@ -61,25 +61,23 @@ cdef class _PandasAPIShim(object):
         self._pd = pd
         self._version = pd.__version__
         self._loose_version = Version(pd.__version__)
-        self._is_v1 = False
 
-        if self._loose_version < Version('1.0.0'):
+        if self._loose_version < Version('2.0.3'):
             self._have_pandas = False
             if raise_:
                 raise ImportError(
-                    f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is "
+                    f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is "
                     "installed"
                 )
             else:
                 warnings.warn(
-                    f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is "
+                    f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is "
                     "installed. Therefore, pandas-specific integration is not "
                     "used.",
                     stacklevel=2
                 )
                 return
 
-        self._is_v1 = self._loose_version < Version('2.0.0')
         self._is_ge_v21 = self._loose_version >= Version('2.1.0')
         self._is_ge_v23 = self._loose_version >= Version('2.3.0.dev0')
         self._is_ge_v3 = self._loose_version >= Version('3.0.0.dev0')
@@ -166,10 +164,6 @@ cdef class _PandasAPIShim(object):
         self._check_import()
         return self._version
 
-    def is_v1(self):
-        self._check_import()
-        return self._is_v1
-
     def is_ge_v21(self):
         self._check_import()
         return self._is_ge_v21

python/pyarrow/pandas_compat.py

@@ -790,8 +790,6 @@ def _reconstruct_block(item, columns=None, extension_columns=None, return_block=
 
 
 def make_datetimetz(unit, tz):
-    if _pandas_api.is_v1():
-        unit = 'ns'  # ARROW-3789: Coerce date/timestamp types to datetime64[ns]
     tz = pa.lib.string_to_tzinfo(tz, prefer_zoneinfo=_pandas_api.is_ge_v3())
     return _pandas_api.datetimetz_type(unit, tz=tz)
 

python/pyarrow/src/arrow/python/arrow_to_pandas.cc

@@ -1293,7 +1293,7 @@ struct ObjectWriterVisitor {
                               PyObject** out) {
       ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr);
       // DateOffset objects do not add nanoseconds component to pd.Timestamp.
-      // as of  Pandas 1.3.3
+      // as of Pandas 1.3.3
       // (https://github.com/pandas-dev/pandas/issues/43892).
       // So convert microseconds and remainder to preserve data
       // but give users more expected results.

python/pyarrow/table.pxi

@@ -4070,10 +4070,6 @@ def table_to_blocks(options, Table table, categories, extension_columns):
         c_options.extension_columns = make_shared[unordered_set[c_string]](
             unordered_set[c_string]({tobytes(col) for col in extension_columns}))
 
-    if pandas_api.is_v1():
-        # ARROW-3789: Coerce date/timestamp types to datetime64[ns]
-        c_options.coerce_temporal_nanoseconds = True
-
     if c_options.self_destruct:
         # Move the shared_ptr, table is now unsafe to use further
         c_table = move(table.sp_table)

python/pyarrow/tests/interchange/test_conversion.py

@@ -17,7 +17,6 @@
 
 from datetime import datetime as dt
 import pyarrow as pa
-from pyarrow.vendored.version import Version
 import pytest
 
 try:
@@ -122,9 +121,6 @@ def test_offset_of_sliced_array():
     ]
 )
 def test_pandas_roundtrip(uint, int, float, np_float_str):
-    if Version(pd.__version__) < Version("1.5.0"):
-        pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
     arr = [1, 2, 3]
     table = pa.table(
         {
@@ -153,9 +149,6 @@ def test_pandas_roundtrip(uint, int, float, np_float_str):
 @pytest.mark.pandas
 def test_pandas_roundtrip_string():
     # See https://github.com/pandas-dev/pandas/issues/50554
-    if Version(pd.__version__) < Version("1.6"):
-        pytest.skip("Column.size() bug in pandas")
-
     arr = ["a", "", "c"]
     table = pa.table({"a": pa.array(arr)})
 
@@ -182,46 +175,32 @@ def test_pandas_roundtrip_string():
 @pytest.mark.pandas
 def test_pandas_roundtrip_large_string():
     # See https://github.com/pandas-dev/pandas/issues/50554
-    if Version(pd.__version__) < Version("1.6"):
-        pytest.skip("Column.size() bug in pandas")
-
     arr = ["a", "", "c"]
     table = pa.table({"a_large": pa.array(arr, type=pa.large_string())})
 
     from pandas.api.interchange import (
         from_dataframe as pandas_from_dataframe
     )
 
-    if Version(pd.__version__) >= Version("2.0.1"):
-        pandas_df = pandas_from_dataframe(table)
-        result = pi.from_dataframe(pandas_df)
-
-        assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
-        assert pa.types.is_large_string(table["a_large"].type)
-        assert pa.types.is_large_string(result["a_large"].type)
+    pandas_df = pandas_from_dataframe(table)
+    result = pi.from_dataframe(pandas_df)
 
-        table_protocol = table.__dataframe__()
-        result_protocol = result.__dataframe__()
+    assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
+    assert pa.types.is_large_string(table["a_large"].type)
+    assert pa.types.is_large_string(result["a_large"].type)
 
-        assert table_protocol.num_columns() == result_protocol.num_columns()
-        assert table_protocol.num_rows() == result_protocol.num_rows()
-        assert table_protocol.num_chunks() == result_protocol.num_chunks()
-        assert table_protocol.column_names() == result_protocol.column_names()
+    table_protocol = table.__dataframe__()
+    result_protocol = result.__dataframe__()
 
-    else:
-        # large string not supported by pandas implementation for
-        # older versions of pandas
-        # https://github.com/pandas-dev/pandas/issues/52795
-        with pytest.raises(AssertionError):
-            pandas_from_dataframe(table)
+    assert table_protocol.num_columns() == result_protocol.num_columns()
+    assert table_protocol.num_rows() == result_protocol.num_rows()
+    assert table_protocol.num_chunks() == result_protocol.num_chunks()
+    assert table_protocol.column_names() == result_protocol.column_names()
 
 
 @pytest.mark.pandas
 def test_pandas_roundtrip_string_with_missing():
     # See https://github.com/pandas-dev/pandas/issues/50554
-    if Version(pd.__version__) < Version("1.6"):
-        pytest.skip("Column.size() bug in pandas")
-
     arr = ["a", "", "c", None]
     table = pa.table({"a": pa.array(arr),
                       "a_large": pa.array(arr, type=pa.large_string())})
@@ -230,29 +209,20 @@ def test_pandas_roundtrip_string_with_missing():
         from_dataframe as pandas_from_dataframe
     )
 
-    if Version(pd.__version__) >= Version("2.0.2"):
-        pandas_df = pandas_from_dataframe(table)
-        result = pi.from_dataframe(pandas_df)
+    pandas_df = pandas_from_dataframe(table)
+    result = pi.from_dataframe(pandas_df)
 
-        assert result["a"].to_pylist() == table["a"].to_pylist()
-        assert pa.types.is_string(table["a"].type)
-        assert pa.types.is_large_string(result["a"].type)
+    assert result["a"].to_pylist() == table["a"].to_pylist()
+    assert pa.types.is_string(table["a"].type)
+    assert pa.types.is_large_string(result["a"].type)
 
-        assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
-        assert pa.types.is_large_string(table["a_large"].type)
-        assert pa.types.is_large_string(result["a_large"].type)
-    else:
-        # older versions of pandas do not have bitmask support
-        # https://github.com/pandas-dev/pandas/issues/49888
-        with pytest.raises(NotImplementedError):
-            pandas_from_dataframe(table)
+    assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
+    assert pa.types.is_large_string(table["a_large"].type)
+    assert pa.types.is_large_string(result["a_large"].type)
 
 
 @pytest.mark.pandas
 def test_pandas_roundtrip_categorical():
-    if Version(pd.__version__) < Version("2.0.2"):
-        pytest.skip("Bitmasks not supported in pandas interchange implementation")
-
     arr = ["Mon", "Tue", "Mon", "Wed", "Mon", "Thu", "Fri", "Sat", None]
     table = pa.table(
         {"weekday": pa.array(arr).dictionary_encode()}
@@ -299,21 +269,14 @@ def test_pandas_roundtrip_categorical():
 @pytest.mark.pandas
 @pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns'])
 def test_pandas_roundtrip_datetime(unit):
-    if Version(pd.__version__) < Version("1.5.0"):
-        pytest.skip("__dataframe__ added to pandas in 1.5.0")
     from datetime import datetime as dt
 
     # timezones not included as they are not yet supported in
     # the pandas implementation
     dt_arr = [dt(2007, 7, 13), dt(2007, 7, 14), dt(2007, 7, 15)]
     table = pa.table({"a": pa.array(dt_arr, type=pa.timestamp(unit))})
 
-    if Version(pd.__version__) < Version("1.6"):
-        # pandas < 2.0 always creates datetime64 in "ns"
-        # resolution
-        expected = pa.table({"a": pa.array(dt_arr, type=pa.timestamp('ns'))})
-    else:
-        expected = table
+    expected = table
 
     from pandas.api.interchange import (
         from_dataframe as pandas_from_dataframe
@@ -337,9 +300,6 @@ def test_pandas_roundtrip_datetime(unit):
     "np_float_str", ["float32", "float64"]
 )
 def test_pandas_to_pyarrow_with_missing(np_float_str):
-    if Version(pd.__version__) < Version("1.5.0"):
-        pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
     np_array = np.array([0, np.nan, 2], dtype=np.dtype(np_float_str))
     datetime_array = [None, dt(2007, 7, 14), dt(2007, 7, 15)]
     df = pd.DataFrame({
@@ -359,9 +319,6 @@ def test_pandas_to_pyarrow_with_missing(np_float_str):
 
 @pytest.mark.pandas
 def test_pandas_to_pyarrow_float16_with_missing():
-    if Version(pd.__version__) < Version("1.5.0"):
-        pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
     # np.float16 errors if ps.is_nan is used
     # pyarrow.lib.ArrowNotImplementedError: Function 'is_nan' has no kernel
     # matching input types (halffloat)
@@ -482,9 +439,6 @@ def test_nan_as_null():
 
 @pytest.mark.pandas
 def test_allow_copy_false():
-    if Version(pd.__version__) < Version("1.5.0"):
-        pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
     # Test that an error is raised when a copy is needed
     # to create a bitmask
 
@@ -501,9 +455,6 @@ def test_allow_copy_false():
 
 @pytest.mark.pandas
 def test_allow_copy_false_bool_categorical():
-    if Version(pd.__version__) < Version("1.5.0"):
-        pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
     # Test that an error is raised for boolean
     # and categorical dtype (copy is always made)
 

python/pyarrow/tests/parquet/test_pandas.py

@@ -524,9 +524,6 @@ def test_pandas_categorical_roundtrip():
 @pytest.mark.pandas
 def test_categories_with_string_pyarrow_dtype(tempdir):
     # gh-33727: writing to parquet should not fail
-    if Version(pd.__version__) < Version("1.3.0"):
-        pytest.skip("PyArrow backed string data type introduced in pandas 1.3.0")
-
     df1 = pd.DataFrame({"x": ["foo", "bar", "foo"]}, dtype="string[pyarrow]")
     df1 = df1.astype("category")