chore: upgrade setuptools & pin cmake (#3152)

* fix: update setuptools in layers

* use cmake<4

* add comment

* clarify comment

* clarify comment

Author: kukushking

building/lambda/Dockerfile

@@ -34,9 +34,14 @@ RUN ln -s /usr/bin/gcc10-ld /usr/bin/ld
 FROM ${python_version}
 COPY pyproject.toml poetry.lock ./
 
-RUN pip3 install --upgrade pip wheel setuptools>=64 setuptools_scm>=8
+# Setuptools is a build dependency of arrow and runtime dependency of some of our dependencies (mainly redshift-connector).
+# Remove when arrow version shipped with lambda layers and dependencies are updated.
+RUN pip3 install --upgrade pip wheel setuptools>=78.1.1 setuptools_scm>=8
 RUN pip3 install --upgrade urllib3==1.26.16  # temporary to avoid https://github.com/urllib3/urllib3/issues/2168 (TODO remove when the AL2 image updates to support OpenSSL 1.1.1+)
-RUN pip3 install --upgrade six cython cmake hypothesis poetry
+# In new CMake 4, compatibility with CMake < 3.5 has been removed. 
+# Unpin CMake when arrow version shipped with lambda layers is updated.
+RUN pip3 install --upgrade cmake==3.31.6
+RUN pip3 install --upgrade six cython hypothesis poetry
 ENV PIP_NO_BINARY="numpy,pandas"
 RUN poetry config virtualenvs.create false --local && poetry install --no-root --only main
 

building/lambda/Dockerfile.al2023

@@ -23,8 +23,14 @@ WORKDIR /root
 FROM ${python_version}
 COPY pyproject.toml poetry.lock ./
 
-RUN pip3 install --upgrade pip wheel setuptools>=64 setuptools_scm>=8
-RUN pip3 install --upgrade six cython cmake hypothesis poetry
+# Setuptools is a build dependency of arrow and runtime dependency of some of our dependencies (mainly redshift-connector).
+# Remove when arrow version shipped with lambda layers and dependencies are updated.
+RUN pip3 install --upgrade pip wheel setuptools>=78.1.1 setuptools_scm>=8
+# In new CMake 4, compatibility with CMake < 3.5 has been removed. 
+# Unpin CMake when arrow version shipped with lambda layers is updated.
+RUN pip3 install --upgrade cmake==3.31.6
+RUN pip3 install --upgrade six cython hypothesis poetry
+
 ENV PIP_NO_BINARY="numpy,pandas"
 RUN poetry config virtualenvs.create false --local && poetry install --no-root --only main