Harrli/python3.12 and automation (#2247)

* python 3.12 integration

* updated baseimage tag

* added dockerfiles and tests

* fixed build issue

* python version update

* BUG FIX

* issue fix

* python 3.12 sdk

* updated package version

* python 3.12 issue fix

* python pip issue fix in binaries

* updated python binary fix

* issue fix

* minor bug fix

* pip req_set file issue fix

* quick fix

* test

* python version in the image fix

* testing the container

* python 3.12 changes

* updated baseimage tag

* addressed PR comments.

* Fixed Python 3.12 pipeline failure

* Added required dependencies for lxml package

* Fixing lxml build issue

* Fixing missed gcc

* Update orjson version

* Modified orjson version

* Pipeline failure fix

* Added cargo to resolve pipeline error

* Temporarily remove orjson from python 3.12 runtime

* Addressed comments and enabled new Python test

* Added django-utils-six manually

* Removed incompatible integration test for Python 3.12

---------

Co-authored-by: Wali Bhuiyan <[email protected]>

.gitignore

@@ -290,4 +290,4 @@ src/startupscriptgenerator/**/vendor/
 src/startupscriptgenerator/src/**/__debug_bin
 src/startupscriptgenerator/src/**/run.sh
 !**/dynamic/Dockerfile
-!**/dynamic/buster.Dockerfile
+!**/dynamic/buster.Dockerfile

build/__pythonVersions.sh

@@ -9,3 +9,4 @@ PYTHON38_VERSION='3.8.18'
 PYTHON39_VERSION='3.9.18'
 PYTHON310_VERSION='3.10.13'
 PYTHON311_VERSION='3.11.6'
+PYTHON312_VERSION='3.12.0'

build/buildPythonSdkByVersion.sh

@@ -165,12 +165,12 @@ getPythonGpgByVersion() {
 
 echo
 # TODO: Determine if we need to continue building newer versions of Python from scratch
-echo "Building python 3.12 or newer from source code..."
+echo "Building python 3.13 or newer from source code..."
 
 getPythonGpgByVersion "/tmp/versionsToBuild.txt" $version
 IFS='.' read -ra SPLIT_VERSION <<< "$PYTHON_VERSION"
 
-if  [ "${SPLIT_VERSION[0]}" == "3" ] && [ "${SPLIT_VERSION[1]}" -ge "12" ]
+if  [ "${SPLIT_VERSION[0]}" == "3" ] && [ "${SPLIT_VERSION[1]}" -ge "13" ]
 then
     buildPythonfromSource $version $pythonVersionGPG
 else

build/buildRunTimeImages.sh

@@ -108,6 +108,9 @@ function getRuntimeTagVersion()
     elif [ "$PLATFORM_NAME" == "python" ]
     then
         case $PLATFORM_VERSION in
+            3.12)
+                FULL_RUNTIME_TAG_VERSION=$PYTHON312_VERSION
+                ;;
             3.11)
                 FULL_RUNTIME_TAG_VERSION=$PYTHON311_VERSION
                 ;;

build/constants.yaml

@@ -158,6 +158,7 @@
     python39-version: 3.9.18
     python310-version: 3.10.13
     python311-version: 3.11.6
+    python312-version: 3.12.0
     runtime-versions:
       - 3.7-debian-bullseye
       - 3.7-debian-buster
@@ -168,6 +169,7 @@
       - 3.10-debian-bullseye
       - 3.10-debian-buster
       - 3.11-debian-bullseye
+      - 3.12-debian-bullseye
       - dynamic-debian-buster
   outputs:
     - type: csharp

build/tools/Automation/Python/PythonConstants.cs

@@ -24,6 +24,7 @@ public class PythonConstants
             { "3.9", "E3FF2839C048B25C084DEBE9B26995E310250568" },
             { "3.10", "A035C8C19219BA821ECEA86B64E628F8D684696D" },
             { "3.11", "A035C8C19219BA821ECEA86B64E628F8D684696D" },
+            { "3.12", "7169605F62C751356D054A26A821E680E5FA6305" },
         };
     }
 }

doc/supportedPlatformVersions.md

@@ -1219,6 +1219,7 @@
 - 3.11.0b1
 - 3.11.0
 - 3.11.6
+- 3.12.0
 
 ### buster
 

doc/supportedRuntimeVersions.md

@@ -87,6 +87,7 @@ The following are supported runtime versions for the image `mcr.microsoft.com/or
 - `3.9`
 - `3.10`
 - `3.11`
+- `3.12`
 
 ## Ruby
 

images/build/Dockerfiles/gitHubActions.Dockerfile

@@ -75,6 +75,13 @@ RUN if [ "${DEBIAN_FLAVOR}" = "bookworm" ]; then \
             libssl1.1 \
             libyaml-dev \
             libxml2 \
+            # Adding lxml depended packages to avoid build failures
+            # https://lxml.de/installation.html#requirements
+            libxml2-dev \
+            libxslt-dev \
+            python3-dev \
+            python3-setuptools \
+            python3-wheel \
         && rm -rf /var/lib/apt/lists/* ; \
     elif [ "${DEBIAN_FLAVOR}" = "buster" ]; then \
         apt-get update \

images/runtime/python/3.12/base.bullseye.Dockerfile

@@ -0,0 +1,91 @@
+ARG DEBIAN_FLAVOR
+# Startup script generator
+FROM mcr.microsoft.com/oss/go/microsoft/golang:1.18-${DEBIAN_FLAVOR} as startupCmdGen
+# GOPATH is set to "/go" in the base image
+WORKDIR /go/src
+COPY src/startupscriptgenerator/src .
+ARG GIT_COMMIT=unspecified
+ARG BUILD_NUMBER=unspecified
+ARG RELEASE_TAG_NAME=unspecified
+ENV RELEASE_TAG_NAME=${RELEASE_TAG_NAME}
+ENV GIT_COMMIT=${GIT_COMMIT}
+ENV BUILD_NUMBER=${BUILD_NUMBER}
+RUN ./build.sh python /opt/startupcmdgen/startupcmdgen
+
+FROM oryxdevmcr.azurecr.io/private/oryx/oryx-run-base-${DEBIAN_FLAVOR} as main
+ARG DEBIAN_FLAVOR
+ARG IMAGES_DIR=/tmp/oryx/images
+ARG BUILD_DIR=/tmp/oryx/build
+ENV DEBIAN_FLAVOR=${DEBIAN_FLAVOR}
+
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends \
+        xz-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD images ${IMAGES_DIR}
+ADD build ${BUILD_DIR}
+RUN find ${IMAGES_DIR} -type f -iname "*.sh" -exec chmod +x {} \;
+RUN find ${BUILD_DIR} -type f -iname "*.sh" -exec chmod +x {} \;
+
+ENV PYTHON_VERSION 3.12.0
+RUN true
+COPY build/__pythonVersions.sh ${BUILD_DIR}
+RUN true
+COPY platforms/__common.sh /tmp/
+RUN true
+COPY platforms/python/prereqs/build.sh /tmp/
+RUN true
+COPY platforms/python/versions/${DEBIAN_FLAVOR}/versionsToBuild.txt /tmp/
+RUN true
+COPY images/receiveGpgKeys.sh /tmp/receiveGpgKeys.sh
+RUN true
+
+RUN chmod +x /tmp/receiveGpgKeys.sh
+RUN chmod +x /tmp/build.sh && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        build-essential \ 
+        tk-dev \
+        uuid-dev \
+        libgeos-dev
+
+RUN --mount=type=secret,id=oryx_sdk_storage_account_access_token \
+    set -e \
+    && export ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN_PATH="/run/secrets/oryx_sdk_storage_account_access_token" \
+    && ${BUILD_DIR}/buildPythonSdkByVersion.sh $PYTHON_VERSION
+
+RUN set -ex \
+ && cd /opt/python/ \
+ && ln -s 3.12.0 3.12 \
+ && ln -s 3.12 3 \
+ && echo /opt/python/3/lib >> /etc/ld.so.conf.d/python.conf \
+ && ldconfig \
+ && cd /opt/python/3/bin \
+ && ln -nsf idle3 idle \
+ && ln -nsf pydoc3 pydoc \
+ && ln -nsf python3-config python-config \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH="/opt/python/3/bin:${PATH}"
+
+# Bake Application Insights key from pipeline variable into final image
+ARG AI_CONNECTION_STRING
+ENV ORYX_AI_CONNECTION_STRING=${AI_CONNECTION_STRING}
+
+RUN ${IMAGES_DIR}/runtime/python/install-dependencies.sh
+
+RUN pip install --upgrade pip \
+    && pip install gunicorn \
+    && pip install debugpy \
+    && pip install viztracer \
+    && pip install vizplugins \
+    && pip install orjson \
+    && ln -s /opt/startupcmdgen/startupcmdgen /usr/local/bin/oryx \
+    && apt-get update \
+    && apt-get upgrade --assume-yes \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/oryx
+
+COPY --from=startupCmdGen /opt/startupcmdgen/startupcmdgen /opt/startupcmdgen/startupcmdgen

images/runtime/python/3.12/bullseye.Dockerfile

@@ -0,0 +1,104 @@
+ARG DEBIAN_FLAVOR
+# Startup script generator
+FROM mcr.microsoft.com/oss/go/microsoft/golang:1.19-${DEBIAN_FLAVOR} as startupCmdGen
+# GOPATH is set to "/go" in the base image
+WORKDIR /go/src
+COPY src/startupscriptgenerator/src .
+ARG GIT_COMMIT=unspecified
+ARG BUILD_NUMBER=unspecified
+ARG RELEASE_TAG_NAME=unspecified
+ENV RELEASE_TAG_NAME=${RELEASE_TAG_NAME}
+ENV GIT_COMMIT=${GIT_COMMIT}
+ENV BUILD_NUMBER=${BUILD_NUMBER}
+#Bake in client certificate path into image to avoid downloading it
+ENV PATH_CA_CERTIFICATE="/etc/ssl/certs/ca-certificate.crt"
+RUN ./build.sh python /opt/startupcmdgen/startupcmdgen
+
+FROM oryxdevmcr.azurecr.io/private/oryx/oryx-run-base-bullseye as main
+ARG DEBIAN_FLAVOR
+ARG IMAGES_DIR=/tmp/oryx/images
+ARG BUILD_DIR=/tmp/oryx/build
+ARG SDK_STORAGE_BASE_URL_VALUE
+ENV DEBIAN_FLAVOR=${DEBIAN_FLAVOR}
+ENV ORYX_SDK_STORAGE_BASE_URL=${SDK_STORAGE_BASE_URL_VALUE}
+
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends \
+        xz-utils \
+        # Install gcc due to error installing viztracer returning gcc not found
+        gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD images ${IMAGES_DIR}
+ADD build ${BUILD_DIR}
+RUN find ${IMAGES_DIR} -type f -iname "*.sh" -exec chmod +x {} \;
+RUN find ${BUILD_DIR} -type f -iname "*.sh" -exec chmod +x {} \;
+
+ENV PYTHON_VERSION 3.12.0
+RUN true
+COPY build/__pythonVersions.sh ${BUILD_DIR}
+RUN true
+COPY platforms/__common.sh /tmp/
+RUN true
+COPY platforms/python/prereqs/build.sh /tmp/
+RUN true
+COPY platforms/python/versions/${DEBIAN_FLAVOR}/versionsToBuild.txt /tmp/
+RUN true
+COPY images/receiveGpgKeys.sh /tmp/receiveGpgKeys.sh
+RUN true
+
+RUN chmod +x /tmp/receiveGpgKeys.sh
+RUN chmod +x /tmp/build.sh
+
+RUN --mount=type=secret,id=oryx_sdk_storage_account_access_token \
+    set -e \
+    && export ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN_PATH="/run/secrets/oryx_sdk_storage_account_access_token" \
+    && ${BUILD_DIR}/buildPythonSdkByVersion.sh $PYTHON_VERSION $DEBIAN_FLAVOR
+
+RUN set -ex \
+ && cd /opt/python/ \
+ && ln -s 3.12.0 3.12 \
+ && ln -s 3.12 3 \
+ && echo /opt/python/3/lib >> /etc/ld.so.conf.d/python.conf \
+ && ldconfig \
+ && if [ "3" = "3" ]; then cd /opt/python/3/bin \
+ && ln -nsf idle3 idle \
+ && ln -nsf pydoc3 pydoc \
+ && ln -nsf python3-config python-config; fi \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH="/opt/python/3/bin:${PATH}"
+
+# Bake Application Insights key from pipeline variable into final image
+ARG AI_CONNECTION_STRING
+ENV ORYX_AI_CONNECTION_STRING=${AI_CONNECTION_STRING}
+#Bake in client certificate path into image to avoid downloading it
+ENV PATH_CA_CERTIFICATE="/etc/ssl/certs/ca-certificate.crt"
+
+# Oryx++ Builder variables
+ENV CNB_STACK_ID="oryx.stacks.skeleton"
+LABEL io.buildpacks.stack.id="oryx.stacks.skeleton"
+
+RUN ${IMAGES_DIR}/runtime/python/install-dependencies.sh
+RUN pip install --upgrade pip \
+    && pip install gunicorn \
+    && pip install debugpy \
+    && pip install viztracer==0.15.6 \
+    && pip install vizplugins==0.1.3 \
+    # Removing orjson only for 3.12 due to build errors
+    && if [ "3.12" != "3.12" ]; then pip install orjson==3.8.10; fi \
+    && if [ "3.12" = "3.7" ] || [ "3.12" = "3.8" ]; then curl -LO http://ftp.de.debian.org/debian/pool/main/libf/libffi/libffi6_3.2.1-9_amd64.deb \
+    && dpkg -i libffi6_3.2.1-9_amd64.deb \
+    && rm libffi6_3.2.1-9_amd64.deb; fi \
+    && ln -s /opt/startupcmdgen/startupcmdgen /usr/local/bin/oryx \
+    && apt-get update \
+    && apt-get upgrade --assume-yes \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/oryx
+
+ENV LANG="en_US.UTF-8" \
+    LANGUAGE="en_US.UTF-8" \
+    LC_ALL="en_US.UTF-8"
+
+COPY --from=startupCmdGen /opt/startupcmdgen/startupcmdgen /opt/startupcmdgen/startupcmdgen

images/runtime/python/generateDockerfiles.sh

@@ -23,7 +23,7 @@ source "$PYTHON_VERSIONS_PATH"
 
 # Please make sure that any changes to debian flavors supported here are also reflected in build/constants.yaml
 declare -r PYTHON_BOOKWORM_VERSION_ARRAY=()
-declare -r PYTHON_BULLSEYE_VERSION_ARRAY=($PYTHON37_VERSION $PYTHON38_VERSION $PYTHON39_VERSION $PYTHON310_VERSION $PYTHON311_VERSION)
+declare -r PYTHON_BULLSEYE_VERSION_ARRAY=($PYTHON37_VERSION $PYTHON38_VERSION $PYTHON39_VERSION $PYTHON310_VERSION $PYTHON311_VERSION $PYTHON312_VERSION)
 declare -r PYTHON_BUSTER_VERSION_ARRAY=($PYTHON37_VERSION $PYTHON38_VERSION $PYTHON39_VERSION $PYTHON310_VERSION)
 ImageDebianFlavor="$1"
 echo "python baseimage type: $ImageDebianFlavor"

images/runtime/python/template.Dockerfile

@@ -26,6 +26,8 @@ RUN apt-get update \
     && apt-get upgrade -y \
     && apt-get install -y --no-install-recommends \
         xz-utils \
+        # Install gcc due to error installing viztracer returning gcc not found
+        gcc \
     && rm -rf /var/lib/apt/lists/*
 
 ADD images ${IMAGES_DIR}
@@ -84,7 +86,8 @@ RUN pip install --upgrade pip \
     && pip install debugpy \
     && pip install viztracer==0.15.6 \
     && pip install vizplugins==0.1.3 \
-    && pip install orjson==3.8.1 \
+    # Removing orjson only for 3.12 due to build errors
+    && if [ "%PYTHON_VERSION%" != "3.12" ]; then pip install orjson==3.8.10; fi \
     && if [ "%PYTHON_VERSION%" = "3.7" ] || [ "%PYTHON_VERSION%" = "3.8" ]; then curl -LO http://ftp.de.debian.org/debian/pool/main/libf/libffi/libffi6_3.2.1-9_amd64.deb \
     && dpkg -i libffi6_3.2.1-9_amd64.deb \
     && rm libffi6_3.2.1-9_amd64.deb; fi \

platforms/python/prereqs/build.sh

@@ -93,8 +93,12 @@ make -j $(nproc)
 
 make install
 
-if  [ "${PYTHON_VERSION[0]}" == "3" ] && [ "${PYTHON_VERSION[1]}" -ge "10" ]
+# python3.12.0 version requires the updated pip version. The older pip version:21.2.4 does not work with python3.12.0.
+IFS='.' read -ra SPLIT_VERSION <<< "$PYTHON_VERSION"
+if [ "${SPLIT_VERSION[0]}" == "3" ] && [ "${SPLIT_VERSION[1]}" -ge "12" ]
 then
+    export LD_LIBRARY_PATH="/opt/python/$PYTHON_VERSION/lib/"
+    /opt/python/$PYTHON_VERSION/bin/python3 --version
     rm -rf /usr/src/python
     find /usr/local -depth \
         \( \
@@ -103,31 +107,29 @@ then
         \) -exec rm -rf '{}' + \
 
     ldconfig
-    python3 --version
 
     # make some useful symlinks that are expected to exist
-    cd /usr/local/bin
+    cd /opt/python/$PYTHON_VERSION/bin/
     ln -s idle3 idle
     ln -s pydoc3 pydoc
     ln -s python3 python
     ln -s python3-config python-config
-
     PYTHON_GET_PIP_SHA256="c518250e91a70d7b20cceb15272209a4ded2a0c263ae5776f129e0d9b5674309"
 
     # Install pip
     wget "$PYTHON_GET_PIP_URL" -O get-pip.py
 
-    python3 get-pip.py \
+    /opt/python/$PYTHON_VERSION/bin/python3 get-pip.py \
         --trusted-host pypi.python.org \
         --trusted-host pypi.org \
         --trusted-host files.pythonhosted.org \
         --disable-pip-version-check \
         --no-cache-dir \
         --no-warn-script-location
-
     rm -rf /configure* /config.* /*.txt /*.md /*.rst /*.toml /*.m4 /tmpFiles
     rm -rf /LICENSE /install-sh /Makefile* /pyconfig* /python.tar* /python-* /libpython3.* /setup.py
     rm -rf /Python /PCbuild /Grammar /python /Objects /Parser /Misc /Tools /Programs /Modules /Include /Mac /Doc /PC /Lib 
+
 else
     # Install pip
     wget "$PYTHON_GET_PIP_URL" -O get-pip.py
@@ -170,7 +172,9 @@ echo
 
 # Replace log level in pip's code as a workaround for https://github.com/pypa/pip/issues/6189
 pipReqSetPath=`find $INSTALLATION_PREFIX/lib -path "*site-packages/pip/_internal/req/req_set.py"`
-sed -i 's|logger\.debug('\''Cleaning up\.\.\.'\'')|logger\.info('\''Cleaning up\.\.\.'\'')|' "$pipReqSetPath"
+if [ -n "$pipReqSetPath" ]; then
+    sed -i 's|logger\.debug('\''Cleaning up\.\.\.'\'')|logger\.info('\''Cleaning up\.\.\.'\'')|' "$pipReqSetPath"
+fi
 
 compressedSdkDir="/tmp/compressedSdk"
 mkdir -p $compressedSdkDir