Merge branch 'main' into test-image

Author: keithly

.github/workflows/ci.yml

@@ -90,7 +90,7 @@ jobs:
 
       - uses: hashicorp/setup-terraform@v3
         with:
-          terraform_version: 1.6.3
+          terraform_version: 1.6.5
 
       - name: Config Terraform plugin cache
         run: |

.tool-versions

@@ -1,3 +1,3 @@
-python 3.11.6
-terraform 1.6.4
+python 3.12.1
+terraform 1.6.5
 tflint 0.49.0

Dockerfile

@@ -1,45 +1,22 @@
-ARG AMAZONLINUX_VERSION=2.0.20231101.0
-ARG ARCH=amd64
-FROM public.ecr.aws/amazonlinux/amazonlinux:${AMAZONLINUX_VERSION}-${ARCH} as base
-ENV LANG=en_US.UTF-8 \
-    TZ=:/etc/localtime \
-    PATH=/var/lang/bin:/usr/local/bin:/usr/bin/:/bin:/opt/bin \
-    LD_LIBRARY_PATH=/var/lang/lib:/lib64:/usr/lib64:/var/runtime:/var/runtime/lib:/var/task:/var/task/lib:/opt/lib \
-    LAMBDA_TASK_ROOT=/var/task \
-    LAMBDA_RUNTIME_DIR=/var/runtime
-RUN yum -y update && \
-    yum -y install shadow-utils && \
-    yum clean all
+ARG AL_PROVIDED_VERSION=al2023.2023.12.06.10
+ARG ARCH=x86_64
+FROM public.ecr.aws/lambda/provided:${AL_PROVIDED_VERSION}-${ARCH} as base
+RUN dnf -y update && \
+    dnf -y install shadow-utils && \
+    dnf clean all
 
 FROM base as builder
-RUN yum -y install yum-utils && \
-    yum -y groupinstall "Development Tools" && \
-    yum-builddep -y python3 && \
-    yum clean all
+RUN dnf -y update && \
+    dnf -y install gcc openssl-devel bzip2-devel libffi-devel xz-devel zlib-devel tar xz && \
+    dnf clean all
 
-ARG OPENSSL_VERSION=1.1.1s
-ARG OPENSSL_KEY=B8EF1A6BA9DA2D5C
-RUN cd "$(mktemp -d)" && \
-    curl https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz.asc --remote-name && \
-    curl https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz --remote-name && \
-    gpg --keyserver hkps://keys.openpgp.org --recv-keys ${OPENSSL_KEY} && \
-    gpg --verify openssl-${OPENSSL_VERSION}.tar.gz.asc openssl-${OPENSSL_VERSION}.tar.gz && \
-    tar xf openssl-${OPENSSL_VERSION}.tar.gz && \
-    cd openssl-${OPENSSL_VERSION} && \
-    ./config --prefix=/var/lang && \
-    make -j "$(nproc)" && \
-    make install
+ARG PYTHON_VERSION=3.12.1
 
-ARG PYTHON_VERSION=3.11.6
-ARG PYTHON_KEY=64E628F8D684696D
 RUN cd "$(mktemp -d)" && \
-    curl https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz.asc --remote-name && \
-    curl https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz --remote-name && \
-    gpg --keyserver hkps://keys.openpgp.org --recv-keys ${PYTHON_KEY} && \
-    gpg --verify Python-${PYTHON_VERSION}.tar.xz.asc Python-${PYTHON_VERSION}.tar.xz && \
+    curl -O https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz && \
     tar xf Python-${PYTHON_VERSION}.tar.xz && \
     cd Python-${PYTHON_VERSION} && \
-    ./configure --prefix=/var/lang --with-openssl=/var/lang --enable-optimizations --with-lto=full --with-system-ffi --with-computed-gotos --enable-loadable-sqlite-extensions && \
+    ./configure --prefix=/var/lang --enable-optimizations --with-lto=full --with-system-ffi --with-computed-gotos --enable-loadable-sqlite-extensions && \
     make -j "$(nproc)" && \
     make install
 
@@ -52,21 +29,19 @@ RUN ln -s /var/lang/bin/python3           /var/lang/bin/python && \
     ln -s /var/lang/bin/pydoc3            /var/lang/bin/pydoc && \
     ln -s /var/lang/bin/python3-config    /var/lang/bin/python-config
 
-COPY lambda/lambda-entrypoint.sh /
-COPY lambda/install-rie.sh /
-COPY lambda/runtime /var/runtime
+WORKDIR /var/task
+COPY lambda /var/task
 
 RUN ./install-rie.sh
 
 RUN python3 -m pip install -U --no-cache-dir pip setuptools wheel && \
-    python3 -m pip install --no-cache-dir --target /var/runtime awslambdaric boto3
+    python3 -m pip install --no-cache-dir --target /var/task awslambdaric boto3
 
-WORKDIR /var/task
 COPY src src
 
 RUN /usr/sbin/useradd lambdauser -d /var/task
 USER lambdauser
 
 ENV APP_VERSION=1.0.0
 
-ENTRYPOINT [ "/lambda-entrypoint.sh", "src/lambda-poc/lambda_function.handler"]
+ENTRYPOINT ["/var/task/lambda-entrypoint.sh", "src/lambda-poc/lambda_function.handler"]

README.md

@@ -1,40 +1,40 @@
 # lambda-python-custom
 
-Use Python >= 3.10 on AWS Lambda
+Use Any Python Version on AWS Lambda
 
-Currently, AWS Lambda only supports Python versions 3.7 - 3.9. This project shows how to use a newer version by creating
-a custom runtime. This is documented by AWS in several different places and is tedious to piece together what's truly
-required.
-
-Update: I've created a [simpler example](simple-example) that starts with the Debian-based official Python image and
-therefore doesn't require building it from source. The only dependency truly required is
-the [awslambdaric](https://github.com/aws/aws-lambda-python-runtime-interface-client), and everything can be copied into
-the same directory in the image.
+This project was created when AWS Lambda only supported Python versions 3.7 - 3.9, despite 3.10 and 3.11 having been
+released for quite a while. Now AWS is again keeping up with Python versions, but this project shows how to use any
+version by creating a custom runtime. The AWS documentation for how do this has improved but is still spread across
+several different sites and pages.
 
 ## Dockerfile
 
-The Docker image is loosely based
-on [the one used by AWS Lambda for Python 3.9](https://gallery.ecr.aws/lambda/python) (see
-also [here](https://github.com/aws/aws-lambda-base-images/tree/python3.9)), and incorporates its default bootstrap files
-under [/lambda](lambda). It's built via GitHub actions and deployed with Terraform.
+The main Docker image is now based on the new
+[Amazon Linux 2023 Provided image for Lambda](https://gallery.ecr.aws/lambda/provided) (also see
+https://aws.amazon.com/blogs/compute/introducing-the-amazon-linux-2023-runtime-for-aws-lambda/). It's built via GitHub
+actions and deployed with Terraform. This means a modern version of OpenSSL is available without having to build it from
+source. However, the minimal image it's based on made verifying the Python source download more difficult
+(see https://github.com/keithly/lambda-python-custom/issues/78).
 
 The Dockerfile follows all the best practices I'm aware of. :) There are several ARGs for passing specific versions of
-the base image, OpenSSL, and Python, but I didn't attempt to pin every dependency. There's a tradeoff between
-reproducibility and convenience.
+the base image and Python, but I didn't attempt to pin every dependency. There's a tradeoff between reproducibility and
+convenience.
 
-- Starts with Amazon Linux 2, creates a builder stage from it, copies build artifacts back into the base.
-- Builds OpenSSL and Python from source, checking pgp signatures. The python build options optimize the build for speed
-  of execution. The dependencies and build options could no doubt be tweaked, but this is the simplest solution I found
-  that makes a functional Python build.
+- Starts with Amazon Linux 2023, creates a builder stage from it, copies build artifacts back into the base.
+- Builds Python from source The python build options optimize the build for speed of execution. The dependencies and
+  build options could no doubt be tweaked, but this is the simplest solution I found that makes a functional Python
+  build.
 - Links "python3" to "python"
-- Curls the latest version of
+- Installs the latest version of
   the [AWS Lambda Runtime Interface Emulator](https://github.com/aws/aws-lambda-runtime-interface-emulator/)
 - Installs the latest versions of pip, setuptools, wheel,
   then [awslambdaric](https://github.com/aws/aws-lambda-python-runtime-interface-client) and boto3
 - runs as a non-root user (though this may not matter for running on Lambda)
 
-Maybe it would make more sense to use the Python 3.9 lambda image as the base. Doing so would likely entail a different
-set of tradeoffs with trying to remove Python 3.9 and edit configs.
+There's also a [simpler example](simple-example) that starts with the Debian-based official Python image and
+therefore doesn't require building it from source. The only dependency truly required is
+the [awslambdaric](https://github.com/aws/aws-lambda-python-runtime-interface-client), and everything can be copied into
+the same directory in the image.
 
 ## AWS Lambda Runtime Interface Emulator
 
@@ -55,8 +55,7 @@ curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d
 
 ## Lambda Function Code
 
-[src/lambda-poc](src/lambda-poc) contains a basic function that returns HTTP 200, printing the Python version and lambda
-event payload.
+[src/lambda-poc](src/lambda-poc) contains a basic function that returns HTTP 200 and some json.
 
 ## AWS Infrastructure
 

infra/tf/.terraform.lock.hcl

@@ -10,7 +10,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "5.27.0"
+      version = "5.30.0"
     }
   }
 }

infra/tf/main.tf

@@ -1,15 +1,12 @@
 #!/bin/sh
-# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 if [ $# -ne 1 ]; then
   echo "entrypoint requires the handler name to be the first argument" 1>&2
   exit 142
 fi
-export _HANDLER="$1"
 
-RUNTIME_ENTRYPOINT=/var/runtime/bootstrap
 if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then
-  exec /usr/local/bin/aws-lambda-rie $RUNTIME_ENTRYPOINT
+  exec /usr/local/bin/aws-lambda-rie /var/lang/bin/python -m awslambdaric --log-level "debug" "$@"
 else
-  exec $RUNTIME_ENTRYPOINT
-fi
+  exec /var/lang/bin/python -m awslambdaric "$@"
+fi