From 599175660eee7434851c70ee6fb6a4ba902a4543 Mon Sep 17 00:00:00 2001 From: Muhammad Mahad Date: Sun, 23 Mar 2025 07:02:56 +0000 Subject: [PATCH 1/8] Add Go configuration for base images and deployment settings --- config/systems.json | 97 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/config/systems.json b/config/systems.json index 5a4077a2..bbfadde4 100644 --- a/config/systems.json +++ b/config/systems.json @@ -64,6 +64,26 @@ ], "packages": [] } + }, + "go": { + "base_images": { + "x64": { + "1.22": "golang:1.22", + "1.23": "golang:1.23", + "1.24": "golang:1.24" + } + }, + "images": [ + "build" + ], + "username": "docker_user", + "deployment": { + "files": [ + "handler.go", + "storage.go" + ], + "packages": [] + } } }, "architecture": ["x64"], @@ -121,6 +141,23 @@ "uuid": "3.4.0" } } + }, + "golang": { + "base_images": { + "x64": { + "1" : "amazon/aws-lambda-go:1.2024.10.04.19" + } + }, + "images": [ + "build" + ], + "deployment": { + "files": [ + "handler.go", + "storage.go" + ], + "packages": [] + } } }, "architecture": ["x64", "arm64"], @@ -181,6 +218,26 @@ "uuid": "3.4.0" } } + }, + "golang": { + "base_images": { + "x64": { + "1.22": "mcr.microsoft.com/devcontainers/go:1.4-1.22", + "1.23": "mcr.microsoft.com/devcontainers/go:1.4-1.23", + "1.24": "mcr.microsoft.com/devcontainers/go:1.4-1.24" + } + }, + "images": [ + "build" + ], + "username": "docker_user", + "deployment": { + "files": [ + "handler.go", + "storage.go" + ], + "packages": [] + } } }, "images": { @@ -250,6 +307,26 @@ "uuid": "3.4.0" } } + }, + "golang": { + "base_images": { + "x64": { + "1.22": "ubuntu:22.04", + "1.23": "ubuntu:22.04", + "1.24": "ubuntu:22.04" + } + }, + "images": [ + "build" + ], + "username": "docker_user", + "deployment": { + "files": [ + "handler.go", + "storage.go" + ], + "packages": [] + } } }, "images": { @@ -316,6 +393,26 @@ "minio": "7.0.16" } } + }, + "golang": { + "base_images": { + "x64": { + "1.21": "openwhisk/action-golang-v1.21", + "1.22": "openwhisk/action-golang-v1.22", + "1.23": "openwhisk/action-golang-v1.23" + } + }, + "images": [ + "build" + ], + "username": "docker_user", + "deployment": { + "files": [ + "handler.go", + "storage.go" + ], + "packages": [] + } } }, "architecture": ["x64"], From 6eeb39323e92c1cdd481435e2eb67a2ff112900b Mon Sep 17 00:00:00 2001 From: Muhammad Mahad Date: Sun, 23 Mar 2025 07:26:59 +0000 Subject: [PATCH 2/8] Add Go Dockerfiles for multiple platform builds and add installer script --- dockerfiles/aws/golang/Dockerfile.build | 23 +++++++++++++++++++++++ dockerfiles/azure/golang/Dockerfile.build | 17 +++++++++++++++++ dockerfiles/gcp/golang/Dockerfile.build | 17 +++++++++++++++++ dockerfiles/golang_installer.sh | 5 +++++ dockerfiles/local/golang/Dockerfile.build | 19 +++++++++++++++++++ sebs/config.py | 3 --- 6 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 dockerfiles/aws/golang/Dockerfile.build create mode 100644 dockerfiles/azure/golang/Dockerfile.build create mode 100644 dockerfiles/gcp/golang/Dockerfile.build create mode 100644 dockerfiles/golang_installer.sh create mode 100644 dockerfiles/local/golang/Dockerfile.build diff --git a/dockerfiles/aws/golang/Dockerfile.build b/dockerfiles/aws/golang/Dockerfile.build new file mode 100644 index 00000000..a14980db --- /dev/null +++ b/dockerfiles/aws/golang/Dockerfile.build @@ -0,0 +1,23 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +####################################### +# useradd, groupmod +RUN yum install -y shadow-utils +ENV GOSU_VERSION 1.14 +# https://github.com/tianon/gosu/releases/tag/1.14 +# key https://keys.openpgp.org/search?q=tianon%40debian.org +RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64" \ + && chmod +x /usr/local/bin/gosu + + +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/entrypoint.sh + +# useradd and groupmod is installed in /usr/sbin which is not in PATH +ENV PATH=/usr/sbin:$PATH +ENV SCRIPT_FILE=/mnt/function/package.sh +CMD /bin/bash /sebs/installer.sh +ENTRYPOINT ["/sebs/entrypoint.sh"] \ No newline at end of file diff --git a/dockerfiles/azure/golang/Dockerfile.build b/dockerfiles/azure/golang/Dockerfile.build new file mode 100644 index 00000000..afc64ebe --- /dev/null +++ b/dockerfiles/azure/golang/Dockerfile.build @@ -0,0 +1,17 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +RUN apt-get update\ + && apt-get install -y --no-install-recommends zip gosu golang-1.23\ + && apt-get purge -y --auto-remove + +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/entrypoint.sh + +# useradd and groupmod is installed in /usr/sbin which is not in PATH +ENV PATH=/usr/sbin:$PATH +ENV SCRIPT_FILE=/mnt/function/package.sh +CMD /bin/bash /sebs/installer.sh +ENTRYPOINT ["/sebs/entrypoint.sh"] \ No newline at end of file diff --git a/dockerfiles/gcp/golang/Dockerfile.build b/dockerfiles/gcp/golang/Dockerfile.build new file mode 100644 index 00000000..afc64ebe --- /dev/null +++ b/dockerfiles/gcp/golang/Dockerfile.build @@ -0,0 +1,17 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +RUN apt-get update\ + && apt-get install -y --no-install-recommends zip gosu golang-1.23\ + && apt-get purge -y --auto-remove + +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/entrypoint.sh + +# useradd and groupmod is installed in /usr/sbin which is not in PATH +ENV PATH=/usr/sbin:$PATH +ENV SCRIPT_FILE=/mnt/function/package.sh +CMD /bin/bash /sebs/installer.sh +ENTRYPOINT ["/sebs/entrypoint.sh"] \ No newline at end of file diff --git a/dockerfiles/golang_installer.sh b/dockerfiles/golang_installer.sh new file mode 100644 index 00000000..581cff82 --- /dev/null +++ b/dockerfiles/golang_installer.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd /mnt/function + +go mod tidy diff --git a/dockerfiles/local/golang/Dockerfile.build b/dockerfiles/local/golang/Dockerfile.build new file mode 100644 index 00000000..1e18bf70 --- /dev/null +++ b/dockerfiles/local/golang/Dockerfile.build @@ -0,0 +1,19 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + + +RUN apt-get update\ + && apt-get install -y --no-install-recommends zip gosu golang-1.23\ + && apt-get purge -y --auto-remove + +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/entrypoint.sh + + +# useradd and groupmod is installed in /usr/sbin which is not in PATH +ENV PATH=/usr/sbin:$PATH +ENV SCRIPT_FILE=/mnt/function/package.sh +CMD /bin/bash /sebs/installer.sh +ENTRYPOINT ["/sebs/entrypoint.sh"] \ No newline at end of file diff --git a/sebs/config.py b/sebs/config.py index aa20a8e3..c3030ea0 100644 --- a/sebs/config.py +++ b/sebs/config.py @@ -46,9 +46,6 @@ def supported_language_versions( ) -> List[str]: languages = self._system_config.get(deployment_name, {}).get("languages", {}) base_images = languages.get(language_name, {}).get("base_images", {}) - - if deployment_name == "local": - return list(base_images.keys()) return list(base_images.get(architecture, {}).keys()) def supported_architecture(self, deployment_name: str) -> List[str]: From 79acf69e6890a640a69ee9433b88c66e3a300961 Mon Sep 17 00:00:00 2001 From: Muhammad Mahad Date: Sun, 23 Mar 2025 08:04:20 +0000 Subject: [PATCH 3/8] Add Golang in Docker build scripts --- dockerfiles/openwhisk/golang/Dockerfile.function | 0 sebs/benchmark.py | 11 +++++++++-- sebs/faas/function.py | 1 + tools/build_docker_images.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 dockerfiles/openwhisk/golang/Dockerfile.function diff --git a/dockerfiles/openwhisk/golang/Dockerfile.function b/dockerfiles/openwhisk/golang/Dockerfile.function new file mode 100644 index 00000000..e69de29b diff --git a/sebs/benchmark.py b/sebs/benchmark.py index 42adb4e7..37314614 100644 --- a/sebs/benchmark.py +++ b/sebs/benchmark.py @@ -252,8 +252,9 @@ def hash_directory(directory: str, deployment: str, language: str): FILES = { "python": ["*.py", "requirements.txt*"], "nodejs": ["*.js", "package.json"], + "golang": ["*.go"], } - WRAPPERS = {"python": "*.py", "nodejs": "*.js"} + WRAPPERS = {"python": "*.py", "nodejs": "*.js", "golang": "*.go"} NON_LANG_FILES = ["*.sh", "*.json"] selected_files = FILES[language] + NON_LANG_FILES for file_type in selected_files: @@ -316,6 +317,7 @@ def copy_code(self, output_dir): FILES = { "python": ["*.py", "requirements.txt*"], "nodejs": ["*.js", "package.json"], + "golang": ["*.go", "*.mod"], } path = os.path.join(self.benchmark_path, self.language_name) for file_type in FILES[self.language_name]: @@ -399,6 +401,9 @@ def add_deployment_package_nodejs(self, output_dir): with open(package_config, "w") as package_file: json.dump(package_json, package_file, indent=2) + def add_deployment_package_golang(self, output_dir): + pass + def add_deployment_package(self, output_dir): from sebs.faas.function import Language @@ -406,6 +411,8 @@ def add_deployment_package(self, output_dir): self.add_deployment_package_python(output_dir) elif self.language == Language.NODEJS: self.add_deployment_package_nodejs(output_dir) + elif self.language == Language.GOLANG: + self.add_deployment_package_golang(output_dir) else: raise NotImplementedError @@ -463,7 +470,7 @@ def install_dependencies(self, output_dir): } # run Docker container to install packages - PACKAGE_FILES = {"python": "requirements.txt", "nodejs": "package.json"} + PACKAGE_FILES = {"python": "requirements.txt", "nodejs": "package.json", "golang": "go.mod"} file = os.path.join(output_dir, PACKAGE_FILES[self.language_name]) if os.path.exists(file): try: diff --git a/sebs/faas/function.py b/sebs/faas/function.py index 0fab7bcf..f53683b0 100644 --- a/sebs/faas/function.py +++ b/sebs/faas/function.py @@ -263,6 +263,7 @@ def deserialize(cached_config: dict) -> "Trigger": class Language(Enum): PYTHON = "python" NODEJS = "nodejs" + GOLANG = "golang" # FIXME: 3.7+ python with future annotations @staticmethod diff --git a/tools/build_docker_images.py b/tools/build_docker_images.py index 5336fb48..9fee91c8 100755 --- a/tools/build_docker_images.py +++ b/tools/build_docker_images.py @@ -13,7 +13,7 @@ "--deployment", default=None, choices=["local", "aws", "azure", "gcp"], action="store" ) parser.add_argument("--type", default=None, choices=["build", "run", "manage"], action="store") -parser.add_argument("--language", default=None, choices=["python", "nodejs"], action="store") +parser.add_argument("--language", default=None, choices=["python", "nodejs", "golang"], action="store") parser.add_argument("--language-version", default=None, type=str, action="store") args = parser.parse_args() config = json.load(open(os.path.join(PROJECT_DIR, "config", "systems.json"), "r")) From 99af2d14947a003913cef26ebd5614d0b88384ae Mon Sep 17 00:00:00 2001 From: Muhammad Mahad Date: Mon, 28 Apr 2025 15:23:53 +0000 Subject: [PATCH 4/8] Add go installation from official website and successful image build --- .dockerignore | 3 ++ dockerfiles/local/golang/Dockerfile.build | 43 ++++++++++++++++++----- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index 84416f19..8fc6b701 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,6 @@ cache python-venv regression-* *_code + +# ignore, will building docker image +scylladb-volume/ \ No newline at end of file diff --git a/dockerfiles/local/golang/Dockerfile.build b/dockerfiles/local/golang/Dockerfile.build index 1e18bf70..83034624 100644 --- a/dockerfiles/local/golang/Dockerfile.build +++ b/dockerfiles/local/golang/Dockerfile.build @@ -1,19 +1,44 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} +# Define Go version as build argument with default +ARG GO_VERSION=1.23.0 +ARG TARGETARCH=amd64 -RUN apt-get update\ - && apt-get install -y --no-install-recommends zip gosu golang-1.23\ - && apt-get purge -y --auto-remove +# Install dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + tar \ + zip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" + +# Verify installation +RUN go version + +# Set up SeBS environment RUN mkdir -p /sebs/ COPY dockerfiles/golang_installer.sh /sebs/installer.sh COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh -RUN chmod +x /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh +# Add /usr/sbin to PATH (for useradd and groupmod) +ENV PATH="/usr/sbin:${PATH}" +ENV SCRIPT_FILE="/mnt/function/package.sh" -# useradd and groupmod is installed in /usr/sbin which is not in PATH -ENV PATH=/usr/sbin:$PATH -ENV SCRIPT_FILE=/mnt/function/package.sh -CMD /bin/bash /sebs/installer.sh -ENTRYPOINT ["/sebs/entrypoint.sh"] \ No newline at end of file +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file From 6882cf58abfced7fe4ca95b932eaaff5985c0682 Mon Sep 17 00:00:00 2001 From: Muhammad Mahad Date: Mon, 28 Apr 2025 15:26:01 +0000 Subject: [PATCH 5/8] Fix typo "go" to "golang" in systems configuration for consistency --- config/systems.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/systems.json b/config/systems.json index bbfadde4..fd3a0bfa 100644 --- a/config/systems.json +++ b/config/systems.json @@ -65,7 +65,7 @@ "packages": [] } }, - "go": { + "golang": { "base_images": { "x64": { "1.22": "golang:1.22", From b42393763ec52b044bda96acb133d037c6843a81 Mon Sep 17 00:00:00 2001 From: Muhammad Mahad Date: Mon, 28 Apr 2025 15:36:41 +0000 Subject: [PATCH 6/8] Update aws, azure, gcp, and openwhisk go installation from official source and fix typo --- .dockerignore | 2 +- dockerfiles/aws/golang/Dockerfile.build | 34 +++++++++++--- dockerfiles/azure/golang/Dockerfile.build | 42 ++++++++++++++--- dockerfiles/gcp/golang/Dockerfile.build | 42 ++++++++++++++--- dockerfiles/openwhisk/golang/Dockerfile.build | 45 +++++++++++++++++++ 5 files changed, 144 insertions(+), 21 deletions(-) create mode 100644 dockerfiles/openwhisk/golang/Dockerfile.build diff --git a/.dockerignore b/.dockerignore index 8fc6b701..c8d2d823 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,5 +8,5 @@ python-venv regression-* *_code -# ignore, will building docker image +# ignore, while building docker image scylladb-volume/ \ No newline at end of file diff --git a/dockerfiles/aws/golang/Dockerfile.build b/dockerfiles/aws/golang/Dockerfile.build index a14980db..256bda00 100644 --- a/dockerfiles/aws/golang/Dockerfile.build +++ b/dockerfiles/aws/golang/Dockerfile.build @@ -1,23 +1,45 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} +# Define Go version as build argument with default +ARG GO_VERSION=1.23.0 +ARG TARGETARCH=amd64 + ####################################### -# useradd, groupmod -RUN yum install -y shadow-utils +# Install required utilities +RUN yum install -y shadow-utils wget ca-certificates tar zip + +# Install gosu for running commands as another user ENV GOSU_VERSION 1.14 # https://github.com/tianon/gosu/releases/tag/1.14 # key https://keys.openpgp.org/search?q=tianon%40debian.org RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64" \ && chmod +x /usr/local/bin/gosu +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" +# Verify installation +RUN go version + +# Set up SeBS environment RUN mkdir -p /sebs/ COPY dockerfiles/golang_installer.sh /sebs/installer.sh COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh -RUN chmod +x /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh # useradd and groupmod is installed in /usr/sbin which is not in PATH -ENV PATH=/usr/sbin:$PATH +ENV PATH=/usr/sbin:${PATH} ENV SCRIPT_FILE=/mnt/function/package.sh -CMD /bin/bash /sebs/installer.sh -ENTRYPOINT ["/sebs/entrypoint.sh"] \ No newline at end of file + +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file diff --git a/dockerfiles/azure/golang/Dockerfile.build b/dockerfiles/azure/golang/Dockerfile.build index afc64ebe..34e1f0ee 100644 --- a/dockerfiles/azure/golang/Dockerfile.build +++ b/dockerfiles/azure/golang/Dockerfile.build @@ -1,17 +1,45 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -RUN apt-get update\ - && apt-get install -y --no-install-recommends zip gosu golang-1.23\ - && apt-get purge -y --auto-remove +# Define Go version as build argument with default +ARG GO_VERSION=1.23.0 +ARG TARGETARCH=amd64 +# Install dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + tar \ + zip \ + gosu \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" + +# Verify installation +RUN go version + +# Set up SeBS environment RUN mkdir -p /sebs/ COPY dockerfiles/golang_installer.sh /sebs/installer.sh COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh -RUN chmod +x /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh # useradd and groupmod is installed in /usr/sbin which is not in PATH -ENV PATH=/usr/sbin:$PATH +ENV PATH=/usr/sbin:${PATH} ENV SCRIPT_FILE=/mnt/function/package.sh -CMD /bin/bash /sebs/installer.sh -ENTRYPOINT ["/sebs/entrypoint.sh"] \ No newline at end of file + +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file diff --git a/dockerfiles/gcp/golang/Dockerfile.build b/dockerfiles/gcp/golang/Dockerfile.build index afc64ebe..34e1f0ee 100644 --- a/dockerfiles/gcp/golang/Dockerfile.build +++ b/dockerfiles/gcp/golang/Dockerfile.build @@ -1,17 +1,45 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -RUN apt-get update\ - && apt-get install -y --no-install-recommends zip gosu golang-1.23\ - && apt-get purge -y --auto-remove +# Define Go version as build argument with default +ARG GO_VERSION=1.23.0 +ARG TARGETARCH=amd64 +# Install dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + tar \ + zip \ + gosu \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" + +# Verify installation +RUN go version + +# Set up SeBS environment RUN mkdir -p /sebs/ COPY dockerfiles/golang_installer.sh /sebs/installer.sh COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh -RUN chmod +x /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh # useradd and groupmod is installed in /usr/sbin which is not in PATH -ENV PATH=/usr/sbin:$PATH +ENV PATH=/usr/sbin:${PATH} ENV SCRIPT_FILE=/mnt/function/package.sh -CMD /bin/bash /sebs/installer.sh -ENTRYPOINT ["/sebs/entrypoint.sh"] \ No newline at end of file + +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file diff --git a/dockerfiles/openwhisk/golang/Dockerfile.build b/dockerfiles/openwhisk/golang/Dockerfile.build new file mode 100644 index 00000000..79b55e31 --- /dev/null +++ b/dockerfiles/openwhisk/golang/Dockerfile.build @@ -0,0 +1,45 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# Define Go version as build argument with default +ARG GO_VERSION=1.23.0 +ARG TARGETARCH=amd64 + +# Install dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + tar \ + zip \ + gosu \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" + +# Verify installation +RUN go version + +# Set up SeBS environment +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh + +# Add /usr/sbin to PATH (for useradd and groupmod) +ENV PATH="/usr/sbin:${PATH}" +ENV SCRIPT_FILE="/mnt/function/package.sh" + +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file From 43b47291ed047fd28a6bf51cab36e649891d2a71 Mon Sep 17 00:00:00 2001 From: Muhammad Mahad Date: Fri, 2 May 2025 11:57:07 +0000 Subject: [PATCH 7/8] Update Go version handling in Dockerfiles and systems configuration for consistency --- config/systems.json | 24 +++++++++---------- dockerfiles/aws/golang/Dockerfile.build | 5 ++-- dockerfiles/azure/golang/Dockerfile.build | 5 ++-- dockerfiles/gcp/golang/Dockerfile.build | 5 ++-- dockerfiles/local/golang/Dockerfile.build | 3 ++- dockerfiles/openwhisk/golang/Dockerfile.build | 5 ++-- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/config/systems.json b/config/systems.json index fd3a0bfa..de9977d0 100644 --- a/config/systems.json +++ b/config/systems.json @@ -68,9 +68,9 @@ "golang": { "base_images": { "x64": { - "1.22": "golang:1.22", - "1.23": "golang:1.23", - "1.24": "golang:1.24" + "1.22.0": "golang:1.22", + "1.23.0": "golang:1.23", + "1.24.0": "golang:1.24" } }, "images": [ @@ -222,9 +222,9 @@ "golang": { "base_images": { "x64": { - "1.22": "mcr.microsoft.com/devcontainers/go:1.4-1.22", - "1.23": "mcr.microsoft.com/devcontainers/go:1.4-1.23", - "1.24": "mcr.microsoft.com/devcontainers/go:1.4-1.24" + "1.22.0": "mcr.microsoft.com/devcontainers/go:1.4-1.22", + "1.23.0": "mcr.microsoft.com/devcontainers/go:1.4-1.23", + "1.24.0": "mcr.microsoft.com/devcontainers/go:1.4-1.24" } }, "images": [ @@ -311,9 +311,9 @@ "golang": { "base_images": { "x64": { - "1.22": "ubuntu:22.04", - "1.23": "ubuntu:22.04", - "1.24": "ubuntu:22.04" + "1.22.0": "ubuntu:22.04", + "1.23.0": "ubuntu:22.04", + "1.24.0": "ubuntu:22.04" } }, "images": [ @@ -397,9 +397,9 @@ "golang": { "base_images": { "x64": { - "1.21": "openwhisk/action-golang-v1.21", - "1.22": "openwhisk/action-golang-v1.22", - "1.23": "openwhisk/action-golang-v1.23" + "1.21.0": "openwhisk/action-golang-v1.21", + "1.22.0": "openwhisk/action-golang-v1.22", + "1.23.0": "openwhisk/action-golang-v1.23" } }, "images": [ diff --git a/dockerfiles/aws/golang/Dockerfile.build b/dockerfiles/aws/golang/Dockerfile.build index 256bda00..404b0b43 100644 --- a/dockerfiles/aws/golang/Dockerfile.build +++ b/dockerfiles/aws/golang/Dockerfile.build @@ -1,8 +1,9 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -# Define Go version as build argument with default -ARG GO_VERSION=1.23.0 +# Define Go version from VERSION build argument +ARG VERSION +ENV GO_VERSION=${VERSION} ARG TARGETARCH=amd64 ####################################### diff --git a/dockerfiles/azure/golang/Dockerfile.build b/dockerfiles/azure/golang/Dockerfile.build index 34e1f0ee..b3810721 100644 --- a/dockerfiles/azure/golang/Dockerfile.build +++ b/dockerfiles/azure/golang/Dockerfile.build @@ -1,8 +1,9 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -# Define Go version as build argument with default -ARG GO_VERSION=1.23.0 +# Define Go version from VERSION build argument +ARG VERSION +ENV GO_VERSION=${VERSION} ARG TARGETARCH=amd64 # Install dependencies diff --git a/dockerfiles/gcp/golang/Dockerfile.build b/dockerfiles/gcp/golang/Dockerfile.build index 34e1f0ee..b3810721 100644 --- a/dockerfiles/gcp/golang/Dockerfile.build +++ b/dockerfiles/gcp/golang/Dockerfile.build @@ -1,8 +1,9 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -# Define Go version as build argument with default -ARG GO_VERSION=1.23.0 +# Define Go version from VERSION build argument +ARG VERSION +ENV GO_VERSION=${VERSION} ARG TARGETARCH=amd64 # Install dependencies diff --git a/dockerfiles/local/golang/Dockerfile.build b/dockerfiles/local/golang/Dockerfile.build index 83034624..64200067 100644 --- a/dockerfiles/local/golang/Dockerfile.build +++ b/dockerfiles/local/golang/Dockerfile.build @@ -2,7 +2,8 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} # Define Go version as build argument with default -ARG GO_VERSION=1.23.0 +ARG VERSION +ENV GO_VERSION=${VERSION} ARG TARGETARCH=amd64 # Install dependencies diff --git a/dockerfiles/openwhisk/golang/Dockerfile.build b/dockerfiles/openwhisk/golang/Dockerfile.build index 79b55e31..a2a8de49 100644 --- a/dockerfiles/openwhisk/golang/Dockerfile.build +++ b/dockerfiles/openwhisk/golang/Dockerfile.build @@ -1,8 +1,9 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -# Define Go version as build argument with default -ARG GO_VERSION=1.23.0 +# Define Go version from VERSION build argument +ARG VERSION +ENV GO_VERSION=${VERSION} ARG TARGETARCH=amd64 # Install dependencies From ff6d40d94dc0c2046f94c19a55a17f2335d8fb0f Mon Sep 17 00:00:00 2001 From: Muhammad Mahad Date: Fri, 2 May 2025 12:27:43 +0000 Subject: [PATCH 8/8] Add Golang storage implementation and update configuration for deployment --- .dockerignore | 3 - benchmarks/wrappers/local/golang/storage.go | 131 ++++++++++++++++++++ config/systems.json | 1 - dockerfiles/golang_installer.sh | 3 + sebs/local/local.py | 1 + 5 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 benchmarks/wrappers/local/golang/storage.go diff --git a/.dockerignore b/.dockerignore index c8d2d823..84416f19 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,3 @@ cache python-venv regression-* *_code - -# ignore, while building docker image -scylladb-volume/ \ No newline at end of file diff --git a/benchmarks/wrappers/local/golang/storage.go b/benchmarks/wrappers/local/golang/storage.go new file mode 100644 index 00000000..67941acd --- /dev/null +++ b/benchmarks/wrappers/local/golang/storage.go @@ -0,0 +1,131 @@ +package golang + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Storage struct { + client *minio.Client +} + +var instance *Storage + +func UniqueName(name string) string { + ext := filepath.Ext(name) + basename := name[:len(name)-len(ext)] + uuidStr := strings.Split(uuid.New().String(), "-")[0] + return fmt.Sprintf("%s.%s%s", basename, uuidStr, ext) +} + +func NewStorage() (*Storage, error) { + address, exists := os.LookupEnv("MINIO_ADDRESS") + if !exists { + return nil, nil + } + + accessKey := os.Getenv("MINIO_ACCESS_KEY") + secretKey := os.Getenv("MINIO_SECRET_KEY") + + parts := strings.Split(address, ":") + endpoint := parts[0] + var port int = 9000 // Default port + + if len(parts) > 1 { + var err error + port, err = strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("invalid port in MINIO_ADDRESS: %v", err) + } + } + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: false, + Port: uint16(port), + }) + if err != nil { + return nil, err + } + + return &Storage{client: client}, nil +} + +func (s *Storage) Upload(bucket, file, filepath string) (string, error) { + keyName := UniqueName(file) + _, err := s.client.FPutObject(context.Background(), bucket, keyName, filepath, minio.PutObjectOptions{}) + if err != nil { + return "", err + } + return keyName, nil +} + +func (s *Storage) Download(bucket, file, filepath string) error { + return s.client.FGetObject(context.Background(), bucket, file, filepath, minio.GetObjectOptions{}) +} + +func (s *Storage) DownloadDirectory(bucket, prefix, path string) error { + ctx := context.Background() + objects := s.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) + + for object := range objects { + if object.Err != nil { + return object.Err + } + + objectPath := filepath.Join(path, object.Key) + if err := os.MkdirAll(filepath.Dir(objectPath), 0755); err != nil { + return err + } + + if err := s.Download(bucket, object.Key, objectPath); err != nil { + return err + } + } + + return nil +} + +func (s *Storage) UploadStream(bucket, file string, data []byte) (string, error) { + keyName := UniqueName(file) + reader := bytes.NewReader(data) + _, err := s.client.PutObject(context.Background(), bucket, keyName, reader, int64(len(data)), minio.PutObjectOptions{}) + if err != nil { + return "", err + } + return keyName, nil +} + +func (s *Storage) DownloadStream(bucket, file string) ([]byte, error) { + obj, err := s.client.GetObject(context.Background(), bucket, file, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + return io.ReadAll(obj) +} + +func GetInstance() (*Storage, error) { + if instance == nil { + var err error + instance, err = NewStorage() + if err != nil { + return nil, err + } + } + return instance, nil +} \ No newline at end of file diff --git a/config/systems.json b/config/systems.json index de9977d0..7dc62f24 100644 --- a/config/systems.json +++ b/config/systems.json @@ -79,7 +79,6 @@ "username": "docker_user", "deployment": { "files": [ - "handler.go", "storage.go" ], "packages": [] diff --git a/dockerfiles/golang_installer.sh b/dockerfiles/golang_installer.sh index 581cff82..9e43086c 100644 --- a/dockerfiles/golang_installer.sh +++ b/dockerfiles/golang_installer.sh @@ -1,5 +1,8 @@ #!/bin/bash cd /mnt/function +go mod init github.com/spcl/serverless-benchmarks +go get github.com/minio/minio-go/v7 +go get github.com/google/uuid go mod tidy diff --git a/sebs/local/local.py b/sebs/local/local.py index 32b9f9ff..a57bf4f0 100644 --- a/sebs/local/local.py +++ b/sebs/local/local.py @@ -124,6 +124,7 @@ def package_code( CONFIG_FILES = { "python": ["handler.py", "requirements.txt", ".python_packages"], "nodejs": ["handler.js", "package.json", "node_modules"], + "golang": ["handler.go", "go.mod", "go.sum"], } package_config = CONFIG_FILES[language_name] function_dir = os.path.join(directory, "function")