feat: bake gapic-generator-java into the hermetic build docker image (#3067)

Makes the gapic-generator-java jar a prepared binary in the Docker image
whose location is now assumed by the scripts. Developers need now to
prepare this well-know location (specified in
`library_generation/DEVELOPMENT.md`.

---------

Co-authored-by: Blake Li <[email protected]>
Co-authored-by: Joe Wang <[email protected]>

Author: 3 people

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -12,11 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# install gapic-generator-java in a separate layer so we don't overload the image
+# with the transferred source code and jars
+FROM gcr.io/cloud-devrel-public-resources/java21 AS ggj-build
+
+WORKDIR /sdk-platform-java
+COPY . .
+# {x-version-update-start:gapic-generator-java:current}
+ENV DOCKER_GAPIC_GENERATOR_VERSION="2.44.1-SNAPSHOT" 
+# {x-version-update-end:gapic-generator-java:current}
+
+RUN mvn install -DskipTests -Dclirr.skip -Dcheckstyle.skip
+RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \
+  "./gapic-generator-java.jar"
+
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
 SHELL [ "/bin/bash", "-c" ]
 
+
 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
 ARG PROTOC_VERSION=25.4
 ARG GRPC_VERSION=1.66.0
@@ -47,7 +62,15 @@ RUN source /src/utils/utilities.sh \
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
 ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 
-# use python 3.11 (the base image has several python versions; here we define the default one)
+
+# Here we transfer gapic-generator-java from the previous stage.
+# Note that the destination is a well-known location that will be assumed at runtime
+# We hard-code the location string to avoid making it configurable (via ARG) as
+# well as to avoid it making it overridable at runtime (via ENV).
+COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
+RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
+
+#  use python 3.11 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
 RUN ln -s $(which python3.11) /usr/local/bin/python
 RUN ln -s $(which python3.11) /usr/local/bin/python3

.github/workflows/ci.yaml

@@ -232,6 +232,17 @@ jobs:
       - name: Showcase golden tests
         working-directory: showcase
         run: |
+          # The golden test directly calls
+          # library_generation/generate_library.sh, which expects the jar to be
+          # located in its well-known location. More info in
+          # library_generation/DEVELOPMENT.md
+          # Here we prepare the jar in such location
+          generator_version=$(grep "gapic-generator-java:" "../versions.txt" \
+            | cut -d: -f3) # the 3rd field is the snapshot version
+          mkdir -p "${HOME}/.library_generation"
+          cp \
+            "${HOME}/.m2/repository/com/google/api/gapic-generator-java/${generator_version}/gapic-generator-java-${generator_version}.jar" \
+            "${HOME}/.library_generation/gapic-generator-java.jar"
           mvn test \
             -P enable-golden-tests \
             --batch-mode \

.gitignore

@@ -24,3 +24,5 @@ target/
 **/*egg-info/
 **/build/
 **/dist/
+library_generation/**/*.jar
+

.kokoro/presubmit/downstream-compatibility-spring.sh

@@ -34,8 +34,8 @@ git clone "https://github.com/GoogleCloudPlatform/spring-cloud-gcp.git" --depth=
 update_all_poms_dependency "spring-cloud-gcp" "gapic-generator-java-bom" "${GAPIC_GENERATOR_VERSION}"
 
 # Install spring-cloud-gcp modules
-pushd spring-cloud-gcp/spring-cloud-generator
-../mvnw \
+pushd spring-cloud-gcp
+./mvnw \
   -U \
   --batch-mode \
   --no-transfer-progress \
@@ -47,10 +47,16 @@ pushd spring-cloud-gcp/spring-cloud-generator
 
 
 # Generate showcase autoconfig
+pushd spring-cloud-generator
+# The script is not executable for non-owners. Here we manually chmod it.
+# TODO(diegomarquezp): remove this line after
+# https://github.com/GoogleCloudPlatform/spring-cloud-gcp/pull/3183 is merged and released.
+chmod 755 ./scripts/generate-showcase.sh
 ./scripts/generate-showcase.sh
 pushd showcase/showcase-spring-starter
 mvn verify
 popd # showcase/showcase-spring-starter
 
-popd # spring-cloud-gcp/spring-cloud-generator
+popd # spring-cloud-generator
+popd # spring-cloud-gcp
 popd # gapic-generator-java/target

library_generation/DEVELOPMENT.md

@@ -4,8 +4,7 @@
 
 # Linting
 
-When contributing, ensure your changes to python code have a valid
-format.
+When contributing, ensure your changes to python code have a valid format.
 
 ```
 python -m pip install black
@@ -30,9 +29,9 @@ python -m unittest test/integration_tests.py
 # Running the unit tests
 
 The unit tests of the hermetic build scripts are contained in several scripts,
-corresponding to a specific component. Every unit test script ends with
-`unit_tests.py`. To avoid them specifying them
-individually, we can use the following command:
+corresponding to a specific component.
+Every unit test script ends with `unit_tests.py`.
+To avoid them specifying them individually, we can use the following command:
 
 ```bash
 python -m unittest discover -s test/ -p "*unit_tests.py"
@@ -45,35 +44,41 @@ python -m unittest discover -s test/ -p "*unit_tests.py"
 # Running the scripts in your local environment
 
 Although the scripts are designed to be run in a Docker container, you can also
-run them directly. This section explains how to run the entrypoint script
+run them directly.
+This section explains how to run the entrypoint script
 (`library_generation/cli/entry_point.py`).
 
-## Installing prerequisites
+## Assumptions made by the scripts
+### The Hermetic Build's well-known folder
+Located in `${HOME}/.library_generation`, this folder is assumed by the scripts
+to contain the generator JAR. 
+Please note that this is a recent feature and only this jar is expected to be
+there. 
+Developers must make sure this folder is properly configured before running the
+scripts locally.
+Note that this relies on the `HOME` en var which is always defined as per
+[POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html).
 
-In order to run the generation scripts directly, there are a few tools we
-need to install beforehand.
+#### Put the gapic-generator-java jar in its well-known location
 
-### Install synthtool
+Run `cd sdk-platform-java && mvn install -DskipTests -Dclirr.skip
+-Dcheckstyle.skip`. 
+This will generate a jar located in
+`~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar`
 
-It requires python 3.x to be installed.
-You will need to specify a committish of the synthtool repo in order to have
-your generation results matching exactly what the docker image would produce.
-You can achieve this by inspecting `SYNTHTOOL_COMMITISH` in
-`.cloudbuild/library_generation/library_generation.Dockerfile`.
+Then `mv` the jar into the well-known location of the jar. 
+The generation scripts will assume the jar is there.
 
-```bash
-# obtained from .cloudbuild/library_generation/library_generation.Dockerfile
-export SYNTHTOOL_COMMITTISH=6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
+```shell
+mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar"
 ```
 
-```bash
-git clone https://github.com/googleapis/synthtool
-cd synthtool
-git checkout "${SYNTHTOOL_COMMITTISH}"
-python -m pip install --require-hashes -r requirements.txt
-python -m pip install --no-deps -e .
-python -m synthtool --version
-```
+
+
+## Installing prerequisites
+
+In order to run the generation scripts directly, there are a few tools we
+need to install beforehand.
 
 ### Install the owl-bot CLI
 
@@ -93,6 +98,7 @@ owl-bot copy-code --version
 The key step is `npm link`, which will make the command available in you current
 shell session.
 
+
 ## Running the script
 The entrypoint script (`library_generation/cli/entry_point.py`) allows you to
 update the target repository with the latest changes starting from the
@@ -132,9 +138,9 @@ This will create an `image-id` file at the root of the repo with the hash ID of
 the image.
 
 ## Run the docker image
-The docker image will perform changes on its internal `/workspace` folder, to which you
-need to map a folder on your host machine (i.e. map your downloaded repo to this
-folder).
+The docker image will perform changes on its internal `/workspace` folder, 
+to which you need to map a folder on your host machine (i.e. map your downloaded
+repo to this folder).
 
 To run the docker container on the google-cloud-java repo, you must run:
 ```bash
@@ -151,9 +157,9 @@ docker run -u "$(id -u)":"$(id -g)"  -v/path/to/google-cloud-java:/workspace $(c
 
 ## Debug the created containers
 If you are working on changing the way the containers are created, you may want
-to inspect the containers to check the setup. It would be convenient in such
-case to have a text editor/viewer available. You can achieve this by modifying
-the Dockerfile as follows:
+to inspect the containers to check the setup.
+It would be convenient in such case to have a text editor/viewer available. 
+You can achieve this by modifying the Dockerfile as follows:
 
 ```docker
 # install OS tools

library_generation/gapic-generator-java-wrapper

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
-
 set -e
+wrapper_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+source "${wrapper_dir}/utils/utilities.sh"
 
 # Wrap gapic-generator-java.jar because protoc requires the plugin to be executable.
-exec java -classpath "gapic-generator-java-${gapic_generator_version}.jar" com.google.api.generator.Main
+exec java -classpath "$(get_gapic_generator_location)" com.google.api.generator.Main

library_generation/generate_composed_library.py

@@ -131,7 +131,6 @@ def __construct_tooling_arg(config: GenerationConfig) -> List[str]:
     :return: arguments containing tooling versions
     """
     arguments = []
-    arguments += util.create_argument("gapic_generator_version", config)
     arguments += util.create_argument("grpc_version", config)
     arguments += util.create_argument("protoc_version", config)
 

library_generation/generate_library.sh

@@ -14,12 +14,6 @@ case $key in
     destination_path="$2"
     shift
     ;;
-  --gapic_generator_version)
-    gapic_generator_version="$2"
-    # export this variable so that it can be used in gapic-generator-java-wrapper.sh
-    export gapic_generator_version
-    shift
-    ;;
   --protoc_version)
     protoc_version="$2"
     shift
@@ -77,17 +71,12 @@ script_dir=$(dirname "$(readlink -f "$0")")
 source "${script_dir}"/utils/utilities.sh
 output_folder="$(get_output_folder)"
 
-if [ -z "${gapic_generator_version}" ]; then
-  echo 'missing required argument --gapic_generator_version'
-  exit 1
-fi
-
 if [ -z "${protoc_version}" ]; then
-  protoc_version=$(get_protoc_version "${gapic_generator_version}")
+  protoc_version=$(get_protoc_version)
 fi
 
 if [ -z "${grpc_version}" ]; then
-  grpc_version=$(get_grpc_version "${gapic_generator_version}")
+  grpc_version=$(get_grpc_version)
 fi
 
 if [ -z "${proto_only}" ]; then
@@ -185,7 +174,7 @@ esac
 # download gapic-generator-java, protobuf and grpc plugin.
 # the download_tools function will create the environment variables "protoc_path"
 # and "grpc_path", to be used in the protoc calls below.
-download_tools "${gapic_generator_version}" "${protoc_version}" "${grpc_version}" "${os_architecture}"
+download_tools "${protoc_version}" "${grpc_version}" "${os_architecture}"
 ##################### Section 1 #####################
 # generate grpc-*/
 #####################################################

library_generation/test/generate_library_unit_tests.py

@@ -0,0 +1,96 @@
+import subprocess
+import unittest
+import os
+from library_generation.utils.utilities import (
+    run_process_and_print_output as bash_call,
+    run_process_and_get_output_string as get_bash_call_output,
+)
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+
+class GenerateLibraryUnitTests(unittest.TestCase):
+    """
+    Confirms the correct behavior of `library_generation/utils/utilities.sh`.
+
+    Note that there is an already existing, shell-based, test suite for
+    generate_library.sh, but these tests will soon be transferred to this one as
+    an effort to unify the implementation of the Hermetic Build scripts as
+    python-only. New tests for `utilities.sh` should be added in this file.
+    """
+
+    TEST_ARCHITECTURE = "linux-x86_64"
+
+    def setUp(self):
+        # we create a simulated home folder that has a fake generator jar
+        # in its well-known location
+        self.simulated_home = get_bash_call_output("mktemp -d")
+        bash_call(f"mkdir {self.simulated_home}/.library_generation")
+        bash_call(
+            f"touch {self.simulated_home}/.library_generation/gapic-generator-java.jar"
+        )
+
+        # We create a per-test directory where all output files will be created into.
+        # Each folder will be deleted after its corresponding test finishes.
+        test_dir = get_bash_call_output("mktemp -d")
+        self.output_folder = self._run_command_and_get_sdout(
+            "get_output_folder",
+            cwd=test_dir,
+        )
+        bash_call(f"mkdir {self.output_folder}")
+
+    def tearDown(self):
+        bash_call(f"rm -rdf {self.simulated_home}")
+
+    def _run_command(self, command, **kwargs):
+        env = os.environ.copy()
+        env["HOME"] = self.simulated_home
+        if "cwd" not in kwargs:
+            kwargs["cwd"] = self.output_folder
+        return bash_call(
+            [
+                "bash",
+                "-exc",
+                f"source {script_dir}/../utils/utilities.sh " + f"&& {command}",
+            ],
+            exit_on_fail=False,
+            env=env,
+            **kwargs,
+        )
+
+    def _run_command_and_get_sdout(self, command, **kwargs):
+        return self._run_command(
+            command, stderr=subprocess.PIPE, **kwargs
+        ).stdout.decode()[:-1]
+
+    def test_get_grpc_version_with_no_env_var_fails(self):
+        # the absence of DOCKER_GRPC_VERSION will make this function to fail
+        result = self._run_command("get_grpc_version")
+        self.assertEquals(1, result.returncode)
+        self.assertRegex(result.stdout.decode(), "DOCKER_GRPC_VERSION is not set")
+
+    def test_get_protoc_version_with_no_env_var_fails(self):
+        # the absence of DOCKER_PROTOC_VERSION will make this function to fail
+        result = self._run_command("get_protoc_version")
+        self.assertEquals(1, result.returncode)
+        self.assertRegex(result.stdout.decode(), "DOCKER_PROTOC_VERSION is not set")
+
+    def test_download_tools_without_baked_generator_fails(self):
+        # This test has the same structure as
+        # download_tools_succeed_with_baked_protoc, but meant for
+        # gapic-generator-java.
+
+        test_protoc_version = "1.64.0"
+        test_grpc_version = "1.64.0"
+        jar_location = (
+            f"{self.simulated_home}/.library_generation/gapic-generator-java.jar"
+        )
+        # we expect the function to fail because the generator jar is not found in
+        # its well-known location. To achieve this, we temporarily remove the fake
+        # generator jar
+        bash_call(f"rm {jar_location}")
+        result = self._run_command(
+            f"download_tools {test_protoc_version} {test_grpc_version} {self.TEST_ARCHITECTURE}"
+        )
+        self.assertEquals(1, result.returncode)
+        self.assertRegex(result.stdout.decode(), "Please configure your environment")