Merge pull request #981 from Faless/build/3.x_qol_backports

[3.x] [SCons] Backport SCons generator, cache.

Author: akien-mga

.github/actions/godot-cache/action.yml

@@ -0,0 +1,22 @@
+name: Setup Godot build cache
+description: Setup Godot build cache.
+inputs:
+  cache-name:
+    description: The cache base name (job name by default).
+    default: "${{github.job}}"
+  scons-cache:
+    description: The scons cache path.
+    default: "${{github.workspace}}/.scons-cache/"
+runs:
+  using: "composite"
+  steps:
+    # Upload cache on completion and check it out now
+    - name: Load .scons_cache directory
+      uses: actions/cache@v3
+      with:
+        path: ${{inputs.scons-cache}}
+        key: ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+        restore-keys: |
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}

.github/workflows/ci.yml

@@ -1,6 +1,14 @@
 name: Continuous integration
 on: [push, pull_request]
 
+env:
+  # Only used for the cache key. Increment version to force clean build.
+  GODOT_BASE_BRANCH: 3.x
+
+concurrency:
+  group: ci-${{github.actor}}-${{github.head_ref || github.run_number}}-${{github.ref}}
+  cancel-in-progress: true
+
 jobs:
   build:
     name: ${{ matrix.name }}
@@ -16,19 +24,22 @@ jobs:
             artifact-path: bin/libgodot-cpp.linux.release.64.a
             godot_zip: Godot_v3.5-stable_linux_server.64.zip
             executable: Godot_v3.5-stable_linux_server.64
+            cache-name: linux-x86_64
 
           - name: 🏁 Windows (x86_64, MSVC)
             os: windows-2019
             platform: windows
             artifact-name: godot-cpp-windows-msvc2019-x86_64-release
             artifact-path: bin/libgodot-cpp.windows.release.64.lib
+            cache-name: windows-x86_64-msvc
 
           - name: 🏁 Windows (x86_64, MinGW)
             os: windows-2019
             platform: windows
             artifact-name: godot-cpp-linux-mingw-x86_64-release
             artifact-path: bin/libgodot-cpp.windows.release.64.a
             flags: use_mingw=yes
+            cache-name: windows-x86_64-mingw
 
           - name: 🍎 macOS (universal)
             os: macos-11
@@ -38,26 +49,38 @@ jobs:
             flags: macos_arch=universal
             godot_zip: Godot_v3.5-stable_osx.universal.zip
             executable: Godot.app/Contents/MacOS/Godot
+            cache-name: macos-unversal
 
           - name: 🤖 Android (arm64)
             os: ubuntu-18.04
             platform: android
             artifact-name: godot-cpp-android-arm64-release
             artifact-path: bin/libgodot-cpp.android.release.arm64v8.a
             flags: ANDROID_NDK_ROOT=$ANDROID_NDK_LATEST_HOME android_arch=arm64v8
+            cache-name: android-arm64
 
           - name: 🍏 iOS (arm64)
             os: macos-11
             platform: ios
             artifact-name: godot-cpp-ios-arm64-release
             artifact-path: bin/libgodot-cpp.ios.release.arm64.a
+            cache-name: ios-arm64
+
+    env:
+      SCONS_CACHE: ${{ github.workspace }}/.scons-cache/
 
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           submodules: recursive
 
+      - name: Setup Godot build cache
+        uses: ./.github/actions/godot-cache
+        with:
+          cache-name: ${{ matrix.cache-name }}
+        continue-on-error: true
+
       - name: Set up Python (for SCons)
         uses: actions/setup-python@v4
         with:
@@ -79,7 +102,7 @@ jobs:
 
       - name: Build godot-cpp (debug)
         run: |
-          scons platform=${{ matrix.platform }} target=debug generate_bindings=yes ${{ matrix.flags }}
+          scons platform=${{ matrix.platform }} target=debug ${{ matrix.flags }}
 
       - name: Build test without rebuilding godot-cpp (debug)
         run: |
@@ -111,6 +134,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
 
       - name: Make apt sources.list use the default Ubuntu repositories
         run: |
@@ -127,3 +152,7 @@ jobs:
       - name: Style checks via clang-format
         run: |
           bash ./misc/scripts/clang_format.sh
+
+      - name: Bindings generation checks (ensures get_file_list returns all generated files)
+        run: |
+          python ./misc/scripts/check_get_file_list.py

.gitignore

@@ -3,6 +3,9 @@ gen/*
 logs/*
 *.log
 
+# The default cache directory
+.scons_cache/
+
 # Binaries
 *.o
 *.os

SConstruct

@@ -3,6 +3,7 @@
 import os
 import sys
 import subprocess
+from binding_generator import scons_generate_bindings, scons_emit_files
 
 if sys.version_info < (3,):
 
@@ -133,15 +134,7 @@ opts.Add(
     )
 )
 opts.Add(PathVariable("custom_api_file", "Path to a custom JSON API file", None, PathVariable.PathIsFile))
-opts.Add(
-    EnumVariable(
-        "generate_bindings",
-        "Generate GDNative API bindings",
-        "auto",
-        allowed_values=["yes", "no", "auto", "true"],
-        ignorecase=2,
-    )
-)
+opts.Add(BoolVariable("generate_bindings", "Force GDNative API bindings generation.", False))
 opts.Add(
     EnumVariable(
         "android_arch",
@@ -472,40 +465,37 @@ elif env["platform"] == "javascript":
     elif env["target"] == "release":
         env.Append(CCFLAGS=["-O3"])
 
-env.Append(
-    CPPPATH=[
-        ".",
-        env["headers_dir"],
-        "include",
-        "include/gen",
-        "include/core",
-    ]
-)
+# Cache
+scons_cache_path = os.environ.get("SCONS_CACHE")
+if scons_cache_path is not None:
+    CacheDir(scons_cache_path)
+    Decider("MD5")
 
-# Generate bindings?
+# Generate bindings
+env.Append(BUILDERS={"GenerateBindings": Builder(action=scons_generate_bindings, emitter=scons_emit_files)})
 json_api_file = ""
 
 if "custom_api_file" in env:
     json_api_file = env["custom_api_file"]
 else:
     json_api_file = os.path.join(os.getcwd(), env["headers_dir"], "api.json")
 
-if env["generate_bindings"] == "auto":
-    # Check if generated files exist
-    should_generate_bindings = not os.path.isfile(os.path.join(os.getcwd(), "src", "gen", "Object.cpp"))
-else:
-    should_generate_bindings = env["generate_bindings"] in ["yes", "true"]
+bindings = env.GenerateBindings(
+    env.Dir("."), [json_api_file, "binding_generator.py"]
+)
 
-if should_generate_bindings:
-    # Actually create the bindings here
-    import binding_generator
+# Forces bindings regeneration.
+if env["generate_bindings"]:
+    AlwaysBuild(bindings)
+    NoCache(bindings)
 
-    binding_generator.generate_bindings(json_api_file, env["generate_template_get_node"])
+# Includes
+env.Append(CPPPATH=[[env.Dir(d) for d in [".", env["headers_dir"], "include", "include/gen", "include/core"]]])
 
 # Sources to compile
 sources = []
 add_sources(sources, "src/core", "cpp")
-add_sources(sources, "src/gen", "cpp")
+sources.extend(f for f in bindings if str(f).endswith(".cpp"))
 
 arch_suffix = env["bits"]
 if env["platform"] == "android":
@@ -530,7 +520,6 @@ if env["build_library"]:
     library = env.StaticLibrary(target=env.File("bin/%s" % library_name), source=sources)
     Default(library)
 
-env.Append(CPPPATH=[env.Dir(f) for f in [env["headers_dir"], "include", "include/gen", "include/core"]])
 env.Append(LIBPATH=[env.Dir("bin")])
 env.Append(LIBS=library_name)
 Return("env")

binding_generator.py

@@ -15,9 +15,9 @@ def correct_method_name(method_list):
 classes = []
 
 
-def print_file_list(api_filepath, output_dir, headers=False, sources=False):
+def get_file_list(api_filepath, output_dir, headers=False, sources=False):
     global classes
-    end = ";"
+    files = []
     with open(api_filepath) as api_file:
         classes = json.load(api_file)
     include_gen_folder = Path(output_dir) / "include" / "gen"
@@ -26,17 +26,35 @@ def print_file_list(api_filepath, output_dir, headers=False, sources=False):
         header_filename = include_gen_folder / (strip_name(_class["name"]) + ".hpp")
         source_filename = source_gen_folder / (strip_name(_class["name"]) + ".cpp")
         if headers:
-            print(str(header_filename.as_posix()), end=end)
+            files.append(str(header_filename.as_posix()))
         if sources:
-            print(str(source_filename.as_posix()), end=end)
+            files.append(str(source_filename.as_posix()))
     icall_header_filename = include_gen_folder / "__icalls.hpp"
     register_types_filename = source_gen_folder / "__register_types.cpp"
     init_method_bindings_filename = source_gen_folder / "__init_method_bindings.cpp"
     if headers:
-        print(str(icall_header_filename.as_posix()), end=end)
+        files.append(str(icall_header_filename.as_posix()))
     if sources:
-        print(str(register_types_filename.as_posix()), end=end)
-        print(str(init_method_bindings_filename.as_posix()), end=end)
+        files.append(str(register_types_filename.as_posix()))
+        files.append(str(init_method_bindings_filename.as_posix()))
+    return files
+
+
+def print_file_list(api_filepath, output_dir, headers=False, sources=False):
+    for f in get_file_list(api_filepath, output_dir, headers, sources):
+        print(f, end=";")
+
+
+def scons_emit_files(target, source, env):
+    files = [env.File(f) for f in get_file_list(str(source[0]), target[0].abspath, True, True)]
+    env.Clean(target, files)
+    env["godot_cpp_gen_dir"] = target[0].abspath
+    return files, source
+
+
+def scons_generate_bindings(target, source, env):
+    generate_bindings(str(source[0]), env["generate_template_get_node"], env["godot_cpp_gen_dir"])
+    return None
 
 
 def generate_bindings(api_filepath, use_template_get_node, output_dir="."):
@@ -48,25 +66,12 @@ def generate_bindings(api_filepath, use_template_get_node, output_dir="."):
     include_gen_folder = Path(output_dir) / "include" / "gen"
     source_gen_folder = Path(output_dir) / "src" / "gen"
 
-    try:
-        include_gen_folder.mkdir(parents=True)
-    except os.error as e:
-        if e.errno == errno.EEXIST:
-            print(str(source_gen_folder) + ": " + os.strerror(e.errno))
-        else:
-            exit(1)
-
-    try:
-        source_gen_folder.mkdir(parents=True)
-    except os.error as e:
-        if e.errno == errno.EEXIST:
-            print(str(source_gen_folder) + ": " + os.strerror(e.errno))
-        else:
-            exit(1)
+    include_gen_folder.mkdir(parents=True, exist_ok=True)
+    source_gen_folder.mkdir(parents=True, exist_ok=True)
 
     for c in classes:
         # print(c['name'])
-        used_classes = get_used_classes(c)
+        used_classes = sorted(get_used_classes(c))
         if use_template_get_node and c["name"] == "Node":
             correct_method_name(c["methods"])
 

misc/scripts/check_get_file_list.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+import os, sys
+
+from pathlib import Path
+
+sys.path.insert(1, os.path.join(os.path.dirname(__file__), "..", ".."))
+
+from binding_generator import get_file_list, generate_bindings
+
+api_filepath = "godot-headers/api.json"
+bits = "64"
+double = "float"
+output_dir = "self_test"
+
+generate_bindings(api_filepath, use_template_get_node=False, output_dir=output_dir)
+flist = get_file_list(api_filepath, output_dir, headers=True, sources=True)
+
+p = Path(output_dir)
+allfiles = [str(f.as_posix()) for f in p.glob("**/*.*")]
+missing = list(filter((lambda f: f not in flist), allfiles))
+extras = list(filter((lambda f: f not in allfiles), flist))
+if len(missing) > 0 or len(extras) > 0:
+    print("Error!")
+    for f in missing:
+        print("MISSING: " + str(f))
+    for f in extras:
+        print("EXTRA: " + str(f))
+    sys.exit(1)
+else:
+    print("OK!")