diff --git a/.github/workflows/external-zstd.yml b/.github/workflows/external-zstd.yml index 6591b1cb..b988a582 100644 --- a/.github/workflows/external-zstd.yml +++ b/.github/workflows/external-zstd.yml @@ -29,7 +29,7 @@ jobs: - name: Build run: | - python -m pip install --config-settings=--global-option=--system-zstd . + python -m pip install --config-settings=--global-option=--system-zstd --config-settings=--global-option=--system-zstd-cffi . macOS: runs-on: 'macos-13' diff --git a/make_cffi.py b/make_cffi.py index 9b5fc0f5..bed91617 100644 --- a/make_cffi.py +++ b/make_cffi.py @@ -16,22 +16,6 @@ import cffi import packaging.tags -HERE = os.path.abspath(os.path.dirname(__file__)) - -SOURCES = [ - "zstd/zstd.c", -] - -# Headers whose preprocessed output will be fed into cdef(). -HEADERS = [ - os.path.join(HERE, "zstd", p) - for p in ("zstd_errors.h", "zstd.h", "zdict.h") -] - -INCLUDE_DIRS = [ - os.path.join(HERE, "zstd"), -] - # cffi can't parse some of the primitives in zstd.h. So we invoke the # preprocessor and feed its output into cffi. compiler = distutils.ccompiler.new_compiler() @@ -168,21 +152,62 @@ def normalize_output(output): return b"\n".join(lines) -# musl 1.1 doesn't define qsort_r. We need to force using the C90 -# variant. -define_macros = [] -for tag in packaging.tags.platform_tags(): - if tag.startswith("musllinux_1_1_"): - define_macros.append(("ZDICT_QSORT", "ZDICT_QSORT_C90")) +def get_ffi(system_zstd = False): + zstd_sources = [] + include_dirs = [] + libraries = [] + + if not system_zstd: + here = os.path.abspath(os.path.dirname(__file__)) + zstd_sources += [ + "zstd/zstd.c", + ] -ffi = cffi.FFI() -# *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning -# when cffi uses the function. Since we statically link against zstd, even -# if we use the deprecated functions it shouldn't be a huge problem. -ffi.set_source( - "zstandard._cffi", - """ + # Headers whose preprocessed output will be fed into cdef(). + headers = [ + os.path.join(here, "zstd", p) + for p in ("zstd_errors.h", "zstd.h", "zdict.h") + ] + + include_dirs += [ + os.path.join(here, "zstd"), + ] + else: + libraries += ["zstd"] + + # Locate headers using the preprocessor. + include_re = re.compile(r'^# \d+ "([^"]+/(?:zstd_errors|zstd|zdict)\.h)"') + with tempfile.TemporaryDirectory() as temp_dir: + with open(os.path.join(temp_dir, "input.h"), "w") as f: + f.write(""" +#include +#include +#include +""") + compiler.preprocess(os.path.join(temp_dir, "input.h"), + os.path.join(temp_dir, "output.h")) + with open(os.path.join(temp_dir, "output.h"), "r") as f: + headers = list({ + m.group(1) for m in map(include_re.match, f) + if m is not None + }) + + # musl 1.1 doesn't define qsort_r. We need to force using the C90 + # variant. + define_macros = [] + for tag in packaging.tags.platform_tags(): + if tag.startswith("musllinux_1_1_"): + define_macros.append(("ZDICT_QSORT", "ZDICT_QSORT_C90")) + + + ffi = cffi.FFI() + # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning + # when cffi uses the function. Since we statically link against zstd, even + # if we use the deprecated functions it shouldn't be a huge problem. + ffi.set_source( + "zstandard._cffi", + """ #define ZSTD_STATIC_LINKING_ONLY #define ZSTD_DISABLE_DEPRECATE_WARNINGS #include @@ -191,48 +216,51 @@ def normalize_output(output): #define ZDICT_DISABLE_DEPRECATE_WARNINGS #include """, - sources=SOURCES, - include_dirs=INCLUDE_DIRS, - define_macros=define_macros, -) - -DEFINE = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)") - -sources = [] - -# Feed normalized preprocessor output for headers into the cdef parser. -for header in HEADERS: - preprocessed = preprocess(header) - sources.append(normalize_output(preprocessed)) - - # #define's are effectively erased as part of going through preprocessor. - # So perform a manual pass to re-add those to the cdef source. - with open(header, "rb") as fh: - for line in fh: - line = line.strip() - m = DEFINE.match(line) - if not m: - continue + sources=zstd_sources, + include_dirs=include_dirs, + define_macros=define_macros, + libraries=libraries, + ) - if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY": - continue + define = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)") - # The parser doesn't like some constants with complex values. - if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"): - continue + sources = [] - # These defines create aliases from old (camelCase) type names - # to the new PascalCase names, which breaks CFFI. - if m.group(1).lower() == m.group(2).lower(): - continue + # Feed normalized preprocessor output for headers into the cdef parser. + for header in headers: + preprocessed = preprocess(header) + sources.append(normalize_output(preprocessed)) + + # #define's are effectively erased as part of going through preprocessor. + # So perform a manual pass to re-add those to the cdef source. + with open(header, "rb") as fh: + for line in fh: + line = line.strip() + m = define.match(line) + if not m: + continue + + if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY": + continue + + # The parser doesn't like some constants with complex values. + if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"): + continue + + # These defines create aliases from old (camelCase) type names + # to the new PascalCase names, which breaks CFFI. + if m.group(1).lower() == m.group(2).lower(): + continue + + # The ... is magic syntax by the cdef parser to resolve the + # value at compile time. + sources.append(b"#define " + m.group(1) + b" ...") - # The ... is magic syntax by the cdef parser to resolve the - # value at compile time. - sources.append(b"#define " + m.group(1) + b" ...") + cdeflines = b"\n".join(sources).splitlines() + cdeflines = [line for line in cdeflines if line.strip()] + ffi.cdef(b"\n".join(cdeflines).decode("latin1")) + return ffi -cdeflines = b"\n".join(sources).splitlines() -cdeflines = [line for line in cdeflines if line.strip()] -ffi.cdef(b"\n".join(cdeflines).decode("latin1")) if __name__ == "__main__": - ffi.compile() + get_ffi().compile() diff --git a/setup.py b/setup.py index 5bc5558f..8df4b686 100755 --- a/setup.py +++ b/setup.py @@ -73,6 +73,7 @@ SUPPORT_LEGACY = False SYSTEM_ZSTD = False +SYSTEM_ZSTD_CFFI = False WARNINGS_AS_ERRORS = False C_BACKEND = True CFFI_BACKEND = True @@ -103,6 +104,10 @@ SYSTEM_ZSTD = True sys.argv.remove("--system-zstd") +if "--system-zstd-cffi" in sys.argv: + SYSTEM_ZSTD_CFFI = True + sys.argv.remove("--system-zstd-cffi") + if "--warnings-as-errors" in sys.argv: WARNINGS_AS_ERRORS = True sys.argv.remove("--warning-as-errors") @@ -138,7 +143,7 @@ if CFFI_BACKEND and cffi: import make_cffi - extensions.append(make_cffi.ffi.distutils_extension()) + extensions.append(make_cffi.get_ffi(system_zstd=SYSTEM_ZSTD_CFFI).distutils_extension()) version = None