[flang] Added lowering and runtime for COMPLEX(16) intrinsics. (#83874)

For `LDBL_MANT_DIG == 113` targets the FortranFloat128Math library
is just an interface library that provides sources and compilation
options to be used for building FortranRuntime - there are not extra
dependencies on other libraries, so it can be a part of FortranRuntime,
which helps to avoid extra linking steps in the compiler driver.
Targets with __float128 support in libc will also use this path.
Other targets, where the math support comes from
FLANG_RUNTIME_F128_MATH_LIB,
FortranFloat128Math is built as a standalone static library,
and the compiler driver needs to conduct the linking.

Flang APIs for COMPLEX(16) are just thin C wrappers around
the C math functions. Flang uses C _Complex ABI for passing/returning
COMPLEX values, so the runtime is aligned to this.

Author: vzakhari

flang/lib/Optimizer/Builder/IntrinsicCall.cpp

@@ -689,22 +689,22 @@ prettyPrintIntrinsicName(fir::FirOpBuilder &builder, mlir::Location loc,
 }
 
 // Generate a call to the Fortran runtime library providing
-// support for 128-bit float math via a third-party library.
-// If the compiler is built without FLANG_RUNTIME_F128_MATH_LIB,
-// this function will report an error.
+// support for 128-bit float math.
+// On 'LDBL_MANT_DIG == 113' targets the implementation
+// is provided by FortranRuntime, otherwise, it is done via
+// FortranFloat128Math library. In the latter case the compiler
+// has to be built with FLANG_RUNTIME_F128_MATH_LIB to guarantee
+// proper linking actions in the driver.
 static mlir::Value genLibF128Call(fir::FirOpBuilder &builder,
                                   mlir::Location loc,
                                   const MathOperation &mathOp,
                                   mlir::FunctionType libFuncType,
                                   llvm::ArrayRef<mlir::Value> args) {
-#ifndef FLANG_RUNTIME_F128_MATH_LIB
-  std::string message = prettyPrintIntrinsicName(
-      builder, loc, "compiler is built without support for '", mathOp.key, "'",
-      libFuncType);
-  fir::emitFatalError(loc, message, /*genCrashDiag=*/false);
-#else  // FLANG_RUNTIME_F128_MATH_LIB
+  // TODO: if we knew that the C 'long double' does not have 113-bit mantissa
+  // on the target, we could have asserted that FLANG_RUNTIME_F128_MATH_LIB
+  // must be specified. For now just always generate the call even
+  // if it will be unresolved.
   return genLibCall(builder, loc, mathOp, libFuncType, args);
-#endif // FLANG_RUNTIME_F128_MATH_LIB
 }
 
 mlir::Value genLibCall(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -926,6 +926,10 @@ constexpr auto FuncTypeInteger8Real16 =
     genFuncType<Ty::Integer<8>, Ty::Real<16>>;
 constexpr auto FuncTypeReal16Complex16 =
     genFuncType<Ty::Real<16>, Ty::Complex<16>>;
+constexpr auto FuncTypeComplex16Complex16 =
+    genFuncType<Ty::Complex<16>, Ty::Complex<16>>;
+constexpr auto FuncTypeComplex16Complex16Complex16 =
+    genFuncType<Ty::Complex<16>, Ty::Complex<16>, Ty::Complex<16>>;
 
 static constexpr MathOperation mathOperations[] = {
     {"abs", "fabsf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
@@ -944,13 +948,17 @@ static constexpr MathOperation mathOperations[] = {
     {"acos", RTNAME_STRING(AcosF128), FuncTypeReal16Real16, genLibF128Call},
     {"acos", "cacosf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"acos", "cacos", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"acos", RTNAME_STRING(CAcosF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"acosh", "acoshf", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"acosh", "acosh", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"acosh", RTNAME_STRING(AcoshF128), FuncTypeReal16Real16, genLibF128Call},
     {"acosh", "cacoshf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>,
      genLibCall},
     {"acosh", "cacosh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genLibCall},
+    {"acosh", RTNAME_STRING(CAcoshF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     // llvm.trunc behaves the same way as libm's trunc.
     {"aint", "llvm.trunc.f32", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genLibCall},
@@ -972,20 +980,26 @@ static constexpr MathOperation mathOperations[] = {
     {"asin", RTNAME_STRING(AsinF128), FuncTypeReal16Real16, genLibF128Call},
     {"asin", "casinf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"asin", "casin", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"asin", RTNAME_STRING(CAsinF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"asinh", "asinhf", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"asinh", "asinh", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"asinh", RTNAME_STRING(AsinhF128), FuncTypeReal16Real16, genLibF128Call},
     {"asinh", "casinhf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>,
      genLibCall},
     {"asinh", "casinh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genLibCall},
+    {"asinh", RTNAME_STRING(CAsinhF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"atan", "atanf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::AtanOp>},
     {"atan", "atan", genFuncType<Ty::Real<8>, Ty::Real<8>>,
      genMathOp<mlir::math::AtanOp>},
     {"atan", RTNAME_STRING(AtanF128), FuncTypeReal16Real16, genLibF128Call},
     {"atan", "catanf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"atan", "catan", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"atan", RTNAME_STRING(CAtanF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"atan2", "atan2f", genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::Atan2Op>},
     {"atan2", "atan2", genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,
@@ -999,6 +1013,8 @@ static constexpr MathOperation mathOperations[] = {
      genLibCall},
     {"atanh", "catanh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genLibCall},
+    {"atanh", RTNAME_STRING(CAtanhF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"bessel_j0", "j0f", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"bessel_j0", "j0", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"bessel_j0", RTNAME_STRING(J0F128), FuncTypeReal16Real16, genLibF128Call},
@@ -1038,11 +1054,15 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::CosOp>},
     {"cos", "ccos", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::CosOp>},
+    {"cos", RTNAME_STRING(CCosF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"cosh", "coshf", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"cosh", "cosh", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"cosh", RTNAME_STRING(CoshF128), FuncTypeReal16Real16, genLibF128Call},
     {"cosh", "ccoshf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"cosh", "ccosh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"cosh", RTNAME_STRING(CCoshF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"divc",
      {},
      genFuncType<Ty::Complex<2>, Ty::Complex<2>, Ty::Complex<2>>,
@@ -1080,6 +1100,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::ExpOp>},
     {"exp", "cexp", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::ExpOp>},
+    {"exp", RTNAME_STRING(CExpF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"feclearexcept", "feclearexcept",
      genFuncType<Ty::Integer<4>, Ty::Integer<4>>, genLibCall},
     {"fedisableexcept", "fedisableexcept",
@@ -1131,6 +1153,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::LogOp>},
     {"log", "clog", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::LogOp>},
+    {"log", RTNAME_STRING(CLogF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"log10", "log10f", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::Log10Op>},
     {"log10", "log10", genFuncType<Ty::Real<8>, Ty::Real<8>>,
@@ -1178,6 +1202,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::PowOp>},
     {"pow", "cpow", genFuncType<Ty::Complex<8>, Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::PowOp>},
+    {"pow", RTNAME_STRING(CPowF128), FuncTypeComplex16Complex16Complex16,
+     genLibF128Call},
     {"pow", RTNAME_STRING(FPow4i),
      genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Integer<4>>,
      genMathOp<mlir::math::FPowIOp>},
@@ -1222,11 +1248,15 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::SinOp>},
     {"sin", "csin", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::SinOp>},
+    {"sin", RTNAME_STRING(CSinF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"sinh", "sinhf", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"sinh", "sinh", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"sinh", RTNAME_STRING(SinhF128), FuncTypeReal16Real16, genLibF128Call},
     {"sinh", "csinhf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"sinh", "csinh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"sinh", RTNAME_STRING(CSinhF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"sqrt", "sqrtf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::SqrtOp>},
     {"sqrt", "sqrt", genFuncType<Ty::Real<8>, Ty::Real<8>>,
@@ -1236,6 +1266,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::SqrtOp>},
     {"sqrt", "csqrt", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::SqrtOp>},
+    {"sqrt", RTNAME_STRING(CSqrtF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"tan", "tanf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::TanOp>},
     {"tan", "tan", genFuncType<Ty::Real<8>, Ty::Real<8>>,
@@ -1245,6 +1277,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::TanOp>},
     {"tan", "ctan", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::TanOp>},
+    {"tan", RTNAME_STRING(CTanF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"tanh", "tanhf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::TanhOp>},
     {"tanh", "tanh", genFuncType<Ty::Real<8>, Ty::Real<8>>,
@@ -1254,6 +1288,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::TanhOp>},
     {"tanh", "ctanh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::TanhOp>},
+    {"tanh", RTNAME_STRING(CTanhF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
 };
 
 // This helper class computes a "distance" between two function types.

flang/runtime/CMakeLists.txt

@@ -57,12 +57,6 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     REAL(16) is mapped to __float128, or libm for targets where REAL(16) \
     is mapped to long double, etc."
     )
-
-  if (NOT FLANG_RUNTIME_F128_MATH_LIB STREQUAL "")
-    add_compile_definitions(
-      -DFLANG_RUNTIME_F128_MATH_LIB="${FLANG_RUNTIME_F128_MATH_LIB}"
-      )
-  endif()
 endif()
 
 include(CheckCXXSymbolExists)
@@ -78,6 +72,16 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+# Check if 128-bit float computations can be done via long double.
+check_cxx_source_compiles(
+  "#include <cfloat>
+   #if LDBL_MANT_DIG != 113
+   #error LDBL_MANT_DIG != 113
+   #endif
+   int main() { return 0; }
+  "
+  HAVE_LDBL_MANT_DIG_113)
+
 check_cxx_compiler_flag(-fno-lto FLANG_RUNTIME_HAS_FNO_LTO_FLAG)
 if (FLANG_RUNTIME_HAS_FNO_LTO_FLAG)
   set(NO_LTO_FLAGS "-fno-lto")
@@ -100,9 +104,7 @@ add_definitions(-U_GLIBCXX_ASSERTIONS)
 add_definitions(-U_LIBCPP_ENABLE_ASSERTIONS)
 
 add_subdirectory(FortranMain)
-if (NOT ${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "")
-  add_subdirectory(Float128Math)
-endif()
+add_subdirectory(Float128Math)
 
 set(sources
   ISO_Fortran_binding.cpp
@@ -319,6 +321,29 @@ if (NOT FLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD STREQUAL "off")
   endif()
 endif()
 
+if (NOT TARGET FortranFloat128Math)
+  # If FortranFloat128Math is not defined, then we are not building
+  # standalone FortranFloat128Math library. Instead, include
+  # the relevant sources into FortranRuntime itself.
+  # The information is provided via FortranFloat128MathILib
+  # interface library.
+  get_target_property(f128_sources
+    FortranFloat128MathILib INTERFACE_SOURCES
+    )
+  if (f128_sources)
+    # The interface may define special macros for Float128Math files,
+    # so we need to propagate them.
+    get_target_property(f128_defs
+      FortranFloat128MathILib INTERFACE_COMPILE_DEFINITIONS
+      )
+    set_property(SOURCE ${f128_sources}
+      APPEND PROPERTY COMPILE_DEFINITIONS
+      ${f128_defs}
+      )
+    list(APPEND sources ${f128_sources})
+  endif()
+endif()
+
 if (NOT DEFINED MSVC)
   add_flang_library(FortranRuntime
     ${sources}

flang/runtime/Float128Math/CMakeLists.txt

@@ -16,34 +16,6 @@
 
 include(CheckLibraryExists)
 
-if (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libquadmath")
-  check_include_file(quadmath.h FOUND_QUADMATH_HEADER)
-  if(FOUND_QUADMATH_HEADER)
-    add_compile_definitions(HAS_QUADMATHLIB)
-  else()
-    message(FATAL_ERROR
-      "FLANG_RUNTIME_F128_MATH_LIB setting requires quadmath.h "
-      "to be available: ${FLANG_RUNTIME_F128_MATH_LIB}"
-      )
-  endif()
-elseif (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libm")
-  check_library_exists(m sinl "" FOUND_LIBM)
-  check_library_exists(m sinf128 "" FOUND_LIBMF128)
-  if (FOUND_LIBM)
-    add_compile_definitions(HAS_LIBM)
-  endif()
-  if (FOUND_LIBMF128)
-    add_compile_definitions(HAS_LIBMF128)
-  endif()
-endif()
-
-if (NOT FOUND_QUADMATH_HEADER AND NOT FOUND_LIBM)
-  message(FATAL_ERROR
-    "Unsupported third-party library for Fortran F128 math runtime: "
-    "${FLANG_RUNTIME_F128_MATH_LIB}"
-    )
-endif()
-
 set(sources
   acos.cpp
   acosh.cpp
@@ -52,8 +24,8 @@ set(sources
   atan.cpp
   atan2.cpp
   atanh.cpp
-  cabs.cpp
   ceil.cpp
+  complex-math.c
   cos.cpp
   cosh.cpp
   erf.cpp
@@ -94,18 +66,61 @@ set(sources
   )
 
 include_directories(AFTER "${CMAKE_CURRENT_SOURCE_DIR}/..")
-add_flang_library(FortranFloat128Math STATIC INSTALL_WITH_TOOLCHAIN ${sources})
+add_library(FortranFloat128MathILib INTERFACE)
 
-if (DEFINED MSVC)
-  set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded)
-  add_flang_library(FortranFloat128Math.static STATIC INSTALL_WITH_TOOLCHAIN
-    ${sources}
-    )
-  set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebug)
-  add_flang_library(FortranFloat128Math.static_dbg STATIC INSTALL_WITH_TOOLCHAIN
-    ${sources}
-    )
-  add_dependencies(FortranFloat128Math FortranFloat128Math.static
-    FortranFloat128Math.static_dbg
-    )
+if (FLANG_RUNTIME_F128_MATH_LIB)
+  if (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libquadmath")
+    check_include_file(quadmath.h FOUND_QUADMATH_HEADER)
+    if(FOUND_QUADMATH_HEADER)
+      add_compile_definitions(HAS_QUADMATHLIB)
+    else()
+      message(FATAL_ERROR
+        "FLANG_RUNTIME_F128_MATH_LIB setting requires quadmath.h "
+        "to be available: ${FLANG_RUNTIME_F128_MATH_LIB}"
+        )
+    endif()
+  else()
+    message(FATAL_ERROR
+      "Unsupported third-party library for Fortran F128 math runtime: "
+      "${FLANG_RUNTIME_F128_MATH_LIB}"
+      )
+  endif()
+
+  add_flang_library(FortranFloat128Math STATIC INSTALL_WITH_TOOLCHAIN
+    ${sources})
+
+  if (DEFINED MSVC)
+    set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded)
+    add_flang_library(FortranFloat128Math.static STATIC INSTALL_WITH_TOOLCHAIN
+      ${sources}
+      )
+    set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebug)
+    add_flang_library(FortranFloat128Math.static_dbg STATIC INSTALL_WITH_TOOLCHAIN
+      ${sources}
+      )
+    add_dependencies(FortranFloat128Math FortranFloat128Math.static
+      FortranFloat128Math.static_dbg
+      )
+  endif()
+elseif (HAVE_LDBL_MANT_DIG_113)
+  # We can use 'long double' versions from libc.
+  check_library_exists(m sinl "" FOUND_LIBM)
+  if (FOUND_LIBM)
+    target_compile_definitions(FortranFloat128MathILib INTERFACE
+      HAS_LIBM
+      )
+    target_sources(FortranFloat128MathILib INTERFACE ${sources})
+  else()
+    message(FATAL_ERROR "FortranRuntime cannot build without libm")
+  endif()
+else()
+  # We can use '__float128' version from libc, if it has them.
+  check_library_exists(m sinf128 "" FOUND_LIBMF128)
+  if (FOUND_LIBMF128)
+    target_compile_definitions(FortranFloat128MathILib INTERFACE
+      HAS_LIBMF128
+      )
+    # Enable this, when math-entries.h and complex-math.h is ready.
+    # target_sources(FortranFloat128MathILib INTERFACE ${sources})
+  endif()
 endif()