diff --git a/docs/type_caster_iwyu.rst b/docs/type_caster_iwyu.rst new file mode 100644 index 00000000..a4716602 --- /dev/null +++ b/docs/type_caster_iwyu.rst @@ -0,0 +1,68 @@ +pybind11 ``type_caster`` design & `clangd Include Cleaner`_ +=========================================================== + +This document is purely to explain — in a nutshell — why pybind11 include +files with ``type_caster`` specializations require this: + +.. code-block:: cpp + + // IWYU pragma: always_keep + +For general technical information about the pybind11 ``type_caster`` mechanism +see the `Custom type casters`_ section. + +The problem +----------- + +The `clangd Include Cleaner`_ cannot possibly have the ability to detect which +``type_caster`` specialization is the correct one to use in a given C++ +translation unit. Without IWYU pragmas, Include Cleaner is likely to suggest +removing ``type_caster`` include files. + +The consequences +---------------- + +1. Incorrect runtime behavior. +2. `ODR violations`_. + +Example for 1. Incorrect runtime behavior +----------------------------------------- + +.. code-block:: cpp + + #include + #include + +If the stl.h include is removed (as suggested by Include Cleaner), conversions +between e.g. ``std::vector`` and Python ``list`` will no longer work. In most +cases this will be very obvious at runtime, but there are less obvious +situations, most notably for ``type_caster`` specializations that inherit from +``type_caster_base`` (e.g. `pybind11_abseil/status_caster.h`_). +Some conversions may still work correctly, while others will have unexpected +behavior. + +Explanation for 2. ODR violations +--------------------------------- + +This problem only arises if two or more C++ translation units are statically +linked (e.g. one ``.so`` Python extension built from multiple .cpp files), or +all visibility-restricting features (e.g. ``-fvisibility=hidden``, +``namespace pybind11`` ``__attribute__((visibility("hidden")))``, +``RTLD_LOCAL``) are disabled. + +Consider the same simple code example as above: if the stl.h include is missing +(as suggested by Include Cleaner) in one translation unit, but not in another +(maybe because Include Cleaner was never applied), the resulting Python +extension will have link incompatibilities, which is often referred to as +ODR violations. The behavior is entirely undefined. For better or worse, the +extension may perform as desired for years, until one day it suddenly does +not, because of some unrelated change in the environment (e.g. new compiler +version, a system library update), resulting in seemingly inexplicable failures. + +See also: `pybind/pybind11#4022`_ (pybind11 smart_holder branch) + +.. _`clangd Include Cleaner`: https://clangd.llvm.org/design/include-cleaner +.. _`Custom type casters`: https://pybind11.readthedocs.io/en/stable/advanced/cast/custom.html +.. _`ODR violations`: https://en.cppreference.com/w/cpp/language/definition +.. _`pybind11_abseil/status_caster.h`: https://github.com/pybind/pybind11_abseil/blob/4b883e48ae749ff984c220484d54fdeb0cb4302c/pybind11_abseil/status_caster.h#L52-L53). +.. _`pybind/pybind11#4022`: https://github.com/pybind/pybind11/pull/4022 diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 6a2d35a8..8309f938 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -10,7 +10,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "cast.h" #include diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index 75aec0ba..a94acda4 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -9,7 +9,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 7d485f63..2565bd1b 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -8,14 +8,18 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/descr.h" #include "detail/native_enum_data.h" #include "detail/type_caster_base.h" #include "detail/type_caster_odr_guard.h" #include "detail/typeid.h" +// IWYU pragma: end_exports #include "pytypes.h" #include diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index 167ea0e3..48a5e10d 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -8,6 +8,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "pybind11.h" diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index 8a831c12..087903c5 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "pybind11.h" diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 26498df6..1fcac992 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 330eaeac..f38478c6 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #define PYBIND11_VERSION_MAJOR 3 diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index a18754b6..d0a0187c 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -10,6 +10,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "common.h" diff --git a/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h b/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h index 7c00fe98..f369a19d 100644 --- a/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h +++ b/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "common.h" diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index be434112..f021b054 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -4,6 +4,8 @@ // For background see the description of PR google/pybind11k#30099. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "../attr.h" diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index bc92e1e1..84f7e841 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "class.h" diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index cbad7db7..0bda7378 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "common.h" diff --git a/include/pybind11/detail/native_enum_data.h b/include/pybind11/detail/native_enum_data.h index 5895dc6c..339eb0ec 100644 --- a/include/pybind11/detail/native_enum_data.h +++ b/include/pybind11/detail/native_enum_data.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #define PYBIND11_HAS_NATIVE_ENUM diff --git a/include/pybind11/detail/struct_smart_holder.h b/include/pybind11/detail/struct_smart_holder.h index b1e24d7b..09d27ab0 100644 --- a/include/pybind11/detail/struct_smart_holder.h +++ b/include/pybind11/detail/struct_smart_holder.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + /* Proof-of-Concept for smart pointer interoperability. High-level aspects: diff --git a/include/pybind11/detail/try_as_void_ptr_capsule_get_pointer.h b/include/pybind11/detail/try_as_void_ptr_capsule_get_pointer.h index e2d3cbf6..e4b06c49 100644 --- a/include/pybind11/detail/try_as_void_ptr_capsule_get_pointer.h +++ b/include/pybind11/detail/try_as_void_ptr_capsule_get_pointer.h @@ -5,6 +5,8 @@ // in the Google-internal environment, but the current implementation is lacking // any safety checks. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "../pytypes.h" diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index db6ad8f6..640cfe24 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include diff --git a/include/pybind11/detail/type_caster_odr_guard.h b/include/pybind11/detail/type_caster_odr_guard.h index 46ea1af8..f118484a 100644 --- a/include/pybind11/detail/type_caster_odr_guard.h +++ b/include/pybind11/detail/type_caster_odr_guard.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "descr.h" diff --git a/include/pybind11/detail/typeid.h b/include/pybind11/detail/typeid.h index a67b5213..cbea6787 100644 --- a/include/pybind11/detail/typeid.h +++ b/include/pybind11/detail/typeid.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include diff --git a/include/pybind11/detail/value_and_holder.h b/include/pybind11/detail/value_and_holder.h index ca37d70a..1aebb8d2 100644 --- a/include/pybind11/detail/value_and_holder.h +++ b/include/pybind11/detail/value_and_holder.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "common.h" diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index 80e72815..3c3b12ff 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -7,11 +7,15 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include +// IWYU pragma: begin_exports #include "common.h" +// IWYU pragma: end_exports /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 79df26e8..ed8be030 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -5,11 +5,15 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include +// IWYU pragma: begin_exports #include "common.h" +// IWYU pragma: end_exports #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 0ac53f8e..6036a2a2 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #define PYBIND11_HAS_TYPE_CASTER_STD_FUNCTION_SPECIALIZATIONS diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index 88881049..e3735b6d 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -9,12 +9,16 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include #if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +// IWYU pragma: begin_exports # include "detail/internals.h" +// IWYU pragma: end_exports #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) diff --git a/include/pybind11/gil_safe_call_once.h b/include/pybind11/gil_safe_call_once.h index 5f9e1b03..915bcc8d 100644 --- a/include/pybind11/gil_safe_call_once.h +++ b/include/pybind11/gil_safe_call_once.h @@ -2,7 +2,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "gil.h" #include diff --git a/include/pybind11/native_enum.h b/include/pybind11/native_enum.h index a4120a2c..b292da57 100644 --- a/include/pybind11/native_enum.h +++ b/include/pybind11/native_enum.h @@ -4,9 +4,11 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/native_enum_data.h" #include "detail/type_caster_base.h" +// IWYU pragma: end_exports #include "cast.h" #include diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 09894cf7..108f88c3 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -7,10 +7,14 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "pybind11.h" +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "complex.h" #include "gil_safe_call_once.h" #include "pytypes.h" diff --git a/include/pybind11/options.h b/include/pybind11/options.h index 1b212252..381d84ef 100644 --- a/include/pybind11/options.h +++ b/include/pybind11/options.h @@ -9,7 +9,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index accba3fe..b8684ee8 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -10,12 +10,14 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/class.h" #include "detail/dynamic_raw_ptr_cast_if_possible.h" #include "detail/function_record_pyobject.h" #include "detail/init.h" #include "detail/native_enum_data.h" #include "detail/using_smart_holder.h" +// IWYU pragma: end_exports #include "attr.h" #include "gil.h" #include "gil_safe_call_once.h" diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 9b641aa0..5ee66f12 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -9,7 +9,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "buffer_info.h" #include diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 05351d20..e9bf89da 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -7,12 +7,16 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "pybind11.h" +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/descr.h" #include "detail/type_caster_base.h" +// IWYU pragma: end_exports #include #include diff --git a/include/pybind11/stl/filesystem.h b/include/pybind11/stl/filesystem.h index c16a9ae5..06957c88 100644 --- a/include/pybind11/stl/filesystem.h +++ b/include/pybind11/stl/filesystem.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 15a9046d..eb2231fc 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -7,10 +7,14 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/type_caster_base.h" +// IWYU pragma: end_exports #include "cast.h" #include "operators.h" diff --git a/include/pybind11/type_caster_pyobject_ptr.h b/include/pybind11/type_caster_pyobject_ptr.h index 7789c040..279a2447 100644 --- a/include/pybind11/type_caster_pyobject_ptr.h +++ b/include/pybind11/type_caster_pyobject_ptr.h @@ -1,9 +1,13 @@ // Copyright (c) 2023 The pybind Community. +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/descr.h" +// IWYU pragma: end_exports #include "cast.h" #include "pytypes.h" diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index b0feb946..deef0e97 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -10,7 +10,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "cast.h" #include "pytypes.h"