Draft : Data Library Template + Specification Generation tools

.gitignore

@@ -1,3 +1,8 @@
 build
 dist
 .DS_Store
+.cache
+
+# temporary addition while we're iterating on building the specification from source
+documents/SpecificationBuild
+

CMakeLists.txt

@@ -36,7 +36,9 @@ project(MaterialX VERSION ${MATERIALX_LIBRARY_VERSION})
 option(MATERIALX_BUILD_PYTHON "Build the MaterialX Python package from C++ bindings. Requires Python 3.9 or greater." OFF)
 option(MATERIALX_BUILD_VIEWER "Build the MaterialX Viewer." OFF)
 option(MATERIALX_BUILD_GRAPH_EDITOR "Build the MaterialX Graph Editor." OFF)
-option(MATERIALX_BUILD_DOCS "Create HTML documentation using Doxygen. Requires that Doxygen be installed." OFF)
+option(MATERIALX_BUILD_DOCS "(deprecated) use MATERIALX_BUILD_DOXYGEN_DOCS." OFF)
+option(MATERIALX_BUILD_DOXYGEN_DOCS "Create HTML documentation using Doxygen. Requires that Doxygen be installed." OFF)
+option(MATERIALX_BUILD_SPECIFICATION_DOCS "Create Specification documentation, using the data library templates." OFF)
 
 option(MATERIALX_BUILD_GEN_GLSL "Build the GLSL shader generator back-end." ON)
 option(MATERIALX_BUILD_GEN_OSL "Build the OSL shader generator back-end." ON)
@@ -51,6 +53,8 @@ option(MATERIALX_BUILD_BENCHMARK_TESTS "Build benchmark tests." OFF)
 
 option(MATERIALX_BUILD_SHARED_LIBS "Build MaterialX libraries as shared rather than static." OFF)
 option(MATERIALX_BUILD_DATA_LIBRARY "Build generated products from the MaterialX data library." OFF)
+option(MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS "Process the data library files at build time to expand any template elements." ON)
+option(MATERIALX_BUILD_BAKE_NAMED_VALUES "Process the data library files at build time to bake out the named values." ON)
 option(MATERIALX_BUILD_MONOLITHIC "Build a single monolithic MaterialX library." OFF)
 option(MATERIALX_BUILD_USE_CCACHE "Enable the use of ccache to speed up build time, if present." ON)
 option(MATERIALX_PYTHON_LTO "Enable link-time optimizations for MaterialX Python." ON)
@@ -68,6 +72,10 @@ if (MATERIALX_BUILD_IOS)
     set(CMAKE_SYSTEM_NAME iOS)
 endif()
 
+list(APPEND CMAKE_MODULE_PATH
+        ${PROJECT_SOURCE_DIR}/cmake/macros)
+include(Public)
+
 # Apple ecosystem cross-compilation
 # https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html#cross-compiling-for-ios-tvos-visionos-or-watchos
 set(MATERIALX_BUILD_APPLE_EMBEDDED OFF)
@@ -107,6 +115,11 @@ if (MATERIALX_BUILD_JS)
     set(MATERIALX_BUILD_TESTS OFF)
 endif()
 
+# to maintain legacy cmake argument
+if (MATERIALX_BUILD_DOCS)
+    set(MATERIALX_BUILD_DOXYGEN_DOCS ON)
+endif()
+
 # All hardware shading languages currently depend on the GLSL shader generator.
 if(MATERIALX_BUILD_GEN_MSL)
     set(MATERIALX_BUILD_GEN_GLSL ON)
@@ -437,6 +450,16 @@ else()
     set(BUILD_SHARED_LIBS "OFF")
 endif()
 
+# If we're baking the named "Constant:" entries - then we have to also expand any
+# template elements
+if (MATERIALX_BUILD_BAKE_NAMED_VALUES)
+    set(MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS ON)
+endif()
+
+if (MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS OR MATERIALX_BUILD_BAKE_NAMED_VALUES)
+    set(MATERIALX_BUILD_DATA_LIBRARY ON)
+endif()
+
 # Build a monolithic target - needs to be added before the other build targets that may be included.
 if (MATERIALX_BUILD_MONOLITHIC)
     set(MATERIALX_MONOLITHIC_TARGET MaterialX)
@@ -446,6 +469,32 @@ endif()
 # Add core subdirectories
 add_subdirectory(source/MaterialXCore)
 add_subdirectory(source/MaterialXFormat)
+if (MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS OR MATERIALX_BUILD_BAKE_NAMED_VALUES)
+    if (CMAKE_CROSSCOMPILING)
+        set(_MaterialXBuildLibrary "${CMAKE_BINARY_DIR}/BUILD_TOOLS/MaterialXBuildLibrary")
+
+        add_custom_command(
+                OUTPUT ${_MaterialXBuildLibrary}
+                COMMAND ${CMAKE_COMMAND}
+                -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+                -DCMAKE_RUNTIME_OUTPUT_DIRECTORY:PATH=${CMAKE_BINARY_DIR}/BUILD_TOOLS
+                -DMATERIALX_MAJOR_VERSION=${MATERIALX_MAJOR_VERSION}
+                -DMATERIALX_MINOR_VERSION=${MATERIALX_MINOR_VERSION}
+                -DMATERIALX_BUILD_VERSION=${MATERIALX_BUILD_VERSION}
+                -DMATERIALX_NAMESPACE=${MATERIALX_NAMESPACE}
+                -DMATERIALX_BUILD_BAKE_NAMED_VALUES=ON
+                -B"BUILD_TOOLS"
+                -H"${CMAKE_SOURCE_DIR}/source/MaterialXBuildTools"
+                COMMAND ${CMAKE_COMMAND} --build BUILD_TOOLS
+                DEPENDS MaterialXCore MaterialXFormat
+        )
+        add_custom_target(MaterialXBuildLibrary ALL
+        DEPENDS ${_MaterialXBuildLibrary})
+    else()
+        set(_MaterialXBuildLibrary MaterialXBuildLibrary)
+        add_subdirectory(source/MaterialXBuildTools/buildLibrary)
+    endif()
+endif()
 
 # Add shader generation subdirectories
 add_subdirectory(source/MaterialXGenShader)
@@ -467,6 +516,7 @@ if(MATERIALX_BUILD_GEN_GLSL OR MATERIALX_BUILD_GEN_OSL OR MATERIALX_BUILD_GEN_MD
         add_subdirectory(source/MaterialXGenMsl)
     endif()
     add_subdirectory(libraries)
+    add_subdirectory(templates)
 endif()
 
 # Add rendering and viewer subdirectories
@@ -511,8 +561,11 @@ if(MATERIALX_BUILD_PYTHON)
     add_subdirectory(python)
 endif()
 
-if(MATERIALX_BUILD_DOCS)
+if(MATERIALX_BUILD_DOXYGEN_DOCS OR MATERIALX_BUILD_SPECIFICATION_DOCS)
     add_subdirectory(documents)
+    if (MATERIALX_BUILD_SPECIFICATION_DOCS)
+        add_subdirectory(source/MaterialXBuildTools/buildDocs)
+    endif()
 endif()
 
 if(MATERIALX_BUILD_JS)

cmake/macros/Public.cmake

@@ -0,0 +1,127 @@
+# Shared functions need to be extracted to allow them to be shared with the subproject
+# in source/MaterialXBuildTools
+
+function(assign_source_group prefix)
+    foreach(_source IN ITEMS ${ARGN})
+        if(IS_ABSOLUTE "${_source}")
+            file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}")
+        else()
+            set(_source_rel "${_source}")
+        endif()
+        get_filename_component(_source_path "${_source_rel}" PATH)
+        string(REPLACE "/" "\\" _source_path_msvc "${_source_path}")
+        source_group("${prefix}\\${_source_path_msvc}" FILES "${_source}")
+    endforeach()
+endfunction(assign_source_group)
+
+function(mx_add_library MATERIALX_MODULE_NAME)
+    set(options ADD_OBJECTIVE_C_CODE)
+    set(oneValueArgs EXPORT_DEFINE)
+    set(multiValueArgs
+            SOURCE_FILES
+            HEADER_FILES
+            INLINED_FILES
+            MTLX_MODULES)
+    cmake_parse_arguments(args
+            "${options}"
+            "${oneValueArgs}"
+            "${multiValueArgs}"
+            ${ARGN})
+
+    if (APPLE AND args_ADD_OBJECTIVE_C_CODE)
+        file(GLOB_RECURSE materialx_source_oc "${CMAKE_CURRENT_SOURCE_DIR}/*.m*")
+        set_source_files_properties(${materialx_source_oc} PROPERTIES
+                COMPILE_FLAGS "-x objective-c++")
+        set(args_SOURCE_FILES ${args_SOURCE_FILES} ${materialx_source_oc})
+    endif()
+
+    assign_source_group("Source Files" ${args_SOURCE_FILES})
+    assign_source_group("Source Files" ${args_INLINED_FILES})
+    assign_source_group("Header Files" ${args_HEADER_FILES})
+
+    if (NOT MATERIALX_BUILD_MONOLITHIC)
+        set(TARGET_NAME ${MATERIALX_MODULE_NAME})
+        add_library(${TARGET_NAME})
+
+        # Create version resource
+        if(MATERIALX_BUILD_SHARED_LIBS AND MSVC)
+            configure_file(${PROJECT_SOURCE_DIR}/cmake/modules/MaterialXVersion.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc)
+            target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/version.rc)
+        endif()
+
+        target_link_libraries(${TARGET_NAME}
+                PUBLIC
+                ${args_MTLX_MODULES}
+                ${CMAKE_DL_LIBS})
+
+        target_include_directories(${TARGET_NAME}
+                PUBLIC
+                $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../>
+                $<INSTALL_INTERFACE:${MATERIALX_INSTALL_INCLUDE_PATH}>
+                PRIVATE
+                ${EXTERNAL_INCLUDE_DIRS})
+
+        set_target_properties(
+                ${TARGET_NAME} PROPERTIES
+                OUTPUT_NAME ${MATERIALX_MODULE_NAME}${MATERIALX_LIBNAME_SUFFIX}
+                COMPILE_FLAGS "${EXTERNAL_COMPILE_FLAGS}"
+                LINK_FLAGS "${EXTERNAL_LINK_FLAGS}"
+                INSTALL_RPATH "${MATERIALX_SAME_DIR_RPATH}"
+                VERSION "${MATERIALX_LIBRARY_VERSION}"
+                SOVERSION "${MATERIALX_MAJOR_VERSION}")
+    else()
+        set(TARGET_NAME ${MATERIALX_MONOLITHIC_TARGET})
+        add_library(${MATERIALX_MODULE_NAME} ALIAS ${MATERIALX_MONOLITHIC_TARGET})
+
+        # Store the aliased MaterialX modules name to create CMake export aliases later.
+        set_property(GLOBAL APPEND PROPERTY MATERIALX_MODULES ${MATERIALX_MODULE_NAME})
+    endif()
+
+    set_target_properties(${TARGET_NAME} PROPERTIES CXX_VISIBILITY_PRESET hidden)
+    set_target_properties(${TARGET_NAME} PROPERTIES CMAKE_VISIBILITY_INLINES_HIDDEN 1)
+
+    target_sources(${TARGET_NAME}
+            PRIVATE
+            ${args_SOURCE_FILES}
+            PUBLIC
+            FILE_SET
+            mxHeaders
+            TYPE
+            HEADERS
+            BASE_DIRS
+            ${CMAKE_CURRENT_SOURCE_DIR}/..
+            ${CMAKE_CURRENT_BINARY_DIR}/..
+            FILES
+            ${args_HEADER_FILES}
+            ${args_INLINED_FILES})
+
+    target_include_directories(${TARGET_NAME} PUBLIC
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../>)
+
+    target_compile_definitions(${TARGET_NAME} PRIVATE "-D${args_EXPORT_DEFINE}")
+
+    if(NOT SKBUILD)
+        if(NOT MATERIALX_BUILD_MONOLITHIC)
+            install(TARGETS ${MATERIALX_MODULE_NAME}
+                    EXPORT MaterialX
+                    ARCHIVE DESTINATION ${MATERIALX_INSTALL_LIB_PATH}
+                    LIBRARY DESTINATION ${MATERIALX_INSTALL_LIB_PATH}
+                    RUNTIME DESTINATION ${MATERIALX_INSTALL_BIN_PATH}
+                    FILE_SET mxHeaders DESTINATION ${MATERIALX_INSTALL_INCLUDE_PATH})
+        endif()
+
+        if(MSVC)
+            if(MATERIALX_BUILD_SHARED_LIBS)
+                install(FILES $<TARGET_PDB_FILE:${MATERIALX_MODULE_NAME}>
+                        DESTINATION ${MATERIALX_INSTALL_BIN_PATH} OPTIONAL)
+            else()
+                install(FILES "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/$<CONFIG>/$<TARGET_FILE_BASE_NAME:${MATERIALX_MODULE_NAME}>.pdb"
+                        DESTINATION ${MATERIALX_INSTALL_LIB_PATH} OPTIONAL)
+            endif()
+        endif()
+    endif()
+
+    # Pass TARGET_NAME back to call site, so the caller can modify the build target.
+    set(TARGET_NAME ${TARGET_NAME} PARENT_SCOPE)
+endfunction()
+

documents/CMakeLists.txt

@@ -1,45 +1,81 @@
-set(DOXYGEN_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(DOXYGEN_HTML_OUTPUT_DIR ${DOXYGEN_OUTPUT_DIR}/html)
-set(DOXYGEN_INPUT_LIST ${CMAKE_CURRENT_BINARY_DIR}/MainPage.md)
-
-set(MATERIALX_DOXYGEN_SOURCE_FOLDERS
-    ${PROJECT_SOURCE_DIR}/source/MaterialXCore
-    ${PROJECT_SOURCE_DIR}/source/MaterialXFormat
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenShader
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenShader/Nodes
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenGlsl
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenGlsl/Nodes
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenOsl
-    ${PROJECT_SOURCE_DIR}/source/MaterialXGenMdl
-    ${PROJECT_SOURCE_DIR}/source/MaterialXRender
-    ${PROJECT_SOURCE_DIR}/source/MaterialXRenderHw
-    ${PROJECT_SOURCE_DIR}/source/MaterialXRenderGlsl
-    ${PROJECT_SOURCE_DIR}/source/MaterialXRenderOsl)
-
-find_package(Doxygen REQUIRED)
-
-foreach(FOLDER ${MATERIALX_DOXYGEN_SOURCE_FOLDERS})
-    file(GLOB FOLDER_HEADERS ${FOLDER}/*.h)
-    list(APPEND DOXYGEN_INPUT_LIST ${FOLDER_HEADERS})
-endforeach()
-
-string (REPLACE ";" " " DOXYGEN_INPUT_STR "${DOXYGEN_INPUT_LIST}")
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
-
-add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/MainPage.md
-                   COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DeveloperGuide/MainPage.md ${CMAKE_CURRENT_BINARY_DIR}
-                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                   DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/DeveloperGuide/MainPage.md)
-
-add_custom_command(OUTPUT ${DOXYGEN_HTML_OUTPUT_DIR}/index.html
-                   COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenAwesome ${CMAKE_CURRENT_BINARY_DIR}
-                   COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/Images ${CMAKE_CURRENT_BINARY_DIR}
-                   COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
-                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                   DEPENDS ${DOXYGEN_INPUT_LIST} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
-                   COMMENT "Generating HTML documentation: ${DOXYGEN_HTML_OUTPUT_DIR}/index.html")
-add_custom_target(MaterialXDocs ALL DEPENDS ${DOXYGEN_HTML_OUTPUT_DIR}/index.html)
-
-install(DIRECTORY ${DOXYGEN_HTML_OUTPUT_DIR}
-        DESTINATION "documents" MESSAGE_NEVER)
+
+if (MATERIALX_BUILD_DOXYGEN_DOCS)
+
+    set(DOXYGEN_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
+    set(DOXYGEN_HTML_OUTPUT_DIR ${DOXYGEN_OUTPUT_DIR}/html)
+    set(DOXYGEN_INPUT_LIST ${CMAKE_CURRENT_BINARY_DIR}/MainPage.md)
+
+    set(MATERIALX_DOXYGEN_SOURCE_FOLDERS
+        ${PROJECT_SOURCE_DIR}/source/MaterialXCore
+        ${PROJECT_SOURCE_DIR}/source/MaterialXFormat
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenShader
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenShader/Nodes
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenGlsl
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenGlsl/Nodes
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenOsl
+        ${PROJECT_SOURCE_DIR}/source/MaterialXGenMdl
+        ${PROJECT_SOURCE_DIR}/source/MaterialXRender
+        ${PROJECT_SOURCE_DIR}/source/MaterialXRenderHw
+        ${PROJECT_SOURCE_DIR}/source/MaterialXRenderGlsl
+        ${PROJECT_SOURCE_DIR}/source/MaterialXRenderOsl)
+
+    find_package(Doxygen REQUIRED)
+
+    foreach(FOLDER ${MATERIALX_DOXYGEN_SOURCE_FOLDERS})
+        file(GLOB FOLDER_HEADERS ${FOLDER}/*.h)
+        list(APPEND DOXYGEN_INPUT_LIST ${FOLDER_HEADERS})
+    endforeach()
+
+    string (REPLACE ";" " " DOXYGEN_INPUT_STR "${DOXYGEN_INPUT_LIST}")
+
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
+
+    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/MainPage.md
+                       COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DeveloperGuide/MainPage.md ${CMAKE_CURRENT_BINARY_DIR}
+                       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                       DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/DeveloperGuide/MainPage.md)
+
+    add_custom_command(OUTPUT ${DOXYGEN_HTML_OUTPUT_DIR}/index.html
+                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenAwesome ${CMAKE_CURRENT_BINARY_DIR}
+                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/Images ${CMAKE_CURRENT_BINARY_DIR}
+                       COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
+                       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                       DEPENDS ${DOXYGEN_INPUT_LIST} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
+                       COMMENT "Generating HTML documentation: ${DOXYGEN_HTML_OUTPUT_DIR}/index.html")
+    add_custom_target(MaterialXDocs ALL DEPENDS ${DOXYGEN_HTML_OUTPUT_DIR}/index.html)
+
+    install(DIRECTORY ${DOXYGEN_HTML_OUTPUT_DIR}
+            DESTINATION "documents" MESSAGE_NEVER)
+endif()
+
+
+if (MATERIALX_BUILD_SPECIFICATION_DOCS)
+    set(MATERIALX_DATA_LIBRARY_TEMPLATE_DIR ${PROJECT_SOURCE_DIR}/templates)
+    set(MATERIALX_SPECIFICATION_TEMPLATE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/SpecificationTemplates)
+    #Ulitmately we may decide to move this directly to ${CMAKE_CURRENT_SOURCE_DIR}/Specification
+    set(MATERIALX_SPECIFICATION_DEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/SpecificationBuild) 
+
+    set(DOCS_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/nodes)
+    set(NODE_MD_MANIFEST ${DOCS_OUTPUT_DIR}/_manifest.txt)
+
+    file(GLOB_RECURSE MATERIALX_DATA_LIBRARY_MTLX_SOURCE_FILES
+            LIST_DIRECTORIES false
+            ${MATERIALX_DATA_LIBRARY_TEMPLATE_DIR}/*.mtlx)
+
+    file(GLOB_RECURSE MATERIALX_MD_SOURCE_FILES
+            LIST_DIRECTORIES false
+            ${MATERIALX_SPECIFICATION_TEMPLATE_SOURCE_DIR}/*.template.md)
+
+    add_custom_command(
+            OUTPUT ${NODE_MD_MANIFEST}
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${DOCS_OUTPUT_DIR}
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${MATERIALX_SPECIFICATION_DEST_DIR}
+            COMMAND MaterialXBuildDocs --sourceLibraryRoot ${MATERIALX_DATA_LIBRARY_TEMPLATE_DIR} --destDocRoot ${DOCS_OUTPUT_DIR} --sourceMDRoot  ${MATERIALX_SPECIFICATION_TEMPLATE_SOURCE_DIR} --destMDRoot  ${MATERIALX_SPECIFICATION_DEST_DIR} --manifestFile ${NODE_MD_MANIFEST}
+            DEPENDS ${MATERIALX_DATA_LIBRARY_MTLX_SOURCE_FILES} ${MATERIALX_MD_SOURCE_FILES} MaterialXBuildDocs
+    )
+
+    add_custom_target(MaterialXBuildSpecificationDocs ALL
+            DEPENDS ${NODE_MD_MANIFEST})
+
+endif()
+