build(tablegen): pregen opdef tablegen targets

GitOrigin-RevId: ab7564df14

imperative/CMakeLists.txt

@@ -66,8 +66,8 @@ target_link_libraries(${MODULE_NAME} PRIVATE nlohmann_json::nlohmann_json)
 target_include_directories(
   ${MODULE_NAME}
   PUBLIC src/include
-  PRIVATE ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR} ${MGB_OPDEF_OUT_DIR}
-          ${CPP_REDIS_INCLUDES})
+  PRIVATE ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR} ${CPP_REDIS_INCLUDES})
+target_link_libraries(${MODULE_NAME} PRIVATE mgb_opdef_inc)
 target_compile_definitions(${MODULE_NAME} PRIVATE MODULE_NAME=${MODULE_NAME})
 target_compile_options(${MODULE_NAME} PRIVATE -Wno-unused-parameter)
 if(CXX_SUPPORT_WCLASS_MEMACCESS)

imperative/tablegen/CMakeLists.txt

@@ -1,21 +1,88 @@
-# mgb tablegen executable
-set(TABLE_TARGET mgb-mlir-autogen)
-file(GLOB_RECURSE SRCS ${CMAKE_CURRENT_SOURCE_DIR}/*.h
-     ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
-add_executable(${TABLE_TARGET} ${SRCS})
-target_include_directories(${TABLE_TARGET} PRIVATE ${MLIR_LLVM_INCLUDE_DIR})
-target_link_libraries(${TABLE_TARGET} PRIVATE LLVMTableGen MLIRTableGen LLVMSupport)
-set(MGB_TABLEGEN_EXE ${TABLE_TARGET})
+set(MGB_OPDEF_OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/generated)
+set(MGB_OPDEF_OPS_SRC ${CMAKE_SOURCE_DIR}/src/core/include/megbrain/ir/ops.td)
+set(MGB_OPDEF_PARAMS_SRC ${CMAKE_SOURCE_DIR}/dnn/scripts/opr_param_defs.py)
 
-# generate megbrain opdef c header and python bindings
-set(LLVM_TARGET_DEFINITIONS ${MGE_IR_DIR}/ops.td)
-tablegen(MGB opdef.h.inl ${MGE_IR_INCLUDE_DIRS} "--gen-cpp-header")
-tablegen(MGB opdef.cpp.inl ${MGE_IR_INCLUDE_DIRS} "--gen-cpp-body")
-tablegen(MGB opdef.py.inl ${MGE_IR_INCLUDE_DIRS} "--gen-python-binding")
-tablegen(MGB opdef.cpy.inl ${MGE_IR_INCLUDE_DIRS} "--gen-python-c-extension")
-tablegen(MGB enum_macro.h ${MGE_IR_INCLUDE_DIRS} "--gen-enum-list-macro")
-add_custom_target(mgb_opdef ALL DEPENDS opdef.h.inl opdef.cpp.inl opdef.py.inl
-                                        opdef.cpy.inl enum_macro.h param_defs_tblgen)
-set(MGB_OPDEF_OUT_DIR
-    ${CMAKE_CURRENT_BINARY_DIR}
-    PARENT_SCOPE)
+# we set CMAKE_CONFIGURE_DEPENDS so that when source files or hash.txt was modified,
+# cmake configure would be triggered to update ${MD5_MISMATCH}
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/checkhash.cmake
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  ERROR_QUIET
+  RESULT_VARIABLE MD5_MISMATCH)
+
+if(${MD5_MISMATCH})
+  # mgb tablegen executable
+  set(TABLE_TARGET mgb-mlir-autogen)
+  file(GLOB_RECURSE SRCS ${CMAKE_CURRENT_SOURCE_DIR}/*.h
+       ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
+  add_executable(${TABLE_TARGET} ${SRCS})
+  target_include_directories(${TABLE_TARGET} PRIVATE ${MLIR_LLVM_INCLUDE_DIR})
+  target_link_libraries(${TABLE_TARGET} PRIVATE LLVMTableGen MLIRTableGen LLVMSupport)
+  set(MGB_TABLEGEN_EXE ${TABLE_TARGET})
+
+  # generate megbrain opdef c header and python bindings basically same as
+  # third_party/llvm-project/llvm/cmake/modules/TableGen.cmake but change output folder
+  # and add extra dependency
+  set(LLVM_SOURCE_DIR ${CMAKE_SOURCE_DIR}/third_party/llvm-project/llvm)
+  set(LLVM_BINARY_DIR ${CMAKE_BINARY_DIR}/third_party/llvm-project/llvm)
+  set(MLIR_SOURCE_DIR ${CMAKE_SOURCE_DIR}/third_party/llvm-project/mlir)
+  set(MLIR_BINARY_DIR ${CMAKE_BINARY_DIR}/third_party/llvm-project/mlir)
+  set(MGB_TABLEGEN_INCLUDES
+      -I${LLVM_SOURCE_DIR}/include
+      -I${LLVM_BINARY_DIR}/include
+      -I${MLIR_SOURCE_DIR}/include
+      -I${MLIR_BINARY_DIR}/include
+      -I${CMAKE_SOURCE_DIR}/src/core/include/megbrain/ir
+      -I${CMAKE_BINARY_DIR}/src/core/include/megbrain/ir)
+  set(MGB_TABLEGEN_FLAGS --write-if-changed)
+  set(MGB_TABLEGEN_TARGETS)
+
+  function(tablegen_opdef target output)
+    add_custom_target(
+      mgb_opdef_${target}
+      COMMAND
+        ${MGB_TABLEGEN_EXE} ${MGB_TABLEGEN_INCLUDES} --gen-${target}
+        ${MGB_OPDEF_OPS_SRC} ${MGB_TABLEGEN_FLAGS} -o ${MGB_OPDEF_OUT_DIR}/${output}
+      DEPENDS param_defs_tblgen)
+    set(MGB_TABLEGEN_TARGETS
+        ${MGB_TABLEGEN_TARGETS} mgb_opdef_${target}
+        PARENT_SCOPE)
+  endfunction()
+
+  tablegen_opdef(cpp-header opdef.h.inl)
+  tablegen_opdef(cpp-body opdef.cpp.inl)
+  tablegen_opdef(python-binding opdef.py.inl)
+  tablegen_opdef(python-c-extension opdef.cpy.inl)
+  tablegen_opdef(enum-list-macro enum_macro.h)
+
+  add_custom_target(
+    mgb_opdef_genhash
+    ${CMAKE_COMMAND} -P genhash.cmake
+    DEPENDS ${MGB_TABLEGEN_TARGETS}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  add_custom_target(mgb_opdef DEPENDS ${MGB_TABLEGEN_TARGETS} mgb_opdef_genhash)
+else()
+  # add extra dependencies for auto reconfiguration
+  set_property(
+    DIRECTORY
+    APPEND
+    PROPERTY CMAKE_CONFIGURE_DEPENDS
+             ${MGB_OPDEF_OPS_SRC}
+             ${MGB_OPDEF_PARAMS_SRC}
+             generated/opdef.h.inl
+             generated/opdef.cpp.inl
+             generated/opdef.py.inl
+             generated/opdef.cpy.inl
+             generated/enum_macro
+             generated/hash.txt)
+  # additional check for safety
+  add_custom_target(
+    mgb_opdef_checkhash
+    ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/checkhash.cmake
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  add_custom_target(mgb_opdef DEPENDS mgb_opdef_checkhash)
+endif()
+
+add_library(mgb_opdef_inc INTERFACE)
+target_include_directories(mgb_opdef_inc INTERFACE ${MGB_OPDEF_OUT_DIR})
+add_dependencies(mgb_opdef_inc mgb_opdef)

imperative/tablegen/autogen.cpp

@@ -3,8 +3,9 @@
 #include "./targets/pybind11.h"
 #include "./targets/python_c_extension.h"
 
-using llvm::raw_ostream;
-using llvm::RecordKeeper;
+namespace {
+
+using namespace mlir::tblgen;
 
 enum ActionType { None, CppHeader, CppBody, Pybind, CPython, EnumListMacro };
 
@@ -24,25 +25,35 @@ llvm::cl::opt<ActionType> action(
                         EnumListMacro, "gen-enum-list-macro",
                         "Generate enum param list macro")));
 
-using namespace mlir::tblgen;
+template <llvm::TableGenMainFn* MainFn>
+llvm::TableGenMainFn* WrapMain() {
+    return [](llvm::raw_ostream& os, llvm::RecordKeeper& records) -> bool {
+        os << "// clang-format off\n";
+        auto ret = MainFn(os, records);
+        os << "// clang-format on\n";
+        return ret;
+    };
+}
+
+}  // namespace
 
 int main(int argc, char** argv) {
     llvm::InitLLVM y(argc, argv);
     llvm::cl::ParseCommandLineOptions(argc, argv);
     if (action == ActionType::CppHeader) {
-        return TableGenMain(argv[0], &gen_op_def_c_header);
+        return TableGenMain(argv[0], WrapMain<&gen_op_def_c_header>());
     }
     if (action == ActionType::CppBody) {
-        return TableGenMain(argv[0], &gen_op_def_c_body);
+        return TableGenMain(argv[0], WrapMain<&gen_op_def_c_body>());
     }
     if (action == ActionType::Pybind) {
-        return TableGenMain(argv[0], &gen_op_def_pybind11);
+        return TableGenMain(argv[0], WrapMain<&gen_op_def_pybind11>());
     }
     if (action == ActionType::CPython) {
-        return TableGenMain(argv[0], &gen_op_def_python_c_extension);
+        return TableGenMain(argv[0], WrapMain<&gen_op_def_python_c_extension>());
     }
     if (action == ActionType::EnumListMacro) {
-        return TableGenMain(argv[0], &gen_enum_param_list_macro);
+        return TableGenMain(argv[0], WrapMain<&gen_enum_param_list_macro>());
     }
     return -1;
 }

imperative/tablegen/checkhash.cmake

@@ -0,0 +1,16 @@
+set(SOURCES
+    ../../dnn/scripts/opr_param_defs.py
+    ../../src/core/include/megbrain/ir/ops.td
+    generated/opdef.h.inl
+    generated/opdef.cpp.inl
+    generated/opdef.py.inl
+    generated/opdef.cpy.inl
+    generated/enum_macro.h)
+execute_process(COMMAND ${CMAKE_COMMAND} -E md5sum ${SOURCES}
+                OUTPUT_VARIABLE GENERATED_HASH_CONTENT)
+
+file(READ generated/hash.txt HASH_CONTENT)
+
+if(NOT "${GENERATED_HASH_CONTENT}" STREQUAL "${HASH_CONTENT}")
+  message(FATAL_ERROR "File ops.td was changed, please rerun cmake configure")
+endif()

imperative/tablegen/generated/enum_macro.h

@@ -0,0 +1,34 @@
+// clang-format off
+#define FOR_EACH_ENUM_PARAM(cb) \
+    cb(::megdnn::param::PoolingV0::Mode); \
+    cb(::megdnn::param::Convolution::Format); \
+    cb(::megdnn::param::Argsort::Order); \
+    cb(::megdnn::param::ConvBiasV0::NonlineMode); \
+    cb(::megdnn::param::ConvolutionV0::Mode); \
+    cb(::megdnn::param::ConvolutionV0::Sparse); \
+    cb(::megdnn::param::ConvolutionV1::ComputeMode); \
+    cb(::megdnn::param::BN::ParamDim); \
+    cb(::megdnn::param::BN::FwdMode); \
+    cb(::megdnn::param::MatrixMulV1::ComputeMode); \
+    cb(::megdnn::param::MatrixMul::Format); \
+    cb(::megdnn::param::CollectiveComm::Mode); \
+    cb(::megdnn::param::Convolution3D::Mode); \
+    cb(::megdnn::param::Convolution3D::Sparse); \
+    cb(::megdnn::param::Convolution3D::DataType); \
+    cb(::megdnn::param::Convolution3D::Format); \
+    cb(::megdnn::param::ConvolutionV0::Format); \
+    cb(::megdnn::param::CvtColor::Mode); \
+    cb(::megdnn::param::Elemwise::Mode); \
+    cb(::megdnn::param::ElemwiseMultiType::Mode); \
+    cb(::megdnn::param::Padding::PaddingMode); \
+    cb(::megdnn::param::RNNCell::NonlineMode); \
+    cb(::megdnn::param::ROIAlignV0::Mode); \
+    cb(::megdnn::param::ROIPooling::Mode); \
+    cb(::megdnn::param::Reduce::Mode); \
+    cb(::megdnn::param::Reduce::DataType); \
+    cb(::megdnn::param::WarpPerspectiveV1::InterpolationMode); \
+    cb(::megdnn::param::WarpPerspectiveV1::BorderMode); \
+    cb(::megdnn::param::TopK::Mode);
+#define FOR_EACH_BIT_COMBINED_ENUM_PARAM(cb) \
+    cb(::megdnn::param::ExecutionPolicy::Strategy);
+// clang-format on

imperative/tablegen/generated/hash.txt

@@ -0,0 +1,7 @@
+0df57b38e71a4d1882ed6c24f3a26b57  ../../dnn/scripts/opr_param_defs.py
+759bfbf27fd3f0dd6b6edf06377e1d6b  ../../src/core/include/megbrain/ir/ops.td
+c613316001b5f0294ede198f5563f041  generated/opdef.h.inl
+a1f7f13c909f9d4c173277f4ed28fb61  generated/opdef.cpp.inl
+cf48f9ca352fabaeb6c846c11c6b1662  generated/opdef.py.inl
+12365b938f564e5b3639d309f7c83414  generated/opdef.cpy.inl
+71e1462bf4d882e2615c3c632cb671cc  generated/enum_macro.h

imperative/tablegen/genhash.cmake

@@ -0,0 +1,12 @@
+set(SOURCES
+    ../../dnn/scripts/opr_param_defs.py
+    ../../src/core/include/megbrain/ir/ops.td
+    generated/opdef.h.inl
+    generated/opdef.cpp.inl
+    generated/opdef.py.inl
+    generated/opdef.cpy.inl
+    generated/enum_macro.h)
+execute_process(COMMAND ${CMAKE_COMMAND} -E md5sum ${SOURCES}
+                OUTPUT_VARIABLE HASH_CONTENT)
+message(STATUS "Generating hash.txt for opdefs")
+file(WRITE generated/hash.txt "${HASH_CONTENT}")

imperative/test/CMakeLists.txt

@@ -12,9 +12,9 @@ endif()
 # TODO: turn python binding into a static/object library
 add_executable(imperative_test ${SOURCES} ${SRCS})
 add_dependencies(imperative_test mgb_opdef)
-target_include_directories(
-  imperative_test PRIVATE ${MGB_TEST_DIR}/include ../src/include ${MGB_OPDEF_OUT_DIR}
-                          ${CPP_REDIS_INCLUDES})
+target_include_directories(imperative_test PRIVATE ${MGB_TEST_DIR}/include
+                                                   ../src/include ${CPP_REDIS_INCLUDES})
+target_link_libraries(imperative_test mgb_opdef_inc)
 
 # Python binding
 target_include_directories(