diff --git a/CMakeLists.txt b/CMakeLists.txt index 65e061ce7917..4ace656704d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1088,6 +1088,23 @@ cmake_dependent_option( Will be disabled if not supported by the platform" ON "USE_CUDA OR USE_ROCM" OFF) +cmake_dependent_option( + USE_CMAKE_INSTRUMENTATION + "Use cmake-instrumentation to collect build metrics" + OFF # disabled by default because it causes CI timeouts + "CMAKE_VERSION VERSION_GREATER_EQUAL 4.3" + OFF +) +if(USE_CMAKE_INSTRUMENTATION) + cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1.0 + HOOKS postCMakeBuild + OPTIONS trace + CALLBACK "${Python_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tools/build_trace_summary.py" --root "${CMAKE_CURRENT_SOURCE_DIR}" --index + ) +endif() + # # Cannot be put into Dependencies.cmake due circular dependency: # USE_FLASH_ATTENTION -> USE_ROCM -> Dependencies.cmake -> aotriton.cmake diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e99eea57a171..d95583047c98 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,6 +40,7 @@ aspects of contributing to PyTorch. - [Managing multiple build trees](#managing-multiple-build-trees) - [C++ development tips](#c-development-tips) - [Build only what you need](#build-only-what-you-need) + - [Profile build time](#profile-build-time) - [Code completion and IDE support](#code-completion-and-ide-support) - [Make no-op build fast](#make-no-op-build-fast) - [Use Ninja](#use-ninja) @@ -825,6 +826,18 @@ options passed for the first time will persist; please run `ccmake build/`, run `cmake-gui build/`, or directly edit `build/CMakeCache.txt` to adapt build options. +### Profile build time + +With CMake 4.3 or newer, you can collect timing information for compile, link, +and custom build commands by enabling CMake instrumentation: + +```bash +USE_CMAKE_INSTRUMENTATION=1 pip install --no-build-isolation -v -e . +``` + +The build prints a timing summary and the path to a trace that can be loaded in +[Perfetto](https://ui.perfetto.dev/) for further analysis. + ### Code completion and IDE support When using `python -m pip install -e . -v --no-build-isolation`, PyTorch will generate diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt index 02c369e25c29..4bad8c4a9046 100644 --- a/caffe2/CMakeLists.txt +++ b/caffe2/CMakeLists.txt @@ -1,3 +1,12 @@ +# CMake build instrumentation wraps all custom commands with ctest +# --instrument. ATen codegen custom_command in this directory generates +# a long command that causes ctest to fail with "Argument list too long". +# Setting RULE_LAUNCH_CUSTOM to empty prevents custom commands in this +# directory from being wrapped. +if(USE_CMAKE_INSTRUMENTATION) + set_property(DIRECTORY PROPERTY RULE_LAUNCH_CUSTOM "") +endif() + # ---[ Generate and install header and cpp files include(../cmake/Codegen.cmake) diff --git a/cmake/External/nnpack.cmake b/cmake/External/nnpack.cmake index 8a4a310d6292..5075a3215dc2 100644 --- a/cmake/External/nnpack.cmake +++ b/cmake/External/nnpack.cmake @@ -72,6 +72,12 @@ if(ANDROID OR IOS OR ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" OR ${CMAKE_SYSTEM_NAM if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") unset(CMAKE_POLICY_VERSION_MINIMUM) endif() + # NNPACK custom commands use PYTHONPATH=... shell env var assignment + # syntax, which fails when ctest --instrument wraps them. This can be + # removed if https://github.com/Maratyszcza/NNPACK/pull/224 is merged + if(USE_CMAKE_INSTRUMENTATION) + set_property(DIRECTORY "${NNPACK_SOURCE_DIR}" PROPERTY RULE_LAUNCH_CUSTOM "") + endif() # We build static versions of nnpack and pthreadpool but link # them into a shared library for Caffe2, so they need PIC. set_property(TARGET nnpack PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index 4ed42c0caf6b..c72caf8ecc98 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -23,6 +23,7 @@ function(caffe2_print_configuration_summary) message(STATUS " CMAKE_PREFIX_PATH : ${CMAKE_PREFIX_PATH}") message(STATUS " CMAKE_INSTALL_PREFIX : ${CMAKE_INSTALL_PREFIX}") message(STATUS " USE_GOLD_LINKER : ${USE_GOLD_LINKER}") + message(STATUS " USE_CMAKE_INSTRUMENTATION: ${USE_CMAKE_INSTRUMENTATION}") message(STATUS "") message(STATUS " TORCH_VERSION : ${TORCH_VERSION}") diff --git a/tools/build_trace_summary.py b/tools/build_trace_summary.py new file mode 100644 index 000000000000..4098c012e485 --- /dev/null +++ b/tools/build_trace_summary.py @@ -0,0 +1,83 @@ +import argparse +import enum +import json +import os +from collections import defaultdict + + +class Role(enum.Enum): + CONFIGURE = 0 + GENERATE = 1 + CMAKEBUILD = 2 + CMAKEINSTALL = 3 + CUSTOM = 4 + COMPILE = 5 + LINK = 6 + INSTALL = 7 + + +def format_millis(millis): + seconds, millis = divmod(millis, 1000) + minutes, seconds = divmod(seconds, 60) + return f"{minutes:>5}m {seconds:02}.{millis:03}s" + + +# print the top 10 items from a list of (name, duration) pairs +def log_top10(items, title): + items.sort(key=lambda x: x[1], reverse=True) + print(title) + for (name, millis), _ in zip(items, range(10)): + print(format_millis(millis), "|", name) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--index", help="/.cmake/instrumentation/v1/data/index/index.json" + ) + parser.add_argument("--root", help="Root of the repository") + cmdargs = parser.parse_args() + + with open(cmdargs.index) as index_file: + index = json.load(index_file) + trace_path = os.path.join(index["dataDir"], index["trace"]) + with open(trace_path) as trace_file: + trace = json.load(trace_file) + + custom_command_times = [] + source_compile_times = [] + target_compile_times = defaultdict(lambda: 0) + target_link_times = defaultdict(lambda: 0) + stage_times = defaultdict(lambda: 0) + + print("*** Build Metrics Summary ***") + for item in trace: + args = item["args"] + role = Role[args["role"].upper()] + duration_ms = args["duration"] + match role: + case Role.CUSTOM: + custom_command_times.append((args["command"], duration_ms)) + case Role.LINK: + target_link_times[args["target"]] += duration_ms + case Role.COMPILE: + target_compile_times[args["target"]] += duration_ms + source_compile_times.append( + (os.path.relpath(args["source"], cmdargs.root), duration_ms) + ) + case _ as role: + stage_times[role] += duration_ms + + print("\nStage durations") + for stage in [Role.CONFIGURE, Role.GENERATE, Role.CMAKEBUILD, Role.INSTALL]: + if stage in stage_times: + print(f"{stage.name:>12}: {format_millis(stage_times[stage])}") + log_top10(source_compile_times, "\nSource compile CPU time") + log_top10(list(target_compile_times.items()), "\nTarget compile CPU time") + log_top10(list(target_link_times.items()), "\nTarget link CPU time") + log_top10(custom_command_times, "\nCustom command CPU time") + print(f"\nLoad {trace_path} into ui.perfetto.dev for the complete trace") + + +if __name__ == "__main__": + main()