diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e91fe28d..35eb16910 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,11 @@ if (LLAMA_CUBLAS) set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cuda_driver) + # CUDA graphs are independent of whether GPU architectures were supplied. + if(CUDAToolkit_VERSION VERSION_GREATER 12) + add_compile_definitions(GGML_CUDA_USE_GRAPHS) + endif() + if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES) # 50 == lowest CUDA 12 standard # 60 == f16 CUDA intrinsics @@ -151,11 +156,9 @@ if (LLAMA_CUBLAS) # 80 == Ampere, asynchronous data loading, faster tensor core instructions message("CUDA Toolkit Version: ${CUDAToolkit_VERSION}") if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 13) - add_compile_definitions(GGML_CUDA_USE_GRAPHS) #try enable cuda graphs on cu12 build add_compile_definitions(KCPP_LIMIT_CUDA_MAX_ARCH=860) set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-virtual") # lowest CUDA 13 standard elseif(CUDAToolkit_VERSION VERSION_GREATER 12) - add_compile_definitions(GGML_CUDA_USE_GRAPHS) #try enable cuda graphs on cu12 build add_compile_definitions(KCPP_LIMIT_CUDA_MAX_ARCH=800) set(CMAKE_CUDA_ARCHITECTURES "50-virtual;61-virtual;75-virtual;80-virtual") # lowest CUDA 12 standard + lowest for integer intrinsics else() diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index c1f337dd4..e3b802d88 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2705,6 +2705,9 @@ static bool ggml_cuda_graph_debug_enabled() { static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { bool use_cuda_graph = true; + // Opt-in ADD capture: preserve the existing restriction by default. + // This does not bypass other capture compatibility checks. + static const bool allow_add_capture = ggml_cuda_env_flag("GGML_CUDA_GRAPH_ALLOW_ADD"); // Loop over nodes in GGML graph to obtain info needed for CUDA graph for (int i = 0; i < cgraph->n_nodes; i++) { @@ -2722,13 +2725,18 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { // ref: https://github.com/ggml-org/llama.cpp/pull/18958 use_cuda_graph = false; if (ggml_cuda_graph_debug_enabled()) { - GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__); + GGML_LOG_DEBUG("%s: CUDA%d disabling CUDA graphs at MUL_MAT_ID node '%s'\n", + __func__, ggml_cuda_get_device(), node->name); } } } - if (node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1) { + if (!allow_add_capture && node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1) { use_cuda_graph = false; + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA%d disabling CUDA graphs at ADD node '%s', src1 ne[1]=%lld\n", + __func__, ggml_cuda_get_device(), node->name, (long long) node->src[1]->ne[1]); + } } if (!use_cuda_graph) {