vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-10-20 02:32:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enable CUDA graphs with explicit architectures and add opt-in ADD capture (#2543)

Browse source 
* cmake: enable CUDA graphs with explicit architectures

* cuda: add opt-in ADD graph capture and rejection diagnostics
This commit is contained in:
askmyteapot 2026-10-18 13:15:19 +10:00 • committed by GitHub
parent 9cb9f5edc1
commit 8ccdd068a7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 15 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

7
CMakeLists.txt
View file

@ -142,6 +142,11 @@ if (LLAMA_CUBLAS)
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cuda_driver)
# CUDA graphs are independent of whether GPU architectures were supplied.
if(CUDAToolkit_VERSION VERSION_GREATER 12)
add_compile_definitions(GGML_CUDA_USE_GRAPHS)
endif()
if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
# 50 == lowest CUDA 12 standard
# 60 == f16 CUDA intrinsics
@ -151,11 +156,9 @@ if (LLAMA_CUBLAS)
# 80 == Ampere, asynchronous data loading, faster tensor core instructions
message("CUDA Toolkit Version: ${CUDAToolkit_VERSION}")
if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 13)
add_compile_definitions(GGML_CUDA_USE_GRAPHS) #try enable cuda graphs on cu12 build
add_compile_definitions(KCPP_LIMIT_CUDA_MAX_ARCH=860)
set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-virtual") # lowest CUDA 13 standard
elseif(CUDAToolkit_VERSION VERSION_GREATER 12)
add_compile_definitions(GGML_CUDA_USE_GRAPHS) #try enable cuda graphs on cu12 build
add_compile_definitions(KCPP_LIMIT_CUDA_MAX_ARCH=800)
set(CMAKE_CUDA_ARCHITECTURES "50-virtual;61-virtual;75-virtual;80-virtual") # lowest CUDA 12 standard + lowest for integer intrinsics
else()

12
ggml/src/ggml-cuda/ggml-cuda.cu
View file

@ -2705,6 +2705,9 @@ static bool ggml_cuda_graph_debug_enabled() {
static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {
bool use_cuda_graph = true;
// Opt-in ADD capture: preserve the existing restriction by default.
// This does not bypass other capture compatibility checks.
static const bool allow_add_capture = ggml_cuda_env_flag("GGML_CUDA_GRAPH_ALLOW_ADD");
// Loop over nodes in GGML graph to obtain info needed for CUDA graph
for (int i = 0; i < cgraph->n_nodes; i++) {
@ -2722,13 +2725,18 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {
// ref: https://github.com/ggml-org/llama.cpp/pull/18958
use_cuda_graph = false;
if (ggml_cuda_graph_debug_enabled()) {
GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__);
GGML_LOG_DEBUG("%s: CUDA%d disabling CUDA graphs at MUL_MAT_ID node '%s'\n",
__func__, ggml_cuda_get_device(), node->name);
}
}
}
if (node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1) {
if (!allow_add_capture && node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1) {
use_cuda_graph = false;
if (ggml_cuda_graph_debug_enabled()) {
GGML_LOG_DEBUG("%s: CUDA%d disabling CUDA graphs at ADD node '%s', src1 ne[1]=%lld\n",
__func__, ggml_cuda_get_device(), node->name, (long long) node->src[1]->ne[1]);
}
}
if (!use_cuda_graph) {