Move Cuda usage into a shared library similar to onnxruntime (#973)

This separates out the cuda code into a /src/cuda folder and builds
onnxruntime_genai_cuda.dll/.so.
It is only this shared library that depends on the cuda runtimes.
There is an interface inside the core genai code to route functions to
this cuda library, but my next step is to refactor the code to separate
out provider specific, so this interface will shrink.

There is a new type to manage provider specific memory, DeviceMemory<T>
and DeviceMemorySpan<T>, this will replace RoamingArray once the
refactoring is complete.

Author: RyanUnderhill

.github/workflows/linux-cpu-arm64-build.yml

@@ -92,4 +92,4 @@ jobs:
         run: |
           docker run --rm \
           --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-          -w /onnxruntime_src ort_genai_linux_arm64_gha bash -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/onnxruntime_src/ort/lib/ /onnxruntime_src/build/cpu/test/unit_tests"
+          -w /onnxruntime_src ort_genai_linux_arm64_gha bash -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/onnxruntime_src/ort/lib/ /onnxruntime_src/build/cpu/unit_tests"

.github/workflows/linux-cpu-x64-build.yml

@@ -99,4 +99,4 @@ jobs:
           set -e -x
           export ORTGENAI_LOG_ORT_LIB=1
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/ort/lib
-          ./build/cpu/test/unit_tests
+          ./build/cpu/unit_tests

.github/workflows/linux-gpu-x64-build.yml

@@ -147,4 +147,4 @@ jobs:
             --rm \
             --volume /data/ortgenai_pytorch_models:/data/ortgenai_pytorch_models \
             --volume $GITHUB_WORKSPACE:/ort_genai_src \
-            -w /ort_genai_src onnxruntimecudabuildx64 bash -c "ORTGENAI_LOG_ORT_LIB=1 LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/ort_genai_src/build/cuda/ /ort_genai_src/build/cuda/test/unit_tests"
+            -w /ort_genai_src onnxruntimecudabuildx64 bash -c "ORTGENAI_LOG_ORT_LIB=1 LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/ort_genai_src/build/cuda/ /ort_genai_src/build/cuda/unit_tests"

.github/workflows/mac-cpu-arm64-build.yml

@@ -82,4 +82,4 @@ jobs:
           set -e -x
           export ORTGENAI_LOG_ORT_LIB=1
           export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$GITHUB_WORKSPACE/build/cpu/osx-arm64
-          ./build/cpu/osx-arm64/test/unit_tests
+          ./build/cpu/osx-arm64/unit_tests

.github/workflows/win-cpu-arm64-build.yml

@@ -85,5 +85,5 @@ jobs:
 
     - name: Run tests
       run: |
-        copy $env:GITHUB_WORKSPACE\ort\lib\* .\$env:binaryDir\test\Release
-        & .\$env:binaryDir\test\Release\unit_tests.exe
+        copy $env:GITHUB_WORKSPACE\ort\lib\* .\$env:binaryDir\Release
+        & .\$env:binaryDir\Release\unit_tests.exe

.github/workflows/win-cpu-x64-build.yml

@@ -100,8 +100,8 @@ jobs:
 
     - name: Run tests
       run: |
-        copy $env:GITHUB_WORKSPACE\ort\lib\* .\$env:binaryDir\test\Release
-        & .\$env:binaryDir\test\Release\unit_tests.exe
+        copy $env:GITHUB_WORKSPACE\ort\lib\* .\$env:binaryDir\Release
+        & .\$env:binaryDir\Release\unit_tests.exe
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v3

.github/workflows/win-cuda-x64-build.yml

@@ -103,5 +103,5 @@ jobs:
       run: |
         $env:PATH = "${{ env.cuda_dir }}\\v${{ env.cuda_version }}\\bin;" + $env:PATH 
         echo "Current PATH variable is: $env:PATH" 
-        copy $env:GITHUB_WORKSPACE\ort\lib\* .\$env:binaryDir\test\Release
-        & .\$env:binaryDir\test\Release\unit_tests.exe
+        copy $env:GITHUB_WORKSPACE\ort\lib\* .\$env:binaryDir\Release
+        & .\$env:binaryDir\Release\unit_tests.exe

.github/workflows/win-directml-x64-build.yml

@@ -100,5 +100,5 @@ jobs:
 
     - name: Run tests
       run: |
-        copy $env:GITHUB_WORKSPACE\ort\lib\* $env:GITHUB_WORKSPACE\$env:binaryDir\test\Release
-        & .\$env:binaryDir\test\Release\unit_tests.exe
+        copy $env:GITHUB_WORKSPACE\ort\lib\* $env:GITHUB_WORKSPACE\$env:binaryDir\Release
+        & .\$env:binaryDir\Release\unit_tests.exe

CMakeLists.txt

@@ -110,17 +110,25 @@ if(APPLE)
 target_link_libraries(onnxruntime-genai PRIVATE "-framework Foundation" "-framework CoreML")
 endif()
 
-set_target_properties(onnxruntime-genai PROPERTIES FOLDER "Sources")
-set_target_properties(onnxruntime-genai-static PROPERTIES FOLDER "Sources")
-source_group(TREE ${PROJECT_SOURCE_DIR} FILES ${generator_srcs})
 
+# Build all source files using CUDA as a separate shared library we dynamically load at runtime
 if(USE_CUDA AND CMAKE_CUDA_COMPILER)
-  set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA)
-  target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas curand cufft cudart)
-  #  onnxruntime-genai-static is statically linked under Windows
-  if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
-    set_target_properties(onnxruntime-genai-static PROPERTIES LINKER_LANGUAGE CUDA)
-    target_link_libraries(onnxruntime-genai-static PRIVATE cublasLt cublas curand cufft cudart)
+  add_library(onnxruntime-genai-cuda SHARED ${generator_cudalib_srcs})
+  target_include_directories(onnxruntime-genai-cuda PRIVATE ${ORT_HEADER_DIR})
+  target_include_directories(onnxruntime-genai-cuda PRIVATE ${GENERATORS_ROOT})
+  target_link_libraries(onnxruntime-genai-cuda PRIVATE cublasLt cublas curand cufft cudart)
+  set_target_properties(onnxruntime-genai-cuda PROPERTIES LINKER_LANGUAGE CUDA)
+  add_dependencies(onnxruntime-genai onnxruntime-genai-cuda)
+  source_group(TREE ${GENERATORS_ROOT}/cuda FILES ${generator_cudalib_srcs})
+  list(APPEND ortgenai_embed_libs "$<TARGET_FILE:onnxruntime-genai-cuda>")
+  if(APPLE)
+    set_property(TARGET onnxruntime-genai-cuda APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${GENERATORS_ROOT}/cuda/exported_symbols.lst")
+  elseif(UNIX)
+    set_property(TARGET onnxruntime-genai-cuda APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker --version-script=${GENERATORS_ROOT}/cuda/version_script.lds -Xlinker --gc-sections")
+  elseif(WIN32)
+    set_property(TARGET onnxruntime-genai-cuda APPEND_STRING PROPERTY LINK_FLAGS "-DEF:${GENERATORS_ROOT}/cuda/symbols.def")
+  else()
+    message(FATAL_ERROR "${target} unknown platform, need to specify shared library exports for it")
   endif()
 endif()
 
@@ -202,7 +210,6 @@ if (ENABLE_JAVA)
   add_subdirectory("${SRC_ROOT}/java")
 endif()
 
-
 if(ENABLE_MODEL_BENCHMARK)
   message("------------------Enabling model benchmark------------------")
   add_subdirectory("${REPO_ROOT}/benchmark/c")

cmake/check_cuda.cmake

@@ -39,18 +39,18 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER)
 
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcudafe --diag_suppress=2803 --expt-relaxed-constexpr")
 
-  file(GLOB generator_cuda_srcs CONFIGURE_DEPENDS
-    "${GENERATORS_ROOT}/*.cu"
-    "${GENERATORS_ROOT}/*.cuh"
-    "${MODELS_ROOT}/*.cu"
-    "${MODELS_ROOT}/*.cuh"
+  file(GLOB generator_cudalib_srcs CONFIGURE_DEPENDS
+    "${GENERATORS_ROOT}/cuda/*.cpp"
+    "${GENERATORS_ROOT}/cuda/*.h"
+    "${GENERATORS_ROOT}/cuda/*.cu"
+    "${GENERATORS_ROOT}/cuda/*.cuh"
   )
+
   file(GLOB test_cuda_srcs CONFIGURE_DEPENDS
     "${TESTS_ROOT}/*.cu"
     "${TESTS_ROOT}/*.cuh"
   )
   list(APPEND test_srcs ${test_cuda_srcs})
-  list(APPEND generator_srcs ${generator_cuda_srcs})
   add_compile_definitions(USE_CUDA=1)
   include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
 elseif(USE_CUDA)