Upgrade candle3

Cargo.toml

@@ -46,14 +46,20 @@ serde_json = "1.0"
 thiserror = "1.0"
 rand = "0.9"
 serial_test = "2.0.0"
-# cudarc = { version = "0.13" , features =["cuda-version-from-build-system"]}
-cudarc = { version = "0.10", default-features = false }
-candle = { git = "https://github.com/OlivierDehaene/candle", rev = "7e02ad856104799b73a946ac1e153f0de77feaaf", package = "candle-core" }
-candle-nn = { git = "https://github.com/OlivierDehaene/candle", rev = "7e02ad856104799b73a946ac1e153f0de77feaaf", package = "candle-nn" }
-candle-transformers = { git = "https://github.com/OlivierDehaene/candle", rev = "7e02ad856104799b73a946ac1e153f0de77feaaf", package = "candle-transformers" }
-candle-flash-attn = { git = "https://github.com/OlivierDehaene/candle", rev = "7e02ad856104799b73a946ac1e153f0de77feaaf", package = "candle-flash-attn" }
+cudarc = { version = "0.13" , features =["cuda-12020"]}
+candle = { version = "0.8", package = "candle-core" }
+candle-nn = { version = "0.8", package = "candle-nn" }
+candle-transformers = { version = "0.8", package = "candle-transformers" }
+candle-flash-attn = { version = "0.8", package = "candle-flash-attn" }
 half = { version = "2.3.1", features = ["num-traits"] }
 
+[patch.crates-io]
+cudarc = { git = "https://github.com/Narsil/cudarc" , rev = "1956436aeddea1da04fc3226282bc07c07eeaa35"}
+candle = { git = "https://github.com/Narsil/candle", rev = "2e273ddf31b1b796d3cfcd181ccb98deaa48466e", package = "candle-core" }
+candle-nn = { git = "https://github.com/Narsil/candle", rev = "2e273ddf31b1b796d3cfcd181ccb98deaa48466e", package = "candle-nn" }
+candle-transformers = { git = "https://github.com/Narsil/candle", rev = "2e273ddf31b1b796d3cfcd181ccb98deaa48466e", package = "candle-transformers" }
+candle-flash-attn = { git = "https://github.com/Narsil/candle", rev = "2e273ddf31b1b796d3cfcd181ccb98deaa48466e", package = "candle-flash-attn" }
+
 [profile.release]
 debug = 0
 # lto = "fat"

backends/Cargo.toml

@@ -27,6 +27,5 @@ metal = ["text-embeddings-backend-candle?/metal"]
 mkl = ["text-embeddings-backend-candle?/mkl"]
 mkl-dynamic = ["text-embeddings-backend-candle?/mkl-dynamic"]
 accelerate = ["text-embeddings-backend-candle?/accelerate"]
-static-linking = ["text-embeddings-backend-candle?/static-linking"]
 flash-attn = ["text-embeddings-backend-candle?/flash-attn"]
 flash-attn-v1 = ["text-embeddings-backend-candle?/flash-attn-v1"]

backends/candle/Cargo.toml

@@ -40,9 +40,8 @@ anyhow = { version = "1", features = ["backtrace"] }
 [features]
 accelerate = ["dep:accelerate-src", "candle/accelerate", "candle-nn/accelerate"]
 metal = ["candle/metal", "candle-nn/metal"]
-mkl = ["dep:intel-mkl-src", "intel-mkl-src/mkl-static-lp64-iomp", "candle/mkl", "candle-nn/mkl"]
-mkl-dynamic = ["dep:intel-mkl-src", "intel-mkl-src/mkl-dynamic-lp64-iomp", "candle/mkl-dynamic", "candle-nn/mkl-dynamic"]
+mkl = ["dep:intel-mkl-src", "intel-mkl-src/mkl-static-lp64-iomp", "candle/mkl"]
+mkl-dynamic = ["dep:intel-mkl-src", "intel-mkl-src/mkl-dynamic-lp64-iomp", "candle/mkl"]
 cuda = ["candle/cuda", "candle-nn/cuda", "dep:candle-cublaslt", "dep:candle-layer-norm", "dep:candle-rotary"]
 flash-attn-v1 = ["dep:candle-flash-attn-v1", "cuda"]
 flash-attn = ["dep:candle-flash-attn", "cuda"]
-static-linking = ["candle-cublaslt?/static-linking"]

candle-extensions/candle-cublaslt/Cargo.toml

@@ -9,6 +9,3 @@ description = "CUBLASLt gemm for the candle ML framework."
 candle = { workspace=true, features = ["cuda"]}
 cudarc = { workspace = true, features = [ "cublaslt", "f16" ]}
 half = { workspace = true}
-
-[features]
-static-linking = ["cudarc/static-linking"]

candle-extensions/candle-flash-attn-v1/Cargo.toml

@@ -20,4 +20,4 @@ rayon = "1.7.0"
 
 [dev-dependencies]
 anyhow = { version = "1", features = ["backtrace"] }
-candle-nn = { version = "0.3.0", features = ["cuda"] }
+candle-nn = { workspace = true }

candle-extensions/candle-rotary/Cargo.toml

@@ -19,4 +19,4 @@ bindgen_cuda = "0.1.1"
 
 [dev-dependencies]
 anyhow = { version = "1", features = ["backtrace"] }
-candle-nn = { version = "0.3.0", features = ["cuda"] }
+candle-nn = { workspace = true }

flake.nix

@@ -148,6 +148,11 @@
         #       hash = "sha256-1AN2E9t/lZhbXdVznhTcniy+7ZzlaEp/gwLEAucs6EA=";
         #       # hash = lib.fakeHash;
         #     };
+        mkl2024 = import ./nix/mkl.nix;
+
+        onnxruntimeGcc13 = pkgs.onnxruntime.override {
+          stdenv = pkgs.cudaPackages.backendStdenv;
+        };
 
       in
       #     cargoDeps = pkgs.rustPlatform.fetchCargoVendor {
@@ -195,7 +200,7 @@
         devShells.default =
           pkgs.mkShell.override
             {
-              stdenv = pkgs.gcc13Stdenv;
+              stdenv = pkgs.cudaPackages.backendStdenv;
             }
             {
 
@@ -208,6 +213,8 @@
                 cudaPackages.cudatoolkit
                 python3Packages.python
                 python3Packages.venvShellHook
+                onnxruntimeGcc13
+                mkl
               ];
               venvDir = "./.venv";
               LD_LIBRARY_PATH = "${pkgs.stdenv.cc.cc.lib}/lib:/run/opengl-driver/lib";

router/Cargo.toml

@@ -57,6 +57,9 @@ tonic-health = { version = "0.11.0", optional = true }
 tonic-reflection = { version = "0.11.0", optional = true }
 tokio-stream = { version = "0.1.14", optional = true }
 
+# Optional
+cudarc = { workspace = true, optional = true }
+
 # Malloc trim hack for linux
 [target.'cfg(target_os = "linux")'.dependencies]
 libc = "0.2.149"
@@ -88,5 +91,5 @@ candle = ["text-embeddings-backend/candle"]
 candle-cuda = ["candle", "text-embeddings-backend/flash-attn"]
 candle-cuda-turing = ["candle", "text-embeddings-backend/flash-attn-v1"]
 candle-cuda-volta = ["candle", "text-embeddings-backend/cuda"]
-static-linking = ["text-embeddings-backend/static-linking"]
+static-linking = ["cudarc/static-linking"]
 google = []