ktransformers天坑
当前版本0.3.2 torch12.6 cuda12.6 toolkit12.6
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 60
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 60
export PATH=$PATH:/usr/local/cuda-12.6/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.6/lib64
export CUDA_PATH=/usr/local/cuda-12.6
export CUDA_HOME=/usr/local/cuda-12.6
export TORCH_CUDA_ARCH_LIST="8.6"
export USE_NUMA=1
export USE_BALANCE_SERVE=1
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 14
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 100
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 100
当前会出现layer_type_validation找不到，需更新transformers
pip install git+https://github.com/huggingface/transformers.git@main