普惠AI 如何在 Anolis OS 8 上部署生产可用的 DeepSeek 推理服务 原创

dnf install -y anolis-epao-release
dnf install -y kernel-devel-$(uname -r) nvidia-driver{,-cuda}
1.
2.

dnf config-manager --add-repo=https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
dnf install -y docker-ce nvidia-container-toolkit
systemctl restart docker
1.
2.
3.

dnf install -y curl jq git{,-lfs}
git lfs install
1.
2.

git clone https://www.modelscope.cn/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B.git
1.

docker run -d -p 8000:8000 --gpus all \
  -v ./DeepSeek-R1-Distill-Qwen-7B:/models/DeepSeek-R1-Distill-Qwen-7B \
  -e MODEL_DIR=/models/DeepSeek-R1-Distill-Qwen-7B \
  ac2-registry.cn-hangzhou.cr.aliyuncs.com/ac2/qwen2:7b-pytorch2.4.0-cuda12.1.1-py310-alinux3.2104
1.
2.
3.
4.

INFO 02-11 01:16:33 launcher.py:19] Available routes are:
INFO 02-11 01:16:33 launcher.py:27] Route: /openapi.json, Methods: GET, HEAD
INFO 02-11 01:16:33 launcher.py:27] Route: /docs, Methods: GET, HEAD
INFO 02-11 01:16:33 launcher.py:27] Route: /docs/oauth2-redirect, Methods: GET, HEAD
INFO 02-11 01:16:33 launcher.py:27] Route: /redoc, Methods: GET, HEAD
INFO 02-11 01:16:33 launcher.py:27] Route: /health, Methods: GET
INFO 02-11 01:16:33 launcher.py:27] Route: /tokenize, Methods: POST
INFO 02-11 01:16:33 launcher.py:27] Route: /detokenize, Methods: POST
INFO 02-11 01:16:33 launcher.py:27] Route: /v1/models, Methods: GET
INFO 02-11 01:16:33 launcher.py:27] Route: /version, Methods: GET
INFO 02-11 01:16:33 launcher.py:27] Route: /v1/chat/completions, Methods: POST
INFO 02-11 01:16:33 launcher.py:27] Route: /v1/completions, Methods: POST
INFO 02-11 01:16:33 launcher.py:27] Route: /v1/embeddings, Methods: POST
INFO:     Started server process [10]
INFO:     Waiting for application startup.
INFO:     Application startup complete.
INFO:     Uvicorn running on socket ('0.0.0.0', 8000) (Press CTRL+C to quit)
INFO 02-11 01:16:43 metrics.py:345] Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%.
INFO 02-11 01:16:53 metrics.py:345] Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%.
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.

curl -s http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model": "/models/DeepSeek-R1-Distill-Qwen-7B", 
       "messages": [{"role": "user", "content": "9.9和9.11哪个大？"}], 
       "max_tokens": 1024, "temperature":"0.9" 
      }' | \
  jq '.choices[0].message.content' | \
  xargs echo -e
1.
2.
3.
4.
5.
6.
7.
8.