Qwen3-235B-A22B-W8A8

一. 说明

根据官方脚本，结合网上大佬的旋转量化方法，解决了上个版本 "游戏副本"字段 、"方块乱码" 问题

• https://gitcode.com/Ascend/msit (新地址)
• https://gitee.com/ascend/msit (已暂停)

二. 模型下载(SDK)

# 安装ModelScope
pip install modelscope

from modelscope import snapshot_download
model_dir = snapshot_download('jaywoods/Qwen3-235B-A22B-W8A8')

三. 服务化推理

1. 创建mindie容器

对应的镜像介质，从昇腾官方申请获取

# 拉取镜像
docker pull swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.2.RC1-800I-A2-py311-openeuler24.03-lts

# 创建容器
docker run -it -d --net=host --shm-size=800g \
    --privileged \
    --name qwen3 \
    -v /usr/local/Ascend/driver:/usr/local/Ascend/driver:ro \
    -v /usr/local/sbin:/usr/local/sbin:ro \
    -v /data/models:/data/models \
    swr.cn-south-1.myhuaweicloud.com/ascendhub/mindie:2.2.RC1-800I-A2-py311-openeuler24.03-lts bash

2. 创建配置文件

cd /usr/local/Ascend/mindie/latest/mindie-service
vim conf/config.json

配置文件内容参考如下:

{
    "Version" : "2.2.RC1",

    "ServerConfig" :
    {
        "ipAddress" : "127.0.0.1",
        "managementIpAddress" : "127.0.0.1",
        "port" : 1025,
        "managementPort" : 1026,
        "metricsPort" : 1027,
        "allowAllZeroIpListening" : false,
        "maxLinkNum" : 1000,
        "httpsEnabled" : false,
        "fullTextEnabled" : false,
        "tlsCaPath" : "security/ca/",
        "tlsCaFile" : ["ca.pem"],
        "tlsCert" : "security/certs/server.pem",
        "tlsPk" : "security/keys/server.key.pem",
        "tlsPkPwd" : "security/pass/key_pwd.txt",
        "tlsCrlPath" : "security/certs/",
        "tlsCrlFiles" : ["server_crl.pem"],
        "managementTlsCaFile" : ["management_ca.pem"],
        "managementTlsCert" : "security/certs/management/server.pem",
        "managementTlsPk" : "security/keys/management/server.key.pem",
        "managementTlsPkPwd" : "security/pass/management/key_pwd.txt",
        "managementTlsCrlPath" : "security/management/certs/",
        "managementTlsCrlFiles" : ["server_crl.pem"],
        "kmcKsfMaster" : "tools/pmt/master/ksfa",
        "kmcKsfStandby" : "tools/pmt/standby/ksfb",
        "inferMode" : "standard",
        "interCommTLSEnabled" : true,
        "interCommPort" : 1121,
        "interCommTlsCaPath" : "security/grpc/ca/",
        "interCommTlsCaFiles" : ["ca.pem"],
        "interCommTlsCert" : "security/grpc/certs/server.pem",
        "interCommPk" : "security/grpc/keys/server.key.pem",
        "interCommPkPwd" : "security/grpc/pass/key_pwd.txt",
        "interCommTlsCrlPath" : "security/grpc/certs/",
        "interCommTlsCrlFiles" : ["server_crl.pem"],
        "openAiSupport" : "vllm",
        "tokenTimeout" : 600,
        "e2eTimeout" : 600,
        "distDPServerEnabled":false
    },

    "BackendConfig" : {
        "backendName" : "mindieservice_llm_engine",
        "modelInstanceNumber" : 1,
        "npuDeviceIds" : [[0,1,2,3,4,5,6,7]],
        "tokenizerProcessNumber" : 8,
        "multiNodesInferEnabled" : false,
        "multiNodesInferPort" : 1120,
        "interNodeTLSEnabled" : true,
        "interNodeTlsCaPath" : "security/grpc/ca/",
        "interNodeTlsCaFiles" : ["ca.pem"],
        "interNodeTlsCert" : "security/grpc/certs/server.pem",
        "interNodeTlsPk" : "security/grpc/keys/server.key.pem",
        "interNodeTlsPkPwd" : "security/grpc/pass/mindie_server_key_pwd.txt",
        "interNodeTlsCrlPath" : "security/grpc/certs/",
        "interNodeTlsCrlFiles" : ["server_crl.pem"],
        "interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
        "interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb",
        "ModelDeployConfig" :
        {
            "maxSeqLen" : 32768,
            "maxInputTokenLen" : 16384,
            "truncation" : false,
            "ModelConfig" : [
                {
                    "modelInstanceType" : "Standard",
                    "modelName" : "qwen3-235b",
                    "modelWeightPath" : "/data/models/Qwen/Qwen3-235B-A22B-W8A8",
                    "worldSize" : 8,
                    "cpuMemSize" : 100,
                    "npuMemSize" : -1,
                    "backendType" : "atb",
                    "trustRemoteCode" : true
                }
            ]
        },

        "ScheduleConfig" :
        {
            "templateType" : "Standard",
            "templateName" : "Standard_LLM",
            "cacheBlockSize" : 128,

            "maxPrefillBatchSize" : 30,
            "maxPrefillTokens" : 32768,
            "prefillTimeMsPerReq" : 150,
            "prefillPolicyType" : 0,

            "decodeTimeMsPerReq" : 50,
            "decodePolicyType" : 0,

            "maxBatchSize" : 30,
            "maxIterTimes" : 8192,
            "maxPreemptCount" : 0,
            "supportSelectBatch" : false,
            "maxQueueDelayMicroseconds" : 5000
        }
    }
}

3. 调整模型文件权限

chmod 750 /data/models/Qwen/Qwen3-235B-A22B-W8A8/config.json

4. 启动

cd /usr/local/Ascend/mindie/latest/mindie-service
nohup bin/mindieservice_daemon > mindieservice.log 2>&1 &

四. 接口请求示例

# 支持兼容vllm格式

curl 'http://127.0.0.1:1025/v1/chat/completions'  \
  -H 'Content-Type: application/json'  \
  -d '{
  "model": "qwen3-235b",
  "stream": false,
  "chat_template_kwargs": {
        "enable_thinking": false
  },
  "messages": [
    {
        "content": [{
            "type": "text",
            "text": "你是谁?"
        }],
        "role": "user"
    }
  ]
}'