LLM 推理管理

Service：推理服务的顶层概念，包含模型配置、引擎配置等
Role：服务中的不同组件，如 worker、controller 等
Instance：某个 role 的具体运行实例，对应一个 Kubernetes Pod
Engine：推理引擎类型，如 vllm、triton 等

客户端初始化

from siflow import SiFlow

client = SiFlow(
    region="cn-shanghai",
    cluster="hercules",
    access_key_id="your_key_id",
    access_key_secret="your_secret"
)

inference = client.inference

创建服务

from siflow.types import (
    ServiceCreateParams,
    LLMServiceConfig,
    LLMServingEngineConfig,
    LLMModelConfig,
    LLMWorkloadConfig,
    LLMResourceConfig,
    LLMRoleConfig,
    LLMModelSource,
    LLMStorage,
    LLMPVCConfig,
    LLMServicePort,
    LLMProbe,
    LLMProbeConfig,
)

model_source = LLMModelSource(
    #storageType="pvc",
    #storage=LLMStorage(
    #    pvc=LLMPVCConfig(
    #        persistentVolumeClaimName="model-pvc",
    #        modelPath="/models/llama-2-7b",
    #        mountPath="/mnt/models"
    #    )
    #)
)

model_config = LLMModelConfig(
    modelSource=model_source
)

serving_engine_config = LLMServingEngineConfig(
    engineType="vllm",  # custom,vllm,dynamo-vllm,dynamo-sglang
    #engineVersion="0.8.5",
    executeType="single-node"
)

service_config = LLMServiceConfig(
    servicePort=LLMServicePort(
        name="http",
        port=8000
    ),
    readinessProbe=LLMProbe(
        probeType="http",
        probeConfig=LLMProbeConfig(
            probePath="/health",
            probePort=8000
        ),
        timeout=60
    ),
    livenessProbe=LLMProbe(
        probeType="http",
        probeConfig=LLMProbeConfig(
            probePath="/health",
            probePort=8000
        ),
        timeout=60
    )
)
# Advanced Options: Workload Configuration (usually left blank)
# workload_config = LLMWorkloadConfig(
#     type="deployment"
# )

# Define role configuration
worker_role_config = LLMRoleConfig(
    replicas=1,
    # required fo customer engine
    # image="vllm/vllm-openai:latest",
    # command=""，
    resourceConfig=LLMResourceConfig(
        instanceName="sci.g16-2",
        instanceQuantity=1
    )
)

service_params = ServiceCreateParams(
    name="llama-2-7b-service",
    description="LLaMA 2 7B inference service",
    tenant="simaas",
    owner="smadmin",
    resourcePool="cn-shanghai-hercules-simaas-ondemand-shared",
    serviceConfig=service_config,
    servingEngineConfig=serving_engine_config,
    #modelConfig=model_config,
    #workloadConfig=workload_config,
    roleConfig={"worker": worker_role_config}
)

service_id = inference.create_service(service_params=service_params)
print(f"Service created with ID: {service_id}")

查询服务

services = inference.list_services()
for service in services:
    print(f"ID: {service.id}, Name: {service.name}, Status: {service.status.status}")

service_details = inference.get_service(service_id=service_id)
print(f"Service: {service_details.name}")
print(f"Status: {service_details.status.status}")
print(f"Created: {service_details.created_at}")
print(f"urlInCluster: {service_details.status.url_in_cluster}")
print(f"urlExternal: {service_details.status.url_external}")

下线/上线服务

# offline
inference.offline_service(service_id=service_id)

# online
inference.online_service(service_id=service_id)

删除服务

inference.delete_service(service_id=service_id)

示例

vLLM 单节点

#!/usr/bin/env python3
"""
Example usage of the SiFlow Inference API

This example demonstrates how to use the inference service API to:
1. Create a new inference service
2. List services
3. Get service details
4. Scale a service
5. Manage service lifecycle (online/offline/delete)
"""

from siflow import SiFlow
from siflow.types import (
    ServiceCreateParams,
    LLMServiceConfig,
    LLMServingEngineConfig,
    LLMModelConfig,
    LLMWorkloadConfig,
    LLMResourceConfig,
    LLMRoleConfig,
    LLMModelSource,
    LLMStorage,
    LLMPVCConfig,
    LLMServicePort,
    LLMProbe,
    LLMProbeConfig,
)

def main():
    # Initialize the SiFlow client
    client = SiFlow(
        region="cn-shanghai",
        cluster="hercules",
        access_key_id="XXX",
        access_key_secret="XXX"
    )

    # Example 1: Create a new inference service
    print("Creating a new inference service...")

    # Define the model source (using PVC storage)
    model_source = LLMModelSource(
        storageType="pvc",
        storage=LLMStorage(
            pvc=LLMPVCConfig(
                persistentVolumeClaimName="model-pvc",
                modelPath="/models/llama-2-7b",
                mountPath="/mnt/models"
            )
        )
    )

    # Define the model configuration
    model_config = LLMModelConfig(
        modelSource=model_source
    )

    # Define the serving engine configuration
    serving_engine_config = LLMServingEngineConfig(
        engineType="vllm",
        engineVersion="0.8.5",
        executeType="single-node"
    )

    # Define the service configuration
    service_config = LLMServiceConfig(
        # Add service port, probes, etc. as needed
        servicePort=LLMServicePort(
            name="http",
            port=8000
        ),
        readinessProbe=LLMProbe(
            probeType="http",
            probeConfig=LLMProbeConfig(
                probePath="/health",
                probePort=8000
            ),
            timeout=60
        ),
        livenessProbe=LLMProbe(
            probeType="http",
            probeConfig=LLMProbeConfig(
                probePath="/health",
                probePort=8000
            ),
            timeout=60
        )
    )

    # Define the workload configuration
    workload_config = LLMWorkloadConfig(
        type="deployment"
    )

    # Define role configuration for the worker
    worker_role_config = LLMRoleConfig(
        replicas=1,
        image="vllm/vllm-openai:latest",
        resourceConfig=LLMResourceConfig(
            instanceName="sci.g16-2",
            instanceQuantity=1
        )
    )

    # Create service parameters
    service_params = ServiceCreateParams(
        name="llama-2-7b-service",
        description="LLaMA 2 7B inference service",
        tenant="simaas",
        owner="smadmin",
        resourcePool="cn-shanghai-hercules-simaas-ondemand-shared",
        serviceConfig=service_config,
        servingEngineConfig=serving_engine_config,
        modelConfig=model_config,
        workloadConfig=workload_config,
        roleConfig={"worker": worker_role_config}
    )

    try:
        service_id = client.inference.create_service(service_params=service_params)
        print(f"Service created successfully with ID: {service_id}")
    except Exception as e:
        print(f"Failed to create service: {e}")
        return

    # Example 2: List all services
    print("\nListing all services...")
    try:
        services = client.inference.list_services()
        print(f"Found {len(services)} services:")
        for service in services:
            print(f"  - ID: {service.id}, Name: {service.name}, Status: {service.status.status if service.status else 'Unknown'}")
    except Exception as e:
        print(f"Failed to list services: {e}")

    # Example 3: Get service details
    print(f"\nGetting details for service {service_id}...")
    try:
        service_details = client.inference.get_service(service_id=service_id)
        print(f"Service details:")
        print(f"  - Name: {service_details.name}")
        print(f"  - Description: {service_details.description}")
        print(f"  - Status: {service_details.status.status if service_details.status else 'Unknown'}")
        print(f"  - Created: {service_details.created_at}")
    except Exception as e:
        print(f"Failed to get service details: {e}")

    # Example 4: Scale the service
    print(f"\nScaling service {service_id}...")
    try:
        from siflow.types import ServiceScaleParams

        # Scale to 2 replicas
        scale_params = ServiceScaleParams(
            roleConfig={"worker": {"replicas": 2}}
        )

        client.inference.scale_service(service_id=service_id, scale_params=scale_params)
        print("Service scaled successfully")
    except Exception as e:
        print(f"Failed to scale service: {e}")

    # Example 5: List service instances (pods)
    print(f"\nListing instances for service {service_id}...")
    try:
        instances = client.inference.list_service_instances(service_id=service_id)
        print(f"Service instances:")
        for role_type, role_instances in instances.items():
            print(f"  {role_type}:")
            for instance in role_instances:
                print(f"    - {instance.name}: {instance.status}")
    except Exception as e:
        print(f"Failed to list service instances: {e}")

    # Example 6: Take service offline
    print(f"\nTaking service {service_id} offline...")
    try:
        client.inference.offline_service(service_id=service_id)
        print("Service taken offline successfully")
    except Exception as e:
        print(f"Failed to take service offline: {e}")

    # Example 7: Bring service back online
    print(f"\nBringing service {service_id} back online...")
    try:
        client.inference.online_service(service_id=service_id)
        print("Service brought back online successfully")
    except Exception as e:
        print(f"Failed to bring service online: {e}")

    # Example 8: List available engine options
    print("\nListing available engine options...")
    try:
        engine_options = client.inference.list_engine_options()
        print("Available engine options:")
        for option in engine_options:
            print(f"  - {option.engine_type} {option.engine_version} ({option.execute_type})")
    except Exception as e:
        print(f"Failed to list engine options: {e}")

    # Example 9: Delete the service (commented out for safety)
    # print(f"\nDeleting service {service_id}...")
    # try:
    #     client.inference.delete_service(service_id=service_id)
    #     print("Service deleted successfully")
    # except Exception as e:
    #     print(f"Failed to delete service: {e}")

if __name__ == "__main__":
    main()

自定义引擎单节点

from siflow import SiFlow

client = SiFlow(
    region="cn-beijing",
    cluster="auriga",
    access_key_id="xxx",
    access_key_secret="xxx"
)

inference = client.inference

from siflow.types import (
    ServiceCreateParams,
    LLMServiceConfig,
    LLMServingEngineConfig,
    LLMModelConfig,
    LLMWorkloadConfig,
    LLMResourceConfig,
    LLMRoleConfig,
    LLMModelSource,
    LLMStorage,
    LLMPVCConfig,
    LLMServicePort,
    LLMProbe,
    LLMProbeConfig,
    LLMStorageConfig,
    LLMVolume,
)

model_source = LLMModelSource(
#     storageType="pvc",
#     storage=LLMStorage(
#         pvc=LLMPVCConfig(
#             persistentVolumeClaimName="pt-train",
#             modelPath="/volume/pt-train/models/Qwen3-0.6B",
#             mountPath="/volume/pt-train"
#         )
#     )
)

model_config = LLMModelConfig(
    modelSource=model_source
)

serving_engine_config = LLMServingEngineConfig(
    engineType="custom",
    executeType="single-node"
)

service_config = LLMServiceConfig(
    servicePort=LLMServicePort(
        name="http",
        port=8000
    ),
    readinessProbe=LLMProbe(
        probeType="http",
        probeConfig=LLMProbeConfig(
            probePath="/health",
            probePort=8000
        ),
        timeout=60
    ),
    livenessProbe=LLMProbe(
        probeType="http",
        probeConfig=LLMProbeConfig(
            probePath="/health",
            probePort=8000
        ),
                timeout=60
    )
)


worker_role_config = LLMRoleConfig(
    replicas=1,
    image="registry-cn-beijing.siflow.cn/siflow/vllm:v0.8.5-ba41cc9",
    command="bash /volume/pt-train/users/cliu05/launch-vllm-serve.sh",
    resourceConfig=LLMResourceConfig(
        instanceName="sci.g21-3",
        instanceQuantity=8
    )
)

storage_config = LLMStorageConfig(
    fileSystemVolumes=[
        LLMVolume(
            volumeId=9,
            name="pt-train",
            mountPath="/volume/pt-train",
        )
    ]
)

service_params = ServiceCreateParams(
    name="qwen3-32b-service",
    description="Qwen3 32B inference service",
    tenant="simaas",
    owner="smadmin",
    resourcePool="pt-eval",
    serviceConfig=service_config,
    servingEngineConfig=serving_engine_config,
    roleConfig={"worker": worker_role_config},
    storageConfig=storage_config,
        env={
        "CONDA_ENV_NAME": "/volume/pt-train/users/wzhang/lyw/conda/envs/vllm",
        "MAX_MODEL_LEN": "32768",
        "TENSOR_PARALLEL_SIZE": "4",
        "DATA_PARALLEL_SIZE": "2",
        "PIPELINE_PARALLEL_SIZE": "1",
        "GPU_MEMORY_UTILIZATION": "0.9",
        "REASONING_PARSER": "",  # e.g., deepseek_r1, qwen2_r1
        "TOOL_CALL_PARSER": "",     # e.g., hermes, qwen2_v1.5
    }
)

service_id = inference.create_service(service_params=service_params)
#service_id=347
print(f"Service created with ID: {service_id}")

services = inference.list_services()
for service in services:
    print(f"ID: {service.id}, Name: {service.name}, Status: {service.status.status}")

service_details = inference.get_service(service_id=service_id)
print(f"Service: {service_details.name}")
print(f"Status: {service_details.status.status}")
print(f"Created: {service_details.created_at}")
print(f"urlInCluster: {service_details.status.url_in_cluster}")
print(f"urlExternal: {service_details.status.url_external}")


inference.offline_service(service_id=service_id)
inference.delete_service(service_id=service_id)
# inference.online_service(service_id=service_id)

自定义引擎多节点

from siflow import SiFlow

client = SiFlow(
    region="cn-beijing",
    cluster="auriga",
    access_key_id="xxx",
    access_key_secret="xxx"
)

inference = client.inference

from siflow.types import (
    ServiceCreateParams,
    LLMServiceConfig,
    LLMServingEngineConfig,
    LLMModelConfig,
    LLMWorkloadConfig,
    LLMResourceConfig,
    LLMRoleConfig,
    LLMModelSource,
    LLMStorage,
    LLMPVCConfig,
    LLMServicePort,
    LLMProbe,
    LLMProbeConfig,
    LLMStorageConfig,
    LLMVolume,
)

model_source = LLMModelSource(
#     storageType="pvc",
#     storage=LLMStorage(
#         pvc=LLMPVCConfig(
#             persistentVolumeClaimName="pt-train",
#             modelPath="/volume/pt-train/models/Qwen3-0.6B",
#             mountPath="/volume/pt-train"
#         )
#     )
)

model_config = LLMModelConfig(
    modelSource=model_source
)

serving_engine_config = LLMServingEngineConfig(
    engineType="custom",
    executeType="distributed"
)

service_config = LLMServiceConfig(
    servicePort=LLMServicePort(
        name="http",
        port=8000
    ),
    readinessProbe=LLMProbe(
        probeType="http",
        probeConfig=LLMProbeConfig(
            probePath="/health",
            probePort=8000
        ),
        timeout=60
    ),
    livenessProbe=LLMProbe(
        probeType="http",
        probeConfig=LLMProbeConfig(
            probePath="/health",
            probePort=8000
        ),
        timeout=60
    )
)

leader_role_config = LLMRoleConfig(
    replicas=1,
    image="registry-cn-beijing.siflow.cn/siflow/vllm:v0.8.5-ba41cc9",
    command="source /volume/pt-train/miniconda3/etc/profile.d/conda.sh;conda activate /volume/pt-train/miniconda3/envs/cheliu_vllm_seed_oss;bash /vllm-workspace/examples/online_serving/multi-node-serving.sh leader --ray_cluster_size=$LWS_GROUP_SIZE; bash /volume/pt-train/users/mingjie/hzl/test/launch-vllm-serve.sh \"default\" $LWS_LEADER_ADDRESS",
    resourceConfig=LLMResourceConfig(
        instanceName="sci.g21-3",
        instanceQuantity=8
    )
)

worker_role_config = LLMRoleConfig(
    replicas=1,
    image="registry-cn-beijing.siflow.cn/siflow/vllm:v0.8.5-ba41cc9",
    command="source /volume/pt-train/miniconda3/etc/profile.d/conda.sh;conda activate /volume/pt-train/miniconda3/envs/cheliu_vllm_seed_oss;bash /vllm-workspace/examples/online_serving/multi-node-serving.sh worker --ray_address=$LWS_LEADER_ADDRESS",
    resourceConfig=LLMResourceConfig(
        instanceName="sci.g21-3",
        instanceQuantity=8
    )
)

storage_config = LLMStorageConfig(
    fileSystemVolumes=[
        LLMVolume(
            volumeId=9,
            name="pt-train",
            mountPath="/volume/pt-train",
        )
    ]
)

service_params = ServiceCreateParams(
    name="qwen3-32b-service",
    description="Qwen3 32B inference service",
    tenant="simaas",
    owner="smadmin",
    resourcePool="pt-eval",
    serviceConfig=service_config,
    servingEngineConfig=serving_engine_config,
    roleConfig={"worker": worker_role_config, "leader": leader_role_config},
    storageConfig=storage_config,
        env={
        "CONDA_ENV_NAME": "/volume/pt-train/users/wzhang/lyw/conda/envs/vllm",
        "MAX_MODEL_LEN": "32768",
        "TENSOR_PARALLEL_SIZE": "4",
        "DATA_PARALLEL_SIZE": "2",
        "PIPELINE_PARALLEL_SIZE": "1",
        "GPU_MEMORY_UTILIZATION": "0.9",
        "REASONING_PARSER": "",  # e.g., deepseek_r1, qwen2_r1
        "TOOL_CALL_PARSER": "",     # e.g., hermes, qwen2_v1.5
    }
)

service_id = inference.create_service(service_params=service_params)
#service_id=347
print(f"Service created with ID: {service_id}")

services = inference.list_services()
for service in services:
    print(f"ID: {service.id}, Name: {service.name}, Status: {service.status.status}")

service_details = inference.get_service(service_id=service_id)
print(f"Service: {service_details.name}")
print(f"Status: {service_details.status.status}")
print(f"Created: {service_details.created_at}")
print(f"urlInCluster: {service_details.status.url_in_cluster}")
print(f"urlExternal: {service_details.status.url_external}")

inference.offline_service(service_id=service_id)
inference.delete_service(service_id=service_id)
# inference.online_service(service_id=service_id)

开始使用

开发

训练

镜像

部署与服务

SDK

LLM 推理管理

客户端初始化

创建服务

查询服务

下线/上线服务

删除服务

示例

vLLM 单节点

自定义引擎单节点

自定义引擎多节点

​客户端初始化

​创建服务

​查询服务

​下线/上线服务

​删除服务

​示例

​vLLM 单节点

​自定义引擎单节点

​自定义引擎多节点

客户端初始化

创建服务

查询服务

下线/上线服务

删除服务

示例

vLLM 单节点

自定义引擎单节点

自定义引擎多节点