Examples

curl http://localhost:30000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "Qwen/Qwen3-0.6B",
    "messages": [{"role": "user", "content": "What is the capital of France? /no_think"}],
    "max_tokens": 128
  }'

HF_CACHE_DIR="/path/to/huggingface-cache"
docker run --rm --gpus all \
  --shm-size 32g \
  -p 30000:30000 \
  -v "$HF_CACHE_DIR:/root/.cache/huggingface" \
  --ipc=host \
  lmsysorg/sglang:latest \
  sglang serve \
    --trust-remote-code \
    --model-path Qwen/Qwen3.8-27B-FP8 \
    --kv-cache-dtype fp8_e4m3 \
    --mem-fraction-static 0.85 \
    --attention-backend flashinfer \
    --chunked-prefill-size 32768 \
    --max-prefill-tokens 32768 \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen3_coder \
    --mamba-full-memory-ratio 4.59 \
    --host 0.0.0.0 \
    --port 30000 \
    --mamba-radix-cache-strategy extra_buffer \
    --mamba-ssm-dtype float32

uv pip install openai

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:30000/v1",
    api_key="EMPTY",
)
model = "Qwen/Qwen3-0.6B"

stream = client.chat.completions.create(
    model=model,
    messages=[{"role": "user", "content": "Explain prefix caching in two sentences."}],
    stream=True,
)

for chunk in stream:
    if chunk.choices:
        print(chunk.choices[0].delta.content or "", end="", flush=True)
print()

from concurrent.futures import ThreadPoolExecutor

feedback = [
    "Setup was quick, but I could not find the export button.",
    "The new search is faster and finds the documents I need.",
    "Please add a way to share saved reports with my team.",
]

def summarize(text):
    response = client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": f"Summarize in one sentence: {text}"}],
    )
    return response.choices[0].message.content

with ThreadPoolExecutor(max_workers=3) as pool:
    for summary in pool.map(summarize, feedback):
        print(summary)

import json
from openai import OpenAI

client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")
model = "Qwen/Qwen3.8-27B-FP8"

orders = {
    "SGL-1042": {"status": "shipped", "estimated_delivery": "Friday"},
}

def lookup_order(order_id):
    return orders.get(order_id, {"error": "Order not found"})

tools = [{
    "type": "function",
    "function": {
        "name": "lookup_order",
        "description": "Look up an order's shipping status and estimated delivery.",
        "parameters": {
            "type": "object",
            "properties": {"order_id": {"type": "string"}},
            "required": ["order_id"],
            "additionalProperties": False,
        },
    },
}]

messages = [
    {"role": "system", "content": "Use lookup_order for order status. Do not invent order details."},
    {"role": "user", "content": "Where is order SGL-1042, and when should it arrive?"},
]

for _ in range(3):
    response = client.chat.completions.create(
        model=model, messages=messages, tools=tools, tool_choice="auto",
    )
    message = response.choices[0].message
    messages.append(message.model_dump(exclude_none=True))
    if not message.tool_calls:
        print(message.content)
        break

    for call in message.tool_calls:
        try:
            if call.function.name != "lookup_order":
                raise ValueError("Unknown tool")
            arguments = json.loads(call.function.arguments)
            result = lookup_order(**arguments)
        except (ValueError, TypeError) as error:
            result = {"error": str(error)}
        print(f"{call.function.name}: {result}")
        messages.append({
            "role": "tool",
            "tool_call_id": call.id,
            "content": json.dumps(result),
        })
else:
    print("Stopped after three model turns without a final answer.")