> ## Documentation Index > Fetch the complete documentation index at: https://docs.sglang.io/llms.txt > Use this file to discover all available pages before exploring further. # Clef > Deploy Cloudflare's Clef 27B and Clef Flash 9B decision models with SGLang. Select either variant for BF16 text and image decisions on one NVIDIA B200, B300, or H200. export const benchmarks = [{ match: { hw: "h200", variant: "clef", quant: "bf16", strategy: "balanced", nodes: "single" }, accuracy: { gsm8k_decision_pct: 80.71 }, notes: "1,319 problems, each with 4 and 10 choices (2,638 requests)." }, { match: { hw: "h200", variant: "flash", quant: "bf16", strategy: "balanced", nodes: "single" }, accuracy: { gsm8k_decision_pct: 67.32 }, notes: "1,319 problems, each with 4 and 10 choices (2,638 requests)." }, { match: { hw: "b200", variant: "clef", quant: "bf16", strategy: "balanced", nodes: "single" }, accuracy: { gsm8k_decision_pct: 80.59 }, notes: "1,319 problems, each with 4 and 10 choices (2,638 requests)." }, { match: { hw: "b200", variant: "flash", quant: "bf16", strategy: "balanced", nodes: "single" }, accuracy: { gsm8k_decision_pct: 67.36 }, notes: "1,319 problems, each with 4 and 10 choices (2,638 requests)." }, { match: { hw: "b300", variant: "clef", quant: "bf16", strategy: "balanced", nodes: "single" }, accuracy: { gsm8k_decision_pct: 80.59 }, notes: "1,319 problems, each with 4 and 10 choices (2,638 requests)." }, { match: { hw: "b300", variant: "flash", quant: "bf16", strategy: "balanced", nodes: "single" }, accuracy: { gsm8k_decision_pct: 67.36 }, notes: "1,319 problems, each with 4 and 10 choices (2,638 requests)." }]; export const config = { modelName: "Clef", showPlaygroundLink: false, supportedHardware: ["h200", "b200", "b300"], variants: [{ id: "clef", label: "Clef", subtitle: "27B dense, 1 GPU" }, { id: "flash", label: "Clef Flash", subtitle: "9B dense, 1 GPU" }], quantizations: [{ id: "bf16", label: "BF16" }], strategies: [{ id: "balanced", label: "Balanced" }], nodesOptions: [{ id: "single", label: "Single Node" }], modelNames: { "clef|bf16": "Cloudflare/clef", "flash|bf16": "Cloudflare/clef-flash" }, placeholders: { HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, PORT: { target: "command", label: "Bind port", default: "30000" }, CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, CURL_PORT: { target: "curl", label: "Server port", default: "30000" } }, curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/systemone \\ -H 'Content-Type: application/json' \\ -d '{"model":"{{MODEL_NAME}}","state":"My Stripe integration keeps failing. Please help ASAP.","questions":{"urgency":{"type":"noul","instructions":"Does this message express urgency?"}}}'`, accuracyLabels: [["gsm8k_decision_pct", "GSM8K (Decision Index 0.2.1)", "%"]], dockerImages: { h200: "lmsysorg/sglang:dev-clef", b200: "lmsysorg/sglang:dev-clef", b300: "lmsysorg/sglang:dev-clef" }, github: { cookbookModel: "Cloudflare/clef" }, cells: [{ match: { hw: "h200", variant: "clef", quant: "bf16", strategy: "balanced", nodes: "single" }, verified: true, env: [], flags: ["--model-path {{MODEL_NAME}}", "--host {{HOST_IP}}", "--port {{PORT}}"] }, { match: { hw: "h200", variant: "flash", quant: "bf16", strategy: "balanced", nodes: "single" }, verified: true, env: [], flags: ["--model-path {{MODEL_NAME}}", "--host {{HOST_IP}}", "--port {{PORT}}"] }, { match: { hw: "b200", variant: "clef", quant: "bf16", strategy: "balanced", nodes: "single" }, verified: true, env: [], flags: ["--model-path {{MODEL_NAME}}", "--host {{HOST_IP}}", "--port {{PORT}}"] }, { match: { hw: "b200", variant: "flash", quant: "bf16", strategy: "balanced", nodes: "single" }, verified: true, env: [], flags: ["--model-path {{MODEL_NAME}}", "--host {{HOST_IP}}", "--port {{PORT}}"] }, { match: { hw: "b300", variant: "clef", quant: "bf16", strategy: "balanced", nodes: "single" }, verified: true, env: [], flags: ["--model-path {{MODEL_NAME}}", "--host {{HOST_IP}}", "--port {{PORT}}"] }, { match: { hw: "b300", variant: "flash", quant: "bf16", strategy: "balanced", nodes: "single" }, verified: true, env: [], flags: ["--model-path {{MODEL_NAME}}", "--host {{HOST_IP}}", "--port {{PORT}}"] }] }; export const Deployment = ({config, benchmarks, agenticLink}) => { if (!config) { return
Deployment: missing config prop
; } const AMD_RDMA_DOCKER_FLAGS = ["--device /dev/infiniband", "--cap-add IPC_LOCK", "--ulimit memlock=-1", "--ulimit stack=67108864", "--ulimit nofile=1048576:1048576"]; const HARDWARE_CATALOG = { blackwell: [{ id: "b300", label: "B300", vram: "288GB" }, { id: "gb300", label: "GB300", vram: "288GB" }, { id: "b200", label: "B200", vram: "192GB" }, { id: "gb200", label: "GB200", vram: "192GB" }, { id: "dgx-spark", label: "DGX Spark", vram: "128GB", multiNodeDockerFlags: ["--ulimit memlock=-1:-1", "--cap-add IPC_LOCK", "--device /dev/infiniband"] }], hopper: [{ id: "h200", label: "H200", vram: "141GB" }, { id: "h100", label: "H100", vram: "80GB" }, { id: "h20-3e", label: "H20-3e", vram: "141GB" }, { id: "h800", label: "H800", vram: "80GB" }], amd: [{ id: "mi300x", label: "MI300X", vram: "192GB", multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] }, { id: "mi325x", label: "MI325X", vram: "256GB", multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] }, { id: "mi350x", label: "MI350X", vram: "288GB", multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] }, { id: "mi355x", label: "MI355X", vram: "288GB", multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] }], npu: [{ id: "a3", label: "A3 Series", vram: "64GB/die", npuDevices: 16 }, { id: "a5", label: "950PR/DT Series", vram: "128GB", npuDevices: 8 }] }; const makeStyles = isDark => ({ container: { maxWidth: "900px", margin: "0 auto", display: "flex", flexDirection: "column", gap: "3px" }, card: { padding: "5px 10px", border: `1px solid ${isDark ? "#374151" : "#e5e7eb"}`, borderLeft: `3px solid ${isDark ? "#E85D4D" : "#D45D44"}`, borderRadius: "4px", display: "flex", alignItems: "center", gap: "10px", background: isDark ? "#1f2937" : "#fff" }, cardColumn: { padding: "5px 10px", border: `1px solid ${isDark ? "#374151" : "#e5e7eb"}`, borderLeft: `3px solid ${isDark ? "#E85D4D" : "#D45D44"}`, borderRadius: "4px", display: "flex", flexDirection: "column", gap: "4px", background: isDark ? "#1f2937" : "#fff" }, title: { fontSize: "12px", fontWeight: "600", minWidth: "108px", flexShrink: 0, color: isDark ? "#e5e7eb" : "inherit" }, vendorRow: { display: "flex", alignItems: "center", gap: "6px" }, vendorLabel: { fontSize: "10px", fontWeight: "600", color: isDark ? "#9ca3af" : "#6b7280", width: "68px", flexShrink: 0, textTransform: "uppercase", letterSpacing: "0.04em" }, itemsGrid: () => ({ display: "grid", gridTemplateColumns: "repeat(auto-fit, minmax(72px, 1fr))", gap: "4px", flex: 1 }), labelBase: { padding: "2px 8px", border: `1px solid ${isDark ? "#9ca3af" : "#d1d5db"}`, borderRadius: "3px", cursor: "pointer", display: "inline-flex", flexDirection: "column", alignItems: "center", justifyContent: "center", fontWeight: "500", fontSize: "12px", transition: "all 0.2s", userSelect: "none", minHeight: "26px", textAlign: "center", background: isDark ? "#374151" : "#fff", color: isDark ? "#e5e7eb" : "inherit" }, checked: { background: "#D45D44", color: "white", borderColor: "#D45D44" }, disabled: { cursor: "not-allowed", opacity: 0.4 }, subtitle: { display: "block", fontSize: "9px", marginTop: "1px", lineHeight: "1.1", opacity: 0.7 }, commandWrap: { position: "relative", flex: 1, background: isDark ? "#111827" : "#f5f5f5", borderRadius: "6px", border: `1px solid ${isDark ? "#374151" : "#e5e7eb"}`, overflow: "hidden" }, commandHeader: { display: "flex", flexWrap: "wrap", justifyContent: "space-between", alignItems: "center", gap: "6px 10px", padding: "6px 10px", borderBottom: `1px solid ${isDark ? "#374151" : "#e5e7eb"}`, background: isDark ? "#1f2937" : "#fafafa" }, commandPre: { padding: "12px 16px", fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: "12px", lineHeight: "1.5", color: isDark ? "#e5e7eb" : "#374151", whiteSpace: "pre-wrap", overflowX: "auto", margin: 0 }, mtpWarn: { margin: "8px 0 0", padding: "8px 12px", borderRadius: "8px", fontSize: "12px", lineHeight: "1.45", background: isDark ? "#78350f" : "#fef3c7", color: isDark ? "#fde68a" : "#92400e", border: `1px solid ${isDark ? "#92400e" : "#fcd34d"}` }, badge: status => ({ display: "inline-flex", alignItems: "center", gap: "6px", padding: "2px 8px", borderRadius: "10px", background: ({ verified: isDark ? "#064e3b" : "#d1fae5", "in-progress": isDark ? "#1e3a8a" : "#dbeafe", unverified: isDark ? "#78350f" : "#fef3c7" })[verifyStatusOf(status)], color: ({ verified: isDark ? "#a7f3d0" : "#065f46", "in-progress": isDark ? "#bfdbfe" : "#1e40af", unverified: isDark ? "#fde68a" : "#92400e" })[verifyStatusOf(status)], fontSize: "11px", fontWeight: 600, whiteSpace: "nowrap" }), badgeDot: status => ({ width: "8px", height: "8px", borderRadius: "50%", background: ({ verified: "#10b981", "in-progress": "#3b82f6", unverified: "#f59e0b" })[verifyStatusOf(status)] }), iconButton: { padding: "4px 10px", border: `1px solid ${isDark ? "#4b5563" : "#d1d5db"}`, borderRadius: "4px", background: isDark ? "#1f2937" : "#fff", color: isDark ? "#e5e7eb" : "#374151", fontSize: "11px", fontWeight: 500, cursor: "pointer", display: "inline-flex", alignItems: "center", gap: "4px" }, iconRow: { display: "inline-flex", flexWrap: "wrap", gap: "6px" }, runModeWrap: { display: "inline-flex", border: `1px solid ${isDark ? "#4b5563" : "#d1d5db"}`, borderRadius: "10px", overflow: "hidden", fontSize: "11px", fontWeight: 600, userSelect: "none" }, runModeChip: active => ({ padding: "2px 10px", cursor: "pointer", background: active ? isDark ? "#1f2937" : "#fff" : "transparent", color: active ? isDark ? "#e5e7eb" : "#111827" : isDark ? "#9ca3af" : "#6b7280", borderRight: `1px solid ${isDark ? "#4b5563" : "#d1d5db"}` }), runModeChipLast: active => ({ padding: "2px 10px", cursor: "pointer", background: active ? isDark ? "#1f2937" : "#fff" : "transparent", color: active ? isDark ? "#e5e7eb" : "#111827" : isDark ? "#9ca3af" : "#6b7280" }), headerLeft: { display: "inline-flex", flexWrap: "wrap", alignItems: "center", gap: "8px" }, modalBackdrop: { position: "fixed", inset: 0, background: "rgba(0,0,0,0.5)", display: "flex", alignItems: "center", justifyContent: "center", zIndex: 9999 }, modalBox: { background: isDark ? "#1f2937" : "#fff", color: isDark ? "#e5e7eb" : "#111827", borderRadius: "8px", padding: "20px", maxWidth: "720px", width: "92%", maxHeight: "85vh", overflowY: "auto", border: `1px solid ${isDark ? "#374151" : "#e5e7eb"}`, boxShadow: "0 10px 25px rgba(0,0,0,0.25)" }, modalHeader: { display: "flex", justifyContent: "space-between", alignItems: "center", marginBottom: "12px" }, modalTitle: { fontSize: "15px", fontWeight: 600 }, modalCloseBtn: { background: "transparent", border: "none", color: "inherit", fontSize: "20px", cursor: "pointer", padding: "0 6px", lineHeight: 1 }, formField: { display: "flex", flexDirection: "column", gap: "4px", marginBottom: "10px" }, formLabel: { fontSize: "12px", fontWeight: 500, color: isDark ? "#9ca3af" : "#4b5563" }, formInput: { padding: "6px 10px", fontSize: "13px", border: `1px solid ${isDark ? "#4b5563" : "#d1d5db"}`, borderRadius: "4px", background: isDark ? "#111827" : "#fff", color: isDark ? "#e5e7eb" : "#111827", fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace" }, sectionHeading: { fontSize: "12px", fontWeight: 600, textTransform: "uppercase", letterSpacing: "0.04em", color: isDark ? "#9ca3af" : "#6b7280", margin: "12px 0 6px 0" }, primaryBtn: { padding: "6px 14px", background: "#D45D44", color: "white", border: "none", borderRadius: "4px", cursor: "pointer", fontSize: "13px", fontWeight: 500 }, benchCard: { padding: "8px 12px", border: `1px solid ${isDark ? "#374151" : "#e5e7eb"}`, borderLeft: `3px solid ${isDark ? "#E85D4D" : "#D45D44"}`, borderRadius: "4px", background: isDark ? "#1f2937" : "#fff", display: "flex", flexDirection: "column", gap: "8px" }, benchHeader: { display: "flex", flexWrap: "wrap", alignItems: "baseline", justifyContent: "space-between", gap: "6px 12px" }, benchTitle: { fontSize: "13px", fontWeight: 600, color: isDark ? "#e5e7eb" : "inherit" }, benchVersion: { fontSize: "11px", color: isDark ? "#9ca3af" : "#6b7280" }, benchHeaderRight: { display: "flex", flexWrap: "wrap", alignItems: "center", gap: "6px 10px", flexShrink: 0 }, benchChipRow: { display: "flex", alignItems: "center", gap: "6px", flexWrap: "wrap", margin: "2px 0 8px" }, benchChip: { padding: "2px 10px", fontSize: "12px", cursor: "pointer", border: `1px solid ${isDark ? "#4b5563" : "#d1d5db"}`, borderRadius: "4px", background: isDark ? "#1f2937" : "#fff", color: isDark ? "#e5e7eb" : "#374151", fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace" }, benchChipActive: { background: "#D45D44", color: "white", borderColor: "#D45D44" }, benchBlock: { border: `1px solid ${isDark ? "#374151" : "#e5e7eb"}`, borderRadius: "4px", padding: "8px 10px", background: isDark ? "#111827" : "#fafafa" }, benchBlockTitle: { fontSize: "11px", fontWeight: 600, textTransform: "uppercase", letterSpacing: "0.04em", color: isDark ? "#9ca3af" : "#6b7280", marginBottom: "4px" }, benchWorkload: { fontSize: "11px", fontStyle: "italic", color: isDark ? "#9ca3af" : "#6b7280", marginBottom: "6px", lineHeight: "1.3" }, benchRow: { display: "flex", justifyContent: "space-between", fontSize: "12px", padding: "2px 0" }, benchKey: { color: isDark ? "#9ca3af" : "#6b7280" }, benchVal: { color: isDark ? "#e5e7eb" : "#111827", fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontWeight: 500 }, benchNotes: { fontSize: "11px", fontStyle: "italic", color: isDark ? "#9ca3af" : "#6b7280" }, benchLegend: { fontSize: "10px", fontStyle: "italic", color: isDark ? "#6b7280" : "#9ca3af", marginTop: "6px", fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace" }, benchEmpty: { fontSize: "12px", fontStyle: "italic", color: isDark ? "#9ca3af" : "#6b7280" }, benchTableScroll: { overflowX: "auto", overflowY: "hidden", marginTop: "4px" }, benchTable: { display: "grid", columnGap: 0, rowGap: "3px", alignItems: "baseline", minWidth: "max-content" }, benchTableHead: { textAlign: "right", fontWeight: 500, fontSize: "11px", color: isDark ? "#9ca3af" : "#6b7280", paddingLeft: "16px", paddingBottom: "4px", whiteSpace: "nowrap" }, benchTableCornerHead: { paddingBottom: "4px", position: "sticky", left: 0, background: isDark ? "#111827" : "#fafafa", zIndex: 1, alignSelf: "stretch" }, benchTableSeparator: { gridColumn: "1 / -1", height: "1px", background: isDark ? "#374151" : "#e5e7eb", marginTop: "-3px" }, benchTableLabel: { textAlign: "left", fontSize: "12px", color: isDark ? "#9ca3af" : "#6b7280", whiteSpace: "nowrap", position: "sticky", left: 0, background: isDark ? "#111827" : "#fafafa", zIndex: 1 }, benchTableValue: { textAlign: "right", fontSize: "12px", color: isDark ? "#e5e7eb" : "#111827", fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontWeight: 500, paddingLeft: "16px", whiteSpace: "nowrap" }, benchTableValueMissing: { color: isDark ? "#6b7280" : "#9ca3af" } }); const VERIFY_LABEL = { verified: "Verified", "in-progress": "Final Verification In Progress", unverified: "Not Verified" }; const verifyStatusOf = v => typeof v === "string" ? VERIFY_LABEL[v] ? v : "unverified" : v ? "verified" : "unverified"; const cellVerifyStatus = (c, sel) => { if (!c) return "unverified"; const v = typeof c.verificationStatus === "function" ? c.verificationStatus(sel) : c.verificationStatus; return verifyStatusOf(v ?? c.verified); }; const LEGACY_MATCH_DIMS = [{ id: "variant", title: "Model Variant", optionsKey: "variants" }, { id: "quant", title: "Quantization", optionsKey: "quantizations" }, { id: "strategy", title: "Strategy", optionsKey: "strategies" }, { id: "nodes", title: "Nodes", optionsKey: "nodesOptions" }]; const matchDimSpecs = (config.matchDims || LEGACY_MATCH_DIMS).map(d => ({ ...d, options: d.options || config[d.optionsKey] || [] })); const overlayDimSpecs = config.overlayDims || []; const commandBuilder = config.commandBuilder || null; const DIMENSIONS = ["hw", ...matchDimSpecs.map(d => d.id)]; const optionVisible = (opt, sel) => typeof opt.showWhen !== "function" || opt.showWhen(sel); const optionDisabled = (opt, sel) => typeof opt.disabled === "function" ? opt.disabled(sel) : !!opt.disabled; const visibleOptions = (spec, sel) => (spec.options || []).filter(o => optionVisible(o, sel)); const rowVisible = (spec, sel) => (typeof spec.showWhen !== "function" || spec.showWhen(sel)) && visibleOptions(spec, sel).length > 0; const overlayPick = sel => { const picked = []; for (const spec of config.overlayDims || []) { if (!rowVisible(spec, sel)) continue; const opt = (spec.options || []).find(o => o.id === sel[spec.id]); if (opt && !optionDisabled(opt, sel)) picked.push(opt); } return picked; }; const overlayPart = (sel, key) => { const out = []; for (const opt of overlayPick(sel)) { const add = typeof opt[key] === "function" ? opt[key](sel) : opt[key]; if (add) out.push(...add); } return out; }; const overlayCompose = (cellFlags, sel) => { const strip = overlayPart(sel, "stripPrefixes"); const add = overlayPart(sel, "flags"); if (!strip.length) return [...cellFlags || [], ...add]; const used = new Set(); const replacementsFor = tok => { const out = []; add.forEach((f, i) => { if (used.has(i) || f.split(/[\s=]/)[0] !== tok) return; used.add(i); out.push(f); }); return out; }; const out = []; for (const f of cellFlags || []) { const tok = f.split(/[\s=]/)[0]; if (!strip.includes(tok)) out.push(f); else out.push(...replacementsFor(tok)); } add.forEach((f, i) => { if (!used.has(i)) out.push(f); }); return out; }; const optionSoft = (opt, sel) => typeof opt.soft === "function" ? opt.soft(sel) : !!opt.soft; const findCell = (cells, sel) => { const cell = cells.find(c => DIMENSIONS.every(d => c.match[d] === sel[d])); return cell && typeof config.resolveRecipe === "function" ? config.resolveRecipe(cell, sel) : cell; }; const findBenchmark = (list, sel) => { const hits = (list || []).filter(b => Object.entries(b.match || ({})).every(([k, v]) => sel[k] === v)); return hits.sort((a, b) => Object.keys(b.match).length - Object.keys(a.match).length)[0] || null; }; const normalizeSpeed = speed => { if (!speed) return []; return Array.isArray(speed) ? speed : [speed]; }; const effectiveAccuracy = (entry, sel) => entry ? { ...config.defaultAccuracy && config.defaultAccuracy[sel.variant] || ({}), ...entry.accuracy || ({}) } : {}; const benchmarkIsEmpty = (entry, accuracy) => { for (const m of normalizeSpeed(entry && entry.speed)) { if (m && typeof m === "object") { for (const [key, v] of Object.entries(m)) { if (key === "workload") continue; if (v !== null && v !== undefined) return false; } } } if (accuracy && typeof accuracy === "object") { for (const v of Object.values(accuracy)) { if (v !== null && v !== undefined) return false; } } return true; }; const isOptionAvailable = (cells, sel, dim, value) => { const idx = DIMENSIONS.indexOf(dim); const higher = DIMENSIONS.slice(0, idx); return cells.some(c => c.match[dim] === value && higher.every(d => c.match[d] === sel[d])); }; const snapToValidCell = (cells, sel, dim, value) => { const idx = DIMENSIONS.indexOf(dim); const higher = DIMENSIONS.slice(0, idx); const lower = DIMENSIONS.slice(idx + 1); let best = null, bestLowerMatches = -1; for (const c of cells) { if (c.match[dim] !== value) continue; if (!higher.every(d => c.match[d] === sel[d])) continue; let s = 0; for (const d of lower) if (c.match[d] === sel[d]) s++; if (s > bestLowerMatches) { bestLowerMatches = s; best = c; } } if (!best) return sel; const next = { ...sel, [dim]: value }; for (const d of lower) next[d] = best.match[d]; return next; }; const validateSelection = (cells, parsed) => { const valid = {}; for (const dim of DIMENSIONS) { const want = parsed[dim]; const works = cells.some(c => c.match[dim] === want && DIMENSIONS.slice(0, DIMENSIONS.indexOf(dim)).every(d => c.match[d] === valid[d])); if (works) { valid[dim] = want; } else { const fallback = cells.find(c => DIMENSIONS.slice(0, DIMENSIONS.indexOf(dim)).every(d => c.match[d] === valid[d])); valid[dim] = fallback ? fallback.match[dim] : want; } } for (const spec of overlayDimSpecs) { const want = parsed[spec.id]; const opts = spec.options || []; const picked = opts.some(o => o.id === want) ? want : (spec.default ?? (opts[0] && opts[0].id)) ?? ""; const withPick = { ...valid, [spec.id]: picked }; const usable = visibleOptions(spec, withPick).filter(o => !optionDisabled(o, withPick)); valid[spec.id] = usable.some(o => o.id === picked) ? picked : (usable[0] && usable[0].id) ?? picked; } return valid; }; const resolveModelName = sel => { const keys = [`${sel.hw}|${sel.variant}|${sel.quant}`, `${sel.variant}|${sel.quant}`, `${sel.hw}|${sel.quant}`, sel.quant, sel.hw, "default"]; for (const k of keys) { const hit = config.modelNames[k]; if (hit) return hit; } return ""; }; const interpolate = (text, env, modelName) => text.replace(/{{(\w+)}}/g, (_, key) => key === "MODEL_NAME" ? modelName : env[key] ?? `{{${key}}}`); const parseNnodes = id => { if (Number.isInteger(id)) return id; if ((/^\d+$/).test(id || "")) return parseInt(id, 10); if (id === "single") return 1; const m = (/^multi-(\d+)$/).exec(id || ""); return m ? parseInt(m[1], 10) : 1; }; const cellNnodes = (cell, sel) => cell.pd ? cell.nnodes || 1 : sel.nodes !== undefined ? parseNnodes(sel.nodes) : cell.nnodes || 1; const PD_SERVE_PORTS = { prefill: 30000, decode: 30100 }; const overlayEnv = sel => overlayPart(sel, "env"); const overlayHints = sel => overlayPart(sel, "hints"); const renderCommand = (cell, sel, envValues, mode = "python") => { if (!cell) return "# No command available for the current selection."; const modelName = resolveModelName(sel); if (cell.pd && cell.commands) return interpolate(cell.commands[mode] || cell.commands.python, envValues, modelName); const nnodes = cellNnodes(cell, sel); const multinode = nnodes > 1; const cellEnv = [...cell.env || [], ...overlayEnv(sel)]; const flags = overlayCompose(cell.flags, sel); if (multinode) { const PARALLELISM_ANCHORS = new Set(["--attn-dp-size", "--attention-data-parallel-size", "--dp-size", "--dp", "--tp-size", "--tp", "--sp-degree", "--ulysses-degree", "--ring-degree"]); let i = flags.reduce((last, flag, index) => PARALLELISM_ANCHORS.has(flag.split(/[\s=]/)[0]) ? index : last, -1); if (i === -1) i = flags.findIndex(f => f.startsWith("--model-path")); flags.splice(i + 1, 0, `--nnodes ${nnodes}`, `--node-rank {{NODE_RANK}}`, `--dist-init-addr {{NODE0_IP}}:20000`); } const pdServePort = PD_SERVE_PORTS[cell.pd ? cell.pdMode : config.resolveRecipe ? null : sel.pdMode]; if (pdServePort !== undefined) { for (let j = 0; j < flags.length; j++) { if (flags[j].split(/[\s=]/)[0] === "--port") { flags[j] = `--port ${pdServePort}`; } } } let cmd; if (mode === "docker") { const di = config.dockerImages || ({}); const image = cell.pd && cell.dockerImage || di[`${sel.hw}|${sel.variant}|${sel.quant}`] || di[`${sel.variant}|${sel.quant}`] || di[`${sel.hw}|${sel.quant}|${sel.strategy}`] || di[`${sel.hw}|${sel.quant}`] || di[sel.hw] || "lmsysorg/sglang:dev"; const dockerRunCommand = typeof config.dockerRunCommand === "function" ? config.dockerRunCommand(sel) : config.dockerRunCommand || "sglang serve"; const portFlag = flags.find(x => x.split(/[\s=]/)[0] === "--port"); const servePort = portFlag ? portFlag.slice(("--port").length).trim() : "{{PORT}}"; const hostNetwork = multinode || cell.pd || typeof config.dockerHostNetworkWhen === "function" && config.dockerHostNetworkWhen(sel, { flags, env: cellEnv }); const vendorOf = hwId => { for (const [vendor, list] of Object.entries(HARDWARE_CATALOG)) { if (list.some(h => h.id === hwId)) return vendor; } const extra = (config.hardware || []).find(h => h.id === hwId); return extra && extra.vendor || "nvidia"; }; const catalogEntryOf = hwId => { const extra = (config.hardware || []).find(h => h.id === hwId); if (extra) return extra; for (const list of Object.values(HARDWARE_CATALOG)) { const hit = list.find(h => h.id === hwId); if (hit) return hit; } return null; }; const fabricFlagsOf = hwId => (catalogEntryOf(hwId) || ({})).multiNodeDockerFlags || []; const davinciLines = devices => { const lines = []; for (let i = 0; i < devices; i += 4) { const group = []; for (let k = i; k < Math.min(i + 4, devices); k++) { group.push(`--device=/dev/davinci${k}`); } lines.push(" " + group.join(" ")); } return lines; }; const gpuAccessLines = vendorOf(sel.hw) === "amd" ? ["docker run", " --device=/dev/kfd --device=/dev/dri", " --group-add video", " --cap-add=SYS_PTRACE --security-opt seccomp=unconfined", " --shm-size 32g"] : vendorOf(sel.hw) === "npu" ? ["docker run --privileged --shm-size=16g", ...davinciLines((catalogEntryOf(sel.hw) || ({})).npuDevices || 16), " --device=/dev/davinci_manager", " --device=/dev/hisi_hdc", " -v /usr/local/sbin:/usr/local/sbin", " -v /usr/local/Ascend/driver:/usr/local/Ascend/driver", " -v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware", " -v /etc/ascend_install.info:/etc/ascend_install.info", " -v /var/queue_schedule:/var/queue_schedule", " -v ~/.cache/:/root/.cache/"] : ["docker run --gpus all", " --shm-size 32g"]; const dockerLines = [...gpuAccessLines, hostNetwork ? " --network host" : ` -p ${servePort}:${servePort}`, ...multinode || cell.pd ? fabricFlagsOf(sel.hw).map(f => " " + f) : [], ...vendorOf(sel.hw) === "npu" ? [] : [" -v ~/.cache/huggingface:/root/.cache/huggingface"], ...[...config.dockerMounts || [], ...cell.pd ? cell.dockerMounts || [] : []].map(mount => ` -v ${mount}`), ...config.placeholders && config.placeholders.HF_TOKEN ? [` --env "HF_TOKEN={{HF_TOKEN}}"`] : [], ...cellEnv.map(e => ` --env ${e}`), " --ipc=host", ` ${image}`, ` ${dockerRunCommand}`, ...flags.map(f => " " + f)]; cmd = dockerLines.join(" \\\n"); } else { const flagBlock = flags.map(f => " " + f).join(" \\\n"); const envBlock = cellEnv.length ? cellEnv.join(" \\\n") + " \\\n" : ""; cmd = `${envBlock}sglang serve \\\n${flagBlock}`; } const hintLines = [...cell.pd ? cell.hints || [] : [], ...overlayHints(sel), ...multinode && config.multiNodeHints && config.multiNodeHints[sel.hw] ? config.multiNodeHints[sel.hw] : []]; if (hintLines.length) { const hint = hintLines.map(line => line.length ? "# " + line : "#").join("\n"); cmd = `${hint}\n${cmd}`; } cmd = interpolate(cmd, envValues, modelName); if (multinode) { const header = `# Multi-node (${nnodes} nodes). Run the same command on every node with:\n` + `# = 0 on the head node, 1..${nnodes - 1} on the others\n` + `# = IP of the head node (reachable from all others)`; cmd = `${header}\n${cmd}`; } return cmd; }; const ACCURACY_LABELS = config.accuracyLabels || []; const renderBenchmarkCard = entry => { const pct = entry && entry.latencyPercentile || config.latencyPercentile || "P50"; const SPEED_LABELS = [["ttft_ms", `TTFT (${pct})`, "ms"], ["tpot_ms", `TPOT (${pct})`, "ms"], ["tokens_per_sec_per_gpu", "throughput per gpu", "tok/s"], ["interactivity", "interactivity", "tokens/s/user", m => m.tpot_ms != null && m.tpot_ms !== 0 ? Math.round(1000 / m.tpot_ms * 10) / 10 : null]]; const WORKLOAD_KEYS = ["dataset", "isl", "osl", "max_concurrency"]; const fmt = (val, unit) => { if (val === null || val === undefined) return null; return `${val}${unit ? " " + unit : ""}`; }; const formatWorkloadParts = (workload, keys) => { if (!workload) return ""; const parts = []; if (keys.has("dataset") && workload.dataset) parts.push(workload.dataset); if (keys.has("isl") || keys.has("osl")) { if (workload.isl != null || workload.osl != null) { parts.push(`in/out=${workload.isl != null ? workload.isl : "?"}/${workload.osl != null ? workload.osl : "?"}`); } } if (keys.has("max_concurrency") && workload.max_concurrency != null) { parts.push(`max-concurrency=${workload.max_concurrency}`); } return parts.join(", "); }; const ALWAYS_PER_COLUMN = new Set(["max_concurrency"]); const ATOMIC_WORKLOAD_GROUPS = [["isl", "osl"]]; const partitionWorkload = measurements => { const shared = new Set(); const differing = new Set(); for (const k of WORKLOAD_KEYS) { const seen = new Set(); let anyPresent = false; for (const m of measurements) { const v = m && m.workload ? m.workload[k] : undefined; if (v != null) anyPresent = true; seen.add(v); } if (!anyPresent) continue; if (ALWAYS_PER_COLUMN.has(k) || seen.size > 1) differing.add(k); else shared.add(k); } for (const group of ATOMIC_WORKLOAD_GROUPS) { if (!group.some(k => differing.has(k))) continue; for (const k of group) if (shared.delete(k)) differing.add(k); } return { shared, differing }; }; const renderBenchTable = ({title, sharedText, colHeaders, rows, colCount, legend}) => { if (rows.length === 0) return null; const showColHeaders = colHeaders.length > 0 && colHeaders.some(h => h !== ""); return
{title}
{sharedText &&
{sharedText}
} {}
1 ? { tabIndex: 0, role: "group", "aria-label": sharedText ? `${title} — ${sharedText}` : title } : {}}>
{showColHeaders &&
} {showColHeaders && colHeaders.map((h, i) =>
{h}
)} {showColHeaders &&
} {rows.map(r => [
{r.label}
, ...r.values.map((v, i) =>
{v !== null ? v : "—"}
)])}
{legend &&
{(Array.isArray(legend) ? legend : [legend]).map((line, i) =>
{line}
)}
}
; }; const buildSpeedTable = measurements => { if (measurements.length === 0) return null; const {shared, differing} = partitionWorkload(measurements); const sharedText = formatWorkloadParts(measurements[0] && measurements[0].workload, shared); const colHeaders = measurements.map(m => formatWorkloadParts(m && m.workload, differing)); const rows = SPEED_LABELS.map(tup => { const [key, label, unit, compute] = tup; const values = measurements.map(m => { const raw = compute ? compute(m) : m[key]; return fmt(raw, unit); }); return { label, values }; }); return { title: "Speed", sharedText, colHeaders, rows, colCount: measurements.length, legend: [`throughput per gpu = (input+output tokens)/elapsed/GPU`, `interactivity = 1000/TPOT(ms) (tokens/s/user)`] }; }; const buildAccuracyTable = accuracy => { if (!accuracy) return null; const rows = ACCURACY_LABELS.map(([key, label, unit]) => { const v = fmt(accuracy[key], unit); if (v === null) return null; return { label, values: [v] }; }).filter(r => r !== null); if (rows.length === 0) return null; return { title: "Accuracy", sharedText: null, colHeaders: [], rows, colCount: 1 }; }; const accuracy = effectiveAccuracy(entry, sel); const isEmpty = benchmarkIsEmpty(entry, accuracy); const measurements = !isEmpty ? normalizeSpeed(entry && entry.speed) : []; const accuracyTable = !isEmpty ? buildAccuracyTable(accuracy) : null; const speedTable = !isEmpty ? buildSpeedTable(measurements) : null; const hasBenchCmds = !isEmpty && buildBenchCommands(entry, sel) !== null; return
Benchmark
{!isEmpty && entry && entry.sglang_version &&
measured on sglang {entry.sglang_version}
} {hasBenchCmds && }
{isEmpty ?
Benchmark data pending for this combination — submit yours via the Playground's Submit ↗ button.
: <> {accuracyTable && renderBenchTable(accuracyTable)} {speedTable && renderBenchTable(speedTable)} {entry && entry.notes &&
{entry.notes}
} }
; }; const buildBenchCommands = (entry, sel) => { const bc = config.benchmarkCommands; if (!bc) return null; const acc = effectiveAccuracy(entry, sel); const accuracy = []; if (bc.accuracy) { for (const [key, label] of ACCURACY_LABELS) { if (acc[key] == null) continue; const tmpl = bc.accuracy[key]; const resolved = typeof tmpl === "string" ? tmpl : tmpl && tmpl[sel.variant] || null; if (resolved) accuracy.push({ key, label, template: resolved }); } } let speed = null; if (bc.speed && entry) { const ms = normalizeSpeed(entry.speed).filter(m => m && m.workload && m.workload.max_concurrency != null); const concurrencies = [...new Set(ms.map(m => m.workload.max_concurrency))].sort((a, b) => a - b); if (concurrencies.length) { speed = { template: bc.speed, concurrencies, workload: ms[0].workload, numPromptsOf: c => { const m = ms.find(x => x.workload.max_concurrency === c); if (m && m.workload.num_prompts != null) return m.workload.num_prompts; const tbl = bc.numPromptsByConc; if (tbl && tbl[c] != null) return tbl[c]; return Math.max(c * 2, 200); } }; } } if (accuracy.length === 0 && !speed) return null; return { accuracy, speed }; }; const buildHardwareGroups = () => { const supported = new Set(config.supportedHardware); const catalog = {}; for (const [vendor, list] of Object.entries(HARDWARE_CATALOG)) catalog[vendor] = [...list]; for (const hw of config.hardware || []) { const vendor = hw.vendor || "nvidia"; const list = catalog[vendor] || (catalog[vendor] = []); const entry = { id: hw.id, label: hw.label, vram: hw.vram }; const i = list.findIndex(x => x.id === hw.id); if (i >= 0) list[i] = entry; else list.push(entry); } const groups = []; for (const [vendor, list] of Object.entries(catalog)) { const items = list.filter(hw => supported.has(hw.id)).map(hw => ({ id: hw.id, label: hw.label, subtitle: hw.vram })); if (items.length) groups.push({ label: vendor.toUpperCase(), items }); } if (config.groupHardware === false) { return [{ label: null, items: groups.flatMap(group => group.items) }]; } return groups; }; const initialSelectionFromCells = () => { const first = (config.cells || [])[0]; const sel = Object.fromEntries(DIMENSIONS.map(d => [d, first ? first.match[d] : ""])); for (const spec of overlayDimSpecs) { const opts = spec.options || []; sel[spec.id] = (spec.default ?? (opts[0] && opts[0].id)) ?? ""; } if (!commandBuilder) return sel; return { ...sel, hw: commandBuilder.defaultSelection?.hw || config.supportedHardware?.[0] || "", ...commandBuilder.defaultSelection || ({}) }; }; const normalizeBuilderSelection = parsed => { const out = { ...initialSelectionFromCells(), ...parsed }; if (!(config.supportedHardware || []).includes(out.hw)) { out.hw = commandBuilder.defaultSelection?.hw || config.supportedHardware?.[0] || ""; } for (const spec of overlayDimSpecs) { if (spec.kind === "number") { const value = Number.parseInt(out[spec.id], 10); out[spec.id] = Math.min(spec.max, Math.max(spec.min, Number.isFinite(value) ? value : Number(spec.default ?? spec.min))); continue; } const options = spec.options || []; if (!options.some(option => option.id === out[spec.id])) { out[spec.id] = (spec.default ?? options[0]?.id) ?? ""; } } for (const [key, bounds] of Object.entries(commandBuilder.resource?.limits || ({}))) { const fallback = Number((commandBuilder.defaultSelection?.[key] ?? bounds.min) ?? 1); const value = Number.parseInt(out[key], 10); out[key] = Math.min(bounds.max, Math.max(bounds.min, Number.isFinite(value) ? value : fallback)); } for (const key of ["tp_size", "ulysses_degree", "ring_degree"]) { const value = Number.parseInt(out[key], 10); out[key] = Number.isFinite(value) && value > 0 ? value : 1; } out.topology_mode = out.topology_mode === "manual" ? "manual" : "auto"; return out; }; const placeholderDefaults = schema => { const out = {}; for (const [k, v] of Object.entries(schema || ({}))) out[k] = v.default ?? ""; return out; }; const [isDark, setIsDark] = useState(false); useEffect(() => { const check = () => { const html = document.documentElement; setIsDark(html.classList.contains("dark") || html.getAttribute("data-theme") === "dark" || html.style.colorScheme === "dark"); }; check(); const observer = new MutationObserver(check); observer.observe(document.documentElement, { attributes: true, attributeFilter: ["class", "data-theme", "style"] }); return () => observer.disconnect(); }, []); const STORAGE_KEY = "sglang-deploy-env"; const [env, setEnv] = useState(() => placeholderDefaults(config.placeholders)); useEffect(() => { try { const raw = window.localStorage.getItem(STORAGE_KEY); if (raw) { const parsed = JSON.parse(raw); setEnv({ ...placeholderDefaults(config.placeholders), ...parsed }); } } catch {} }, []); const saveEnv = next => { setEnv(next); try { window.localStorage.setItem(STORAGE_KEY, JSON.stringify(next)); } catch {} }; const [sel, setSel] = useState(() => initialSelectionFromCells()); const [selectionHydrated, setSelectionHydrated] = useState(false); const INTERNAL_HASH_STATE_KEY = "__sglangDeployInternalHash"; const DEPLOYMENT_COMPONENT_ID = "deployment-configurator"; useEffect(() => { const hydrate = () => { const raw = window.location.hash.replace(/^#/, ""); if (!raw) return; const params = new URLSearchParams(raw); const initial = initialSelectionFromCells(); const parsed = { ...initial }; let touched = false; params.forEach((value, key) => { if ((key in parsed)) { parsed[key] = value; touched = true; } }); if (!touched) return; setSel(commandBuilder ? normalizeBuilderSelection(parsed) : validateSelection(config.cells, parsed)); const historyState = window.history.state; const isInternalHash = historyState && typeof historyState === "object" && historyState[INTERNAL_HASH_STATE_KEY] === `#${raw}`; if (isInternalHash) return; const el = document.getElementById(DEPLOYMENT_COMPONENT_ID); if (el) el.scrollIntoView({ behavior: "smooth", block: "start" }); }; hydrate(); setSelectionHydrated(true); window.addEventListener("hashchange", hydrate); return () => window.removeEventListener("hashchange", hydrate); }, []); useEffect(() => { if (!selectionHydrated) return; const target = "#" + new URLSearchParams(sel).toString(); if (window.location.hash !== target) { const historyState = window.history.state && typeof window.history.state === "object" ? window.history.state : {}; window.history.replaceState({ ...historyState, [INTERNAL_HASH_STATE_KEY]: target }, "", target); } window.dispatchEvent(new CustomEvent("sglang-deploy-sel", { detail: sel })); }, [sel, selectionHydrated]); const [modal, setModal] = useState(null); useEffect(() => { if (modal === null) return; const onKey = e => { if (e.key === "Escape") setModal(null); }; const prev = document.body.style.overflow; document.body.style.overflow = "hidden"; window.addEventListener("keydown", onKey); return () => { window.removeEventListener("keydown", onKey); document.body.style.overflow = prev; }; }, [modal]); const [copied, setCopied] = useState(false); const [curlCopied, setCurlCopied] = useState(false); const [envDraft, setEnvDraft] = useState(env); const [benchConc, setBenchConc] = useState(null); const [benchAcc, setBenchAcc] = useState(null); const [benchCopied, setBenchCopied] = useState(null); const configuredRunModes = typeof config.runModes === "function" ? config.runModes(sel) : config.runModes; const runModes = configuredRunModes || ["python", "docker"]; const [runMode, setRunMode] = useState(runModes[0]); const [builderScope, setBuilderScope] = useState("base"); const [builderServerSetting, setBuilderServerSetting] = useState(null); const [builderAdvanced, setBuilderAdvanced] = useState(false); const [serveExpanded, setServeExpanded] = useState(false); const [requestExpanded, setRequestExpanded] = useState(false); const [builderHeadAddress, setBuilderHeadAddress] = useState(""); const [builderNodeRank, setBuilderNodeRank] = useState(0); const [blockedNote, setBlockedNote] = useState(null); const flashBlockedNote = (dim, reason) => { const note = { dim, reason }; setBlockedNote(note); setTimeout(() => setBlockedNote(cur => cur === note ? null : cur), 4000); }; useEffect(() => { if (builderNodeRank >= Number(sel.nodes || 1)) setBuilderNodeRank(0); }, [sel.nodes, builderNodeRank]); const hasRunMode = runModes.includes(runMode); const fallbackRunMode = runModes[0]; const activeRunMode = hasRunMode ? runMode : fallbackRunMode; useEffect(() => { if (!hasRunMode) setRunMode(fallbackRunMode); }, [hasRunMode, fallbackRunMode]); useEffect(() => { if (modal === "env") setEnvDraft(env); }, [modal, env]); const [mambaRatio, setMambaRatio] = useState(null); useEffect(() => { const onRatio = e => setMambaRatio(e.detail && (e.detail.baseRatio || e.detail.ratio) || null); window.addEventListener("sglang-k3-mamba-ratio", onRatio); return () => window.removeEventListener("sglang-k3-mamba-ratio", onRatio); }, []); const s = makeStyles(isDark); const cell = commandBuilder ? commandBuilder.resolveDeployment(sel) : findCell(config.cells, sel); const builderMeta = cell && cell.builder || ({}); const verifyStatus = cellVerifyStatus(cell, sel); const cellWithRatio = (() => { if (!cell || !mambaRatio) return cell; if (cell.flags.some(f => f.startsWith("--mamba-full-memory-ratio") || f.startsWith("--max-mamba-cache-size"))) return cell; const flags = [...cell.flags]; const line = `--mamba-full-memory-ratio ${mambaRatio}`; const i = flags.findIndex(f => f.startsWith("--host")); if (i >= 0) flags.splice(i, 0, line); else flags.push(line); return { ...cell, flags }; })(); const commandEnv = commandBuilder ? { ...env, NODE_RANK: String(builderNodeRank), NODE0_IP: builderHeadAddress || "" } : env; const command = renderCommand(cellWithRatio, sel, commandEnv, activeRunMode); const effFlags = cell ? overlayCompose(cell.flags, sel) : []; const specAlgoFlag = effFlags.find(f => f.split(/[\s=]/)[0] === "--speculative-algorithm"); const specMrrFlag = effFlags.find(f => f.split(/[\s=]/)[0] === "--max-running-requests"); const mtpHint = !!specAlgoFlag && !specMrrFlag; const specPinnedHint = !!specAlgoFlag && !!specMrrFlag; const specMrrValue = specMrrFlag ? specMrrFlag.split(/[\s=]/).filter(Boolean)[1] || "" : ""; const SPEC_ALGO_LABEL = { EAGLE: "MTP", EAGLE3: "MTP", FROZEN_KV_MTP: "MTP", DSPARK: "DSpark", DFLASH: "DFlash", NGRAM: "N-gram", STANDALONE: "standalone draft" }; const specAlgoName = (() => { if (!specAlgoFlag) return "MTP"; const v = specAlgoFlag.split(/[\s=]/).filter(Boolean)[1] || ""; return SPEC_ALGO_LABEL[v.toUpperCase()] || v || "MTP"; })(); const renderWarn = text => { const out = []; const re = /\[([^\]]+)\]\(#([^)]+)\)/g; let last = 0; for (let m; m = re.exec(text); last = m.index + m[0].length) { if (m.index > last) out.push(text.slice(last, m.index)); const anchor = m[2]; out.push(); } if (last < text.length) out.push(text.slice(last)); return out; }; const modelName = resolveModelName(sel); const curlTemplate = typeof config.curl === "function" ? config.curl(sel, cell) : config.curl; const curlText = interpolate(curlTemplate || "", cell && cell.pd && cell.router ? { ...env, CURL_PORT: String(cell.router.port) } : env, modelName); const hwGroups = buildHardwareGroups(); const benchEntry = benchmarks ? findBenchmark(benchmarks, sel) : null; const isOverlayDim = dim => overlayDimSpecs.some(d => d.id === dim); const findOption = (dim, value) => { const spec = [...matchDimSpecs, ...overlayDimSpecs].find(d => d.id === dim); return spec && (spec.options || []).find(o => o.id === value); }; const isEnabled = (dim, value) => { const opt = findOption(dim, value); if (opt && optionDisabled(opt, sel)) return false; if (commandBuilder && dim === "hw") return true; return isOverlayDim(dim) || isOptionAvailable(config.cells || [], sel, dim, value); }; const reseatHiddenPicks = next => { let out = next; for (const spec of [...matchDimSpecs, ...overlayDimSpecs]) { const opts = visibleOptions(spec, out).filter(o => !optionDisabled(o, out)); if (!opts.length) continue; if (!opts.some(o => o.id === out[spec.id])) { out = { ...out, [spec.id]: opts[0].id }; } } return out; }; const recommendedBuilderRecipe = hw => { const recipes = commandBuilder.resource?.verifiedRecipes || []; return recipes.find(entry => entry.hw === hw && entry.default) || recipes.find(entry => entry.hw === hw); }; const handleSelect = (dim, value) => { if (commandBuilder) { setSel(prev => { let next = { ...prev, [dim]: value }; if (dim === "hw") { const currentRecipe = recommendedBuilderRecipe(prev.hw); const nextRecipe = recommendedBuilderRecipe(value); const resourcesFollowPlatformDefault = !!currentRecipe && Number(prev.nodes) === Number(currentRecipe.nodes) && Number(prev.gpus_per_node) === Number(currentRecipe.gpus_per_node); next = { ...next, nodes: resourcesFollowPlatformDefault ? nextRecipe?.nodes ?? next.nodes : next.nodes, gpus_per_node: resourcesFollowPlatformDefault ? nextRecipe?.gpus_per_node ?? next.gpus_per_node : next.gpus_per_node, topology_mode: "auto", tp_size: resourcesFollowPlatformDefault ? nextRecipe?.tp_size ?? 1 : next.tp_size, ulysses_degree: resourcesFollowPlatformDefault ? nextRecipe?.ulysses_degree ?? 1 : next.ulysses_degree, ring_degree: resourcesFollowPlatformDefault ? nextRecipe?.ring_degree ?? 1 : next.ring_degree, placement: resourcesFollowPlatformDefault ? nextRecipe?.placement || "auto" : next.placement, encoder: resourcesFollowPlatformDefault ? nextRecipe?.encoder || "auto" : next.encoder }; } return reseatHiddenPicks(normalizeBuilderSelection(next)); }); return; } setSel(prev => reseatHiddenPicks(isOverlayDim(dim) ? { ...prev, [dim]: value } : snapToValidCell(config.cells, prev, dim, value))); }; const commitBuilderNumber = (event, currentValue, bounds, commit) => { const parsed = Number(event.currentTarget.value); if (!Number.isInteger(parsed)) { event.currentTarget.value = String(currentValue); return; } const value = Math.min(bounds.max, Math.max(bounds.min, parsed)); event.currentTarget.value = String(value); commit(value); }; const renderBuilderNumberInput = ({identity, value, min, max, label, onCommit}) => event.currentTarget.select()} onBlur={event => commitBuilderNumber(event, value, { min, max }, onCommit)} onKeyDown={event => { if (event.key === "Enter") event.currentTarget.blur(); }} />; const updateBuilderResource = (key, delta) => { if (!commandBuilder) return; const bounds = commandBuilder.resource?.limits?.[key] || ({ min: 1, max: 8 }); setSel(prev => { const value = Math.min(bounds.max, Math.max(bounds.min, Number(prev[key]) + delta)); return normalizeBuilderSelection({ ...prev, [key]: value, topology_mode: "auto" }); }); }; const setBuilderResource = (key, rawValue) => { if (!commandBuilder) return; const value = Number.parseInt(rawValue, 10); if (!Number.isFinite(value)) return; const bounds = commandBuilder.resource?.limits?.[key] || ({ min: 1, max: 8 }); setSel(prev => normalizeBuilderSelection({ ...prev, [key]: Math.min(bounds.max, Math.max(bounds.min, value)), topology_mode: "auto" })); }; const editBuilderTopology = (key, value) => { if (!commandBuilder) return; setSel(prev => normalizeBuilderSelection({ ...prev, topology_mode: "manual", [key]: Number.parseInt(value, 10) || 1 })); }; const handleCopy = () => { navigator.clipboard.writeText(command); setCopied(true); setTimeout(() => setCopied(false), 1200); }; const copyCurl = () => { navigator.clipboard.writeText(curlText); setCurlCopied(true); setTimeout(() => setCurlCopied(false), 1200); }; const copyBench = (key, text) => { navigator.clipboard.writeText(text); setBenchCopied(key); setTimeout(() => setBenchCopied(null), 1200); }; const placeholderGroups = (() => { const out = { command: [], curl: [] }; for (const [key, meta] of Object.entries(config.placeholders || ({}))) { (out[meta.target] || (out[meta.target] = [])).push({ key, ...meta }); } return out; })(); const renderButton = (item, dim, selectedId) => { const checked = selectedId === item.id; const disabled = !isEnabled(dim, item.id); return ; }; const renderFlatSection = (title, options, dim, selectedId) =>
{title}
{options.map(item => renderButton(item, dim, selectedId))}
; const maxHwCols = Math.max(...hwGroups.map(x => x.items.length)); if (commandBuilder) { const scopeLabel = { base: "Setup", serve: "Server", request: "Request" }; const scopedDims = scope => overlayDimSpecs.filter(dim => { if ((dim.scope || "base") !== scope) return false; if (dim.kind === "number") { return typeof dim.showWhen !== "function" || dim.showWhen(sel); } return rowVisible(dim, sel); }); const baseDims = scopedDims("base"); const serveDims = scopedDims("serve"); const requestDims = scopedDims("request"); const errors = builderMeta.errors || []; const warnings = builderMeta.warnings || []; const invalid = errors.length > 0; const totalGpus = Number(sel.nodes) * Number(sel.gpus_per_node); const topology = builderMeta.topology || ({}); const verification = builderMeta.verification || ({}); const scopeIsVerified = scope => scopedDims(scope).every(dim => { const option = (dim.options || []).find(entry => entry.id === sel[dim.id]); if (option && optionSoft(option, sel)) return false; const predicate = option?.verifiedWhen ?? dim.verifiedWhen; return typeof predicate === "function" ? !!predicate(sel) : predicate !== false; }); const serveStatus = invalid ? "error" : scopeIsVerified("serve") ? verification.serve || verifyStatus : "unverified"; const requestStatus = invalid ? "error" : scopeIsVerified("request") ? verification.request || verifyStatus : "unverified"; const statusText = status => ({ verified: "Verified", unverified: "Unverified", "in-progress": "Verification in progress", error: "Invalid configuration" })[status] || "Unverified"; const activeServerSetting = serveDims.find(dim => dim.id === builderServerSetting) || serveDims[0]; const selectedOption = dim => (dim.options || []).find(option => option.id === sel[dim.id]); const effectiveSetting = dim => builderMeta.resolvedSettings?.[dim.id] || selectedOption(dim)?.label || sel[dim.id] || "—"; const recommendedRecipe = recommendedBuilderRecipe(sel.hw); const recommendedIsVerified = !!recommendedRecipe && !recommendedRecipe.unverified && (typeof recommendedRecipe.verifiedWhen !== "function" || recommendedRecipe.verifiedWhen(sel)); const recommendedInUse = !!recommendedRecipe && Number(sel.nodes) === recommendedRecipe.nodes && Number(sel.gpus_per_node) === recommendedRecipe.gpus_per_node && sel.topology_mode === "auto" && serveDims.every(dim => { const expected = recommendedRecipe[dim.id] ?? dim.default; if (dim.id === "attention") return sel.attention === "platform"; return expected === undefined || sel[dim.id] === expected || ["placement", "encoder"].includes(dim.id) && sel[dim.id] === "auto"; }); const restoreRecommendedRecipe = () => { if (!recommendedRecipe) return; setSel(prev => reseatHiddenPicks(normalizeBuilderSelection({ ...prev, ...Object.fromEntries(serveDims.map(dim => [dim.id, (recommendedRecipe[dim.id] ?? dim.default) ?? dim.options?.[0]?.id])), nodes: recommendedRecipe.nodes, gpus_per_node: recommendedRecipe.gpus_per_node, topology_mode: "auto", tp_size: recommendedRecipe.tp_size, ulysses_degree: recommendedRecipe.ulysses_degree, ring_degree: recommendedRecipe.ring_degree, placement: recommendedRecipe.placement || "auto", attention: "platform", precision: "native", encoder: recommendedRecipe.encoder || "auto", execution: recommendedRecipe.execution || "eager" }))); }; const renderBuilderChoice = (item, dim) => { const checked = sel[dim.id] === item.id; const disabled = !isEnabled(dim.id, item.id); const soft = !disabled && optionSoft(item, sel); const reason = disabled ? item.disableReason || "Not available for this configuration" : soft ? item.softReason || "Runs, but this combination is not a verified recipe yet." : ""; return ; }; const renderBuilderDimension = dim =>
{dim.title} {dim.description && {dim.description}}
5 ? "compact" : "normal"}> {visibleOptions(dim, sel).map(option => renderBuilderChoice(option, dim))}
{blockedNote && blockedNote.dim === dim.id &&

{blockedNote.reason}

}
; const renderStepper = (key, label, detail) => { const bounds = commandBuilder.resource?.limits?.[key] || ({ min: 1, max: 8 }); return
{label} {detail && {detail}}
{renderBuilderNumberInput({ identity: `${key}-${sel[key]}`, value: sel[key], min: bounds.min, max: bounds.max, label, onCommit: value => setBuilderResource(key, value) })}
; }; const renderBaseScope = () =>
{recommendedRecipe &&
{} {recommendedIsVerified ? "Verified recipe" : "Derived recipe"} · {sel.hw.toUpperCase()} {[`${recommendedRecipe.nodes * recommendedRecipe.gpus_per_node} GPUs`, recommendedRecipe.tp_size > 1 && `TP ${recommendedRecipe.tp_size}`, `Ulysses ${recommendedRecipe.ulysses_degree}`, recommendedRecipe.ring_degree > 1 && `Ring ${recommendedRecipe.ring_degree}`, ({ resident: "Resident", fsdp: "FSDP", offload: "Layerwise offload" })[recommendedRecipe.placement]].filter(Boolean).join(" · ")}
{renderStatus(recommendedIsVerified ? "verified" : "unverified")} {recommendedInUse ? In use : }
}
Hardware
{hwGroups.flatMap(group => group.items).map(item => { const selected = sel.hw === item.id; return ; })}
{}
Resources {builderMeta.topologySummary || "No valid topology"}
{renderStepper("nodes", "Nodes")} {renderStepper("gpus_per_node", "GPUs / node")}
{Number(sel.nodes) > 1 &&

{sel.nodes} nodes × {sel.gpus_per_node} {sel.hw.toUpperCase()} = {totalGpus} GPUs

} {builderAdvanced &&

Auto uses an exact verified recipe when one exists; manual values are allowed when the model constraints remain valid.

{[["tp_size", "Tensor parallel", [1, 2, 4, 8]], ["ulysses_degree", "Ulysses", [1, 2, 4, 8, 16]], ["ring_degree", "Ring", [1, 2, 4, 8]]].map(([key, label, values]) => )}
} {(errors.length > 0 || warnings.length > 0) &&
{(errors.length ? errors : warnings).map((message, index) =>

{message}

)}
}
{baseDims.map(renderBuilderDimension)}
; const renderSettingEditor = (dim, className = "", direct = false) => { if (!dim) return null; const options = visibleOptions(dim, sel); const currentOption = selectedOption(dim); return
{direct ? dim.title : `${dim.title} options`} {dim.description &&

{dim.description}

}
{dim.quality && {dim.quality}}
{dim.kind === "number" ?
{renderBuilderNumberInput({ identity: `${dim.id}-${sel[dim.id]}`, value: sel[dim.id], min: dim.min, max: dim.max, label: dim.title, onCommit: value => setSel(prev => ({ ...prev, [dim.id]: value })) })} {dim.unit || "outputs"}
:
{options.map(option => renderBuilderChoice(option, dim))}
} {blockedNote && blockedNote.dim === dim.id &&

{blockedNote.reason}

} {(currentOption?.description || dim.learnMore) &&
{currentOption?.description &&

{currentOption.description}

} {} {dim.learnMore && Learn more } {dim.docsHref && SGLang docs }
}
; }; const renderServerScope = () =>
{serveDims.map(dim => { const isActive = dim.id === activeServerSetting?.id; const option = selectedOption(dim); const recommended = typeof option?.recommendedWhen === "function" ? option.recommendedWhen(sel) : !!option?.recommended; return
{isActive && renderSettingEditor(dim, "sgd-builder-context--inline")}
; })}
{renderSettingEditor(activeServerSetting, "sgd-builder-context--rail")}
; const renderRequestScope = () =>
{requestDims.map(dim =>
{renderSettingEditor(dim, "", true)}
)}
; const renderScopeControls = () => { if (builderScope === "base") return renderBaseScope(); if (builderScope === "serve") return renderServerScope(); return renderRequestScope(); }; const renderStatus = status => ; const renderOutputCard = type => { const serve = type === "serve"; const text = serve ? command : curlText; const canExpand = text.split("\n").length > 9; const expanded = serve ? serveExpanded : requestExpanded; const setExpanded = serve ? setServeExpanded : setRequestExpanded; const status = serve ? serveStatus : requestStatus; const emphasized = builderScope === "base" || builderScope === type; return
{serve ? "1" : "2"}
{serve ? "Serve" : "Request"} {serve ? `${sel.hw.toUpperCase()} · ${activeRunMode === "docker" ? "Docker" : "Python"}` : "cURL"}
{renderStatus(status)}
{serve && runModes.length > 1 &&
{runModes.map(mode => )}
} {serve && Number(sel.nodes) > 1 &&
} 
{text}
{invalid &&
{errors[0]}
}
{canExpand && }
; }; return
{renderScopeControls()}
{builderScope !== "request" && renderOutputCard("serve")} {builderScope !== "serve" && renderOutputCard("request")}
{builderScope === "serve" &&

Tip: For more server options, see the{" "} CLI reference.

} {modal === "env" &&
setModal(null)}>
event.stopPropagation()}>
Command variables
{["command", "curl"].map(target => placeholderGroups[target].length > 0 &&
{target === "command" ? "Serve" : "Request"}
{placeholderGroups[target].map(({key, label}) =>
setEnvDraft({ ...envDraft, [key]: event.target.value })} />
)}
)}
}
; } return
{}
Hardware Platform
{hwGroups.map(g =>
{g.label &&
{g.label}
}
{g.items.map(item => renderButton(item, "hw", sel.hw))} {Array.from({ length: maxHwCols - g.items.length }).map((_, i) =>
)}
)}
{matchDimSpecs.filter(d => rowVisible(d, sel)).map(d =>
{renderFlatSection(d.title, visibleOptions(d, sel), d.id, sel[d.id])}
)} {overlayDimSpecs.filter(d => rowVisible(d, sel)).map(d =>
{renderFlatSection(d.title, visibleOptions(d, sel), d.id, sel[d.id])}
)} {}
Command:
{cell && cell.redirect ? cell.warn &&
⚠️ {renderWarn(cell.warn)}
: <>
{VERIFY_LABEL[verifyStatus]}
{runModes.map((mode, index) => setRunMode(mode)} onKeyDown={e => { if (e.key !== "Enter" && e.key !== " ") return; e.preventDefault(); setRunMode(mode); }} role="tab" tabIndex={0} aria-selected={activeRunMode === mode}> {mode === "docker" ? "Docker" : "Python"} )} 
{command}
{cell && cell.warn &&
⚠️ {renderWarn(cell.warn)}
} {mtpHint &&
⚠️ Speculative decoding ({specAlgoName}) is on — SGLang resets --max-running-requests to 48 when it isn't set. Add --max-running-requests <N> sized for your target concurrency.
} {specPinnedHint &&
ℹ️ Speculative decoding ({specAlgoName}) is on and this recipe pins --max-running-requests to {specMrrValue}. Adjust it to match your target concurrency — if you remove the flag, SGLang falls back to 48.
} }
{} {benchmarks && cell && renderBenchmarkCard(benchEntry)} {} {config.showPlaygroundLink !== false &&
Need to go beyond the verified matrix?
} {} {agenticLink &&
Need to serve Agentic Long-Context workloads?
} {} {modal === "curl" &&
setModal(null)}>
e.stopPropagation()}>
cURL example
Model: {modelName || "(unresolved)"} 
{curlText}

Edit CURL_HOST / CURL_PORT in the Env panel.

} {} {modal === "env" &&
setModal(null)}>
e.stopPropagation()}>
Env / placeholder values
{placeholderGroups.curl.length > 0 &&
cURL placeholders
{placeholderGroups.curl.map(({key, label}) =>
setEnvDraft({ ...envDraft, [key]: e.target.value })} />
)}
} {placeholderGroups.command.length > 0 &&
Command placeholders
{placeholderGroups.command.map(({key, label}) =>
setEnvDraft({ ...envDraft, [key]: e.target.value })} />
)}
}

Values persist in localStorage and are reused the next time you visit any cookbook.

} {} {modal === "bench" && benchEntry && (() => { const bc = buildBenchCommands(benchEntry, sel); if (!bc) return null; const selSummary = [sel.hw && sel.hw.toUpperCase(), sel.variant, sel.quant && sel.quant.toUpperCase(), sel.strategy, sel.kvDsaPair, sel.nodes].filter(part => part !== undefined && part !== null && part !== "").join(" · "); let selConc = null; let speedCmd = null; if (bc.speed) { selConc = bc.speed.concurrencies.includes(benchConc) ? benchConc : bc.speed.concurrencies[0]; const w = bc.speed.workload; speedCmd = interpolate(bc.speed.template, { ...env, DATASET: w.dataset, ISL: w.isl, OSL: w.osl, MAX_CONCURRENCY: selConc, NUM_PROMPTS: bc.speed.numPromptsOf(selConc) }, modelName); } let selAcc = null; let accCmd = null; if (bc.accuracy.length > 0) { selAcc = bc.accuracy.find(a => a.key === benchAcc) || bc.accuracy[0]; accCmd = interpolate(selAcc.template, env, modelName); } return
setModal(null)}>
e.stopPropagation()}>
Benchmark commands

For {selSummary}. Start the server with the Deploy command above, then run these against it.

{selAcc &&
Accuracy
{bc.accuracy.length > 1 &&
benchmark: {bc.accuracy.map(a => )}
}
{selAcc.label}
{accCmd}
{bc.accuracy.length > 1 &&

Switch the benchmark chip to see each eval's command.

}
} {bc.speed &&
Speed
{bc.speed.concurrencies.length > 1 &&
max-concurrency: {bc.speed.concurrencies.map(c => )}
}
max-concurrency = {selConc}
{speedCmd}

One command — switch the concurrency chip (or edit --max-concurrency) to reproduce each Speed column.

}

Edit CURL_HOST / CURL_PORT in the Env panel.

; })()}
; }; ## Deployment Clef support merged after v0.5.21 in [#42721](https://github.com/sgl-project/sglang/pull/42721). Use the pinned nightly wheel or Docker image below. See the [installation guide](/docs/get-started/install) for other methods. This command selects the wheel from [nightly build 37ae292e6f](https://github.com/sgl-project/sglang/actions/runs/37897861964), which includes Clef support. It targets Python 3.10 on Linux x86\_64 with glibc 2.39 or newer. ```bash Command theme={null} pip install --upgrade pip pip install uv uv venv --python 3.10 source .venv/bin/activate uv pip install --prerelease=allow --index-strategy unsafe-best-match --extra-index-url https://docs.sglang.ai/whl/cu130/ "sglang==0.5.22.dev20261009+g37ae292e6f" ``` Run the command from the wizard’s Python tab in this environment. ```bash Command theme={null} docker pull lmsysorg/sglang:dev-clef ``` The [Clef image build](https://github.com/sgl-project/sglang/actions/runs/37893990016) produces this image from the #42721 merge commit. Run the command from the wizard’s Docker tab. Both checkpoints are public; no Hugging Face token is required. Choose your GPU, then Clef or Clef Flash. The wizard updates the model in the launch command and sample request. Both models run on one GPU in BF16 with TP=1 using the default settings; the commands need only the model path, host, and port. ## 1. Model introduction Cloudflare post-trained Clef from Qwen3.8-27B and Clef Flash from Qwen3.5-9B, then released both under the Apache-2.0 license. Each model keeps its backbone and vision encoder and adds a joint schema head. The head reads the backbone’s final hidden states, routes state evidence to each option, and lets questions attend to one another. It returns one logit per allowed option in a single prefill, with no text generation.
Variant Checkpoint Backbone
Clef (27B) Cloudflare/clef Qwen3.8-27B
Clef Flash (9B) Cloudflare/clef-flash Qwen3.5-9B
Pass a text string, JSON object, or JSON array as the state, with optional images. `/v1/systemone` returns one answer per question: a `choice` over allowed options, a `noul` probability of true, or a `score` over ordered levels. ## 2. Configuration tips * **Route.** Send requests to `/v1/systemone`, or set your System One client’s base URL to the server. This includes TypeSafe SDK clients. See the [API reference](/docs/supported-models/decision_models#system-one-compatible-api) for request and response fields. Clef refuses `/v1/decisions`, chat, and generation requests. * **One GPU, one prefill per request.** Keep TP=1 and PP=1. All questions share one prompt, so the model reads the state once. SGLang detects `joint_head_config.json`, loads `joint_head.safetensors`, and enables embedding mode. It disables radix caching, chunked prefill, and CUDA graphs. No reasoning parser, tool-call parser, or chat-template override is needed. * **Answers.** `probabilities` are the softmax of the head’s logits for each question. `confidence` uses the System One adapter formulas. It can differ from the checkpoint’s reference `systemone()`, which reports the top option’s probability as confidence. Answers omit `x_label_mass` because the head scores only the allowed options. * **Long states.** Prompts, including images, are capped at 16,384 tokens, the reference `encode_record` function’s default `max_length`. The effective limit can be lower, depending on the server context, KV token pool, and `--max-prefill-tokens`. The server truncates the state to leave room for the questions and images. If those do not fit even without the state, it rejects the request. * **Images.** Send each image in `images` as a data URL, such as `data:image/jpeg;base64,...`, or an HTTP(S) URL. Keep the data-URL prefix for JPEGs: bare base64 starts with `/`, which the loader treats as a file path, causing the request to fail. Images precede the state. ## 3. Advanced usage ### 3.1 Text decisions Set `MODEL` to the checkpoint you selected in the wizard. The request works with either variant. ```python Example theme={null} import requests MODEL = "Cloudflare/clef-flash" # Or "Cloudflare/clef". response = requests.post( "http://localhost:30000/v1/systemone", json={ "model": MODEL, "state": "My Stripe integration keeps failing. I am losing sales. Please help ASAP.", "questions": { "routing": { "type": "choice", "instructions": "Which team should handle this request?", "criteria": { "billing": "Charges and refunds", "technical_support": "Integration errors", "sales": "Questions about buying a product", }, }, "urgency": {"type": "noul", "instructions": "Does this message express urgency?"}, "frustration": { "type": "score", "instructions": "How frustrated is the customer?", "criteria": ["Calm", "Frustrated but civil", "Very angry"], }, }, }, timeout=60, ) response.raise_for_status() answers = response.json()["answers"] print("Routing:", answers["routing"]["choice"]) print("Urgency:", answers["urgency"]["noul"]) print("Frustration:", answers["frustration"]["score"]) print("Usage:", response.json()["usage"]) ``` Captured with Clef Flash on one H200. Probabilities can vary with the model variant, hardware, and numerical backend. ```text Output theme={null} Routing: technical_support Urgency: 0.9492946352021694 Frustration: 1.7297646970642042 Usage: {'input_tokens': 334, 'output_tokens': 0} ``` ## 4. GSM8K accuracy [Decision Index 0.2.1](https://github.com/apolinario/decision-index/blob/9eb2dbe2a358004c8782c66e40a83ac07b953fec/docs/suite.md) evaluates all 1,319 GSM8K test problems with both four-choice and ten-choice questions, for 2,638 requests per configuration. The table reports accuracy across both formats through `/v1/systemone`. | Model | [Reported](https://huggingface.co/Cloudflare/clef#results) | H200 | B200 | B300 | | - | -: | -: | -: | -: | | Clef | 80.8% | 80.71% | 80.59% | 80.59% | | Clef Flash | 67.3% | 67.32% | 67.36% | 67.36% | All runs use BF16 and TP=1 on one GPU. This documentation is built and hosted on [Mintlify](https://mintlify.com), a developer documentation platform.