┌───────────────────────────────────────────────────────────────────┐
│                         Hanzo Engine                              │
├───────────────────────────────────────────────────────────────────┤
│                                                                   │
│  Clients                                                          │
│  ───────                                                          │
│  OpenAI SDK ─┐                                                    │
│  curl / HTTP ─┤──▶ ┌──────────────────────────────────┐          │
│  Python SDK  ─┤    │   HTTP Server (axum)              │          │
│  Rust SDK    ─┤    │   :36900                          │          │
│  MCP Client  ─┘──▶ │   /v1/chat/completions            │          │
│                     │   /v1/completions                 │          │
│                     │   /v1/embeddings                  │          │
│                     │   /v1/models                      │          │
│                     │   /health                         │          │
│                     │   /docs (SwaggerUI)               │          │
│                     └─────────┬────────────────────────┘          │
│                               │                                   │
│                               ▼                                   │
│                     ┌──────────────────────────────────┐          │
│                     │   Pipeline Engine                 │          │
│                     │   ──────────────────              │          │
│                     │   • Continuous batching           │          │
│                     │   • PagedAttention + prefix cache │          │
│                     │   • ISQ / GGUF / GPTQ / AWQ      │          │
│                     │   • Device mapping (multi-GPU)    │          │
│                     │   • Chat template detection       │          │
│                     └─────────┬────────────────────────┘          │
│                               │                                   │
│              ┌────────────────┼────────────────┐                  │
│              ▼                ▼                 ▼                  │
│  ┌──────────────┐  ┌──────────────┐  ┌──────────────────┐       │
│  │  Metal (GPU) │  │  CUDA (GPU)  │  │  CPU (fallback)  │       │
│  │  Apple Si    │  │  FlashAttn   │  │  AVX2 / MKL      │       │
│  │  AFQ quants  │  │  Marlin kern │  │  GGUF / HQQ      │       │
│  └──────────────┘  └──────────────┘  └──────────────────┘       │
│                                                                   │
│  ┌──────────────────────────────────────────────────────┐        │
│  │   MCP Server (:4321)                                  │        │
│  │   Exposes model tools via JSON-RPC 2.0                │        │
│  └──────────────────────────────────────────────────────┘        │
│                                                                   │
└───────────────────────────────────────────────────────────────────┘

# Serve any Zen model
hanzo-engine serve -m zenlm/zen4-mini --port 8000
hanzo-engine serve -m zenlm/zen4 --port 8000 --isq Q4K
hanzo-engine serve -m zenlm/zen4-coder --port 8000

curl -sSL https://engine.hanzo.ai/install.sh | sh

cargo install hanzo-engine

pip install hanzo-engine               # CPU
pip install hanzo-engine-cuda          # NVIDIA GPU
pip install hanzo-engine-metal         # Apple Silicon
pip install hanzo-engine-mkl           # Intel CPU

# macOS (Metal)
cargo build --package hanzo-engine --release --no-default-features --features metal

# Linux (CUDA)
cargo build --package hanzo-engine --release --features cuda

# Linux (CUDA + FlashAttention)
cargo build --package hanzo-engine --release --features "cuda flash-attn cudnn"

# Interactive chat with auto-detected architecture
hanzo-engine run -m zenlm/zen4-mini

# Start HTTP server with web UI
hanzo-engine serve --ui -m google/gemma-3-4b-it

# Start server with ISQ 4-bit quantization
hanzo-engine serve -m meta-llama/Llama-3.2-3B-Instruct --isq 4

# Run a GGUF quantized model
hanzo-engine run --format gguf -f ./model.Q4_K_M.gguf

# Auto-tune for your hardware
hanzo-engine tune -m zenlm/zen4-mini --emit-config config.toml
hanzo-engine from-config -f config.toml

# System diagnostics
hanzo-engine doctor

curl http://localhost:36900/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer EMPTY" \
  -d '{
    "model": "default",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "Explain PagedAttention in one paragraph."}
    ],
    "max_tokens": 256,
    "temperature": 0.7
  }'

import openai

client = openai.OpenAI(
    base_url="http://localhost:36900/v1",
    api_key="EMPTY",
)

completion = client.chat.completions.create(
    model="default",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Write a haiku about Rust."},
    ],
    max_tokens=128,
)

print(completion.choices[0].message.content)

stream = client.chat.completions.create(
    model="default",
    messages=[{"role": "user", "content": "Count to 10 slowly."}],
    stream=True,
)

for chunk in stream:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)

curl http://localhost:36900/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "default",
    "prompt": "The Rust programming language is",
    "max_tokens": 128
  }'

hanzo-engine serve -m google/embeddinggemma-300m

curl http://localhost:36900/v1/embeddings \
  -H "Content-Type: application/json" \
  -d '{
    "model": "default",
    "input": "Hello, world!"
  }'

# 4-bit quantization (auto-selects best method for your hardware)
hanzo-engine serve -m meta-llama/Llama-3.2-3B-Instruct --isq 4

# 8-bit quantization
hanzo-engine serve -m zenlm/zen4-mini --isq 8

# Direct GGUF file
hanzo-engine run --format gguf -f ./zen4-mini-Q4_K_M.gguf

# Auto-detect GPTQ model from HuggingFace
hanzo-engine run -m kaitchup/Phi-3-mini-4k-instruct-gptq-4bit

hanzo-engine serve -m zenlm/zen4-mini --topology topology.toml

# Custom KV cache memory allocation
hanzo-engine serve -m zenlm/zen4-mini --paged-attn --paged-attn-gpu-memory 8GiB

# KV cache quantization for longer contexts
hanzo-engine serve -m zenlm/zen4-mini --paged-attn --paged-cache-type f8e4m3

hanzo-engine serve \
  -m zenlm/zen4-mini \
  --port 36900 \
  --mcp-port 4321

hanzo-engine serve \
  -m zenlm/zen4-mini \
  --mcp-client-config mcp-tools.json

from hanzo_engine import Runner, Which, ChatCompletionRequest

runner = Runner(
    which=Which.Plain(model_id="zenlm/zen4-mini"),
    in_situ_quant="4",
)

response = runner.send_chat_completion_request(
    ChatCompletionRequest(
        model="default",
        messages=[{"role": "user", "content": "Hello!"}],
        max_tokens=256,
    )
)
print(response.choices[0].message.content)

use anyhow::Result;
use hanzo_engine::{IsqType, TextMessageRole, TextMessages, TextModelBuilder};

#[tokio::main]
async fn main() -> Result<()> {
    let model = TextModelBuilder::new("zenlm/zen4-mini")
        .with_isq(IsqType::Q4K)
        .with_logging()
        .build()
        .await?;

    let messages = TextMessages::new()
        .add_message(TextMessageRole::User, "Hello!");

    let response = model.send_chat_request(messages).await?;
    println!("{}", response.choices[0].message.content.as_ref().unwrap());

    Ok(())
}

# CPU
docker run -p 8000:8000 ghcr.io/hanzoai/engine:latest \
  serve -m zenlm/zen4-mini --port 8000

# NVIDIA GPU
docker run --gpus all -p 8000:8000 ghcr.io/hanzoai/engine:cuda \
  serve -m zenlm/zen4 --port 8000

# Apple Silicon (Metal)
docker run -p 8000:8000 ghcr.io/hanzoai/engine:metal \
  serve -m zenlm/zen4-mini --port 8000

services:
  engine:
    image: ghcr.io/hanzoai/engine:cuda
    ports:
      - "8000:8000"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    volumes:
      - model-cache:/root/.cache/huggingface
    command: serve -m zenlm/zen4 --port 8000
    restart: unless-stopped

volumes:
  model-cache:

apiVersion: apps/v1
kind: Deployment
metadata:
  name: hanzo-engine
spec:
  replicas: 1
  selector:
    matchLabels:
      app: hanzo-engine
  template:
    metadata:
      labels:
        app: hanzo-engine
    spec:
      containers:
        - name: engine
          image: ghcr.io/hanzoai/engine:cuda
          command: ["hanzo-engine", "serve", "-m", "zenlm/zen4", "--port", "8000"]
          ports:
            - containerPort: 8000
          resources:
            limits:
              nvidia.com/gpu: "1"
          volumeMounts:
            - name: model-cache
              mountPath: /root/.cache/huggingface
      volumes:
        - name: model-cache
          persistentVolumeClaim:
            claimName: model-cache-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: hanzo-engine
spec:
  selector:
    app: hanzo-engine
  ports:
    - port: 8000
      targetPort: 8000
  type: ClusterIP

# 2x GPU tensor parallelism
HANZO_ENGINE_LOCAL_WORLD_SIZE=2 hanzo-engine serve \
  -m zenlm/zen4 --port 8000

# 4x GPU
HANZO_ENGINE_LOCAL_WORLD_SIZE=4 hanzo-engine serve \
  -m zenlm/zen4-max --port 8000

# Node 0 (master)
RING_CONFIG=ring_node0.json hanzo-engine serve -m zenlm/zen4 --port 8000

# Node 1
RING_CONFIG=ring_node1.json hanzo-engine serve -m zenlm/zen4 --port 8001

[speculative]
draft_model = "zenlm/zen4-mini"
gamma = 16

hanzo-engine from-config -f config.toml

hanzo-engine bench -m zenlm/zen4-mini --isq Q4K
hanzo-engine tune -m zenlm/zen4-mini --emit-config optimal.toml

# Generate optimal config for your hardware
hanzo-engine tune -m zenlm/zen4-mini --emit-config config.toml

# Run from config
hanzo-engine from-config -f config.toml