# syntax=docker/dockerfile:1.7-labs
# SPDX-License-Identifier: Apache-2.0
# arbi-serve — slim image (~4.4 GB; vs ~10-12 GB for full vLLM image).
#
# Design notes (this is the only image we ship; the prior single-stage
# full variant was removed). vs a naive full cudnn-runtime + system-site
# image:
#   * runtime base = `nvidia/cuda:VERSION-runtime-ubuntu24.04` instead
#     of `cudnn-runtime`. Torch wheels bundle libcudnn under
#     `torch/lib/libcudnn_*.so`; the system cudnn from the base image
#     is unused at inference time.
#   * /opt/venv layout instead of system site-packages. Single COPY
#     from builder; no PEP-668 `--break-system-packages` workaround.
#   * Builder pre-bakes xgrammar's torch cpp_extension into
#     /opt/cache-baked/. The arbi_serve._prebake_loader monkey-patch
#     short-circuits torch.utils.cpp_extension.load via
#     torch.ops.load_library so the runtime image needs no
#     ninja/g++/nvcc for xgrammar.
#   * Strip torch's `tests/`, `examples/`, `dist-info/RECORD` from
#     site-packages. ~200-400 MB savings depending on torch release.
#   * Final image carries no nvcc, no headers, no ninja, no git, no
#     gcc.
#
# What works in the slim image:
#   * /v1/models, /v1/chat/completions API surface
#   * Tokenizer + xgrammar structured generation
#   * Multi-group routing (engine_params override per request)
#
# Attention backend (torch-2.14 stack):
#   * DEFAULT is paged_kv:bf16-turbo (FA4 CuTeDSL prefill + turbo split-K
#     decode), which is FA2-free. flash-attn (FlashAttention-2) is DROPPED —
#     source-building it is forbidden by policy;
#     a measured A/B showed dropping it costs nothing on sm_89 head_dim<=256
#     bf16 decode. The bf16-fa / bf16-fa-persistent backends remain code-only
#     and lazy-fail with a hint pointing at bf16-turbo.
#
#   * --kv-cache-dtype tkv — runtime stage now ships nvcc + g++ +
#     ninja so torch.cpp_extension.load_inline can JIT per-arch
#     kernels for any (head_dim, num_kv_heads) shape on first request.
#     WARMUP_MODEL still recommended for production (skips the ~30s
#     first-request JIT) but no longer required.
#
# Build (context is just arbi-serve/ — turbo-attn installs from PyPI at
# the uv.lock-pinned version, no sibling checkout needed):
#
#   docker build -f arbi-serve/Dockerfile -t arbi-serve:slim arbi-serve/
#
# (turbo-attn was published to PyPI; the old parent-dir context + local
# turbo-attn COPY were removed in favour of a normal locked PyPI install.)
#
# Optional kernel-warmup model (so kernel cache is pre-baked for
# this exact arch + bit width — saves ~30 s on first request):
#   docker build -f arbi-serve/Dockerfile \
#       --build-arg WARMUP_MODEL=/models/Qwen3.8-27B-exl3-4.0bpw \
#       --build-arg WARMUP_K_BITS=4 --build-arg WARMUP_V_BITS=4 \
#       -t arbi-serve:slim .
#
# A group-2 VQ (vq2) KV-codec deployment must ALSO bake the vq2 kernel
# family — it is a different kernel, not a flag on the scalar one, so a
# scalar-only image cannot serve it under TKV_NO_JIT=1:
#   docker build -f arbi-serve/Dockerfile \
#       --build-arg TKV_BAKE_BITS=2,3,4 --build-arg TKV_BAKE_VQ=both \
#       -t arbi-serve:vq2 .

# CUDA Toolkit 13.1, for BOTH the builder (-cudnn-devel) and runtime (-base)
# stages: their two FROM refs carry the same CUDA version and Ubuntu release
# (tests/test_supply_chain_pins.py).
#
# NOTE: this pin was originally taken for the sm89 e4m3 warp-level MMA that
# the (now removed) split-D fp8 prefill path needed — 12.9/13.0 refused it.
# That requirement is retired. The pin now rests on the coordinated
# cutlass-dsl 4.6.x / quack / turbo-attn upgrade documented below; nobody has
# re-tested whether a lower toolkit would serve, so it is kept as-is rather
# than lowered on inference.
ARG PYTHON_VERSION=3.12
# The requested arch list, stated once here and passed explicitly by CI
# (.github/workflows/ci.yml sets GPU_ARCH="8.0;8.6;8.9;9.0;10.0;12.0" for the
# runtime build and --build-arg "GPU_ARCH=..." with the same list for the test
# image): sm_80 (GA100 Ampere: A100, A30), sm_86 (GA10x Ampere: A5000, A5500,
# 3090, A10), sm_89 (Ada 4090, L40S), sm_90 (H100), sm_100 (B200/GB200), sm_120
# (Blackwell 5090). Every one but sm_89 is an ADDED arch (REQUIRED_GPU_ARCH
# stays 8.9), so a missing artifact never blocks the build. sm_121 (GB10 / DGX
# Spark) is not in this x86-64 list: GB10 is an Arm part, so no amd64 host has
# one, and the Arm image passes its own GPU_ARCH with sm_121 (its own artifact:
# the Triton AOT payload and the tkv modules are keyed on the exact arch, and
# an arch-specific sm_120a/f cubin never runs on 12.1).
# sm_80 is its own artifact for the same reason in reverse: CUDA runs an
# sm_X.y cubin only on X.z with z >= y, so an 8.6 cubin never runs on an A100,
# and an image whose sm_80 artifact did not arrive refuses an A100 by name. What the image SERVES
# is the subset whose per-arch kernel artifact arrived (REQUIRED_GPU_ARCH below
# must); the arch record says which. tests/test_ampere_family_arch.py holds the
# statements identical — a local `docker build` with no --build-arg must
# request the same arches the shipped images do.
ARG GPU_ARCH=8.0;8.6;8.9;9.0;10.0;12.0
# Defaults for args the builder reads on both sides of the tkv store bake
# split below; each stage redeclares them bare. Documented where they are used.
ARG TKV_BAKE_BITS=4
ARG TKV_BAKE_VQ=none
ARG TKV_STORE_BAKE_TP=1
ARG TKV_STORE_BAKE_SPEC=0
# Attention SHAPES (tkv.shapes) the store bake plans beside the staged model, and
# their MTP depths: the kernels of any model with that geometry, without naming
# one. Depths are the speculative verify blocks the valve serves (K=3..7).
# scripts/ci/tkv_store_selectors.sh lists every selector run from these.
ARG TKV_STORE_BAKE_SHAPES=gqa8-d128-swa
ARG TKV_STORE_BAKE_SHAPE_SPEC=0,3,4,5,6,7
# The archs the tkv store bake serves: GPU_ARCH, less an ADDED arch (one
# outside REQUIRED_GPU_ARCH) whose prebuilt tkv artifact is not in the
# registry. scripts/build-slim.sh decides it (kernel_artifact.py
# tkv-store-archs) and passes it to every stage below that plans, bakes or
# verifies the store bake; empty means GPU_ARCH. The builder records which
# archs it verified, and the arch record claims no other.
ARG TKV_STORE_BAKE_ARCHS=
# The tkv store bake image the builder COPYs its kernels from: the in-file
# `tkv-store-bake` stage unless scripts/build-slim.sh passes the registry image
# for the bake's content key. See "Store-derived tkv kernel bake" below.
ARG TKV_STORE_KERNEL_BAKE=tkv-store-bake
# The previous tkv store bake, which a new store bake starts from (see
# "Store-derived tkv kernel bake"): empty unless scripts/build-slim.sh passes it.
ARG TKV_STORE_BAKE_SEED=tkv-store-bake-seed-empty
# The stage a per-arch kernel artifact bakes in: `kernel-src` builds for this
# builder's own platform; `xbuild-arm64` cross-compiles an Arm (SBSA) artifact
# on an x86_64 builder (scripts/ci/kernel_arch_artifact.sh picks it).
ARG EXT_BAKE_BASE=kernel-src
# The boot-store packs (Triton/Inductor, CuTeDSL, tkv split-K tables) recorded for every
# config in k8s/served-configs.json on an RTX 4090 (scripts/ci/record_boot_store.sh, published
# by scripts/ci/boot_store_artifact.sh). `runtime` and `test` ship them in the image tier, so
# a fresh boot of a listed config compiles and sweeps nothing. Packs are keyed on the
# toolchain and the card: after a torch / turbo-attn / Triton bump they miss until
# re-recorded, and the boot compiles (and says so).
ARG BOOT_STORE=registry.arbi.work/arbi-boot-store:ta0.68.1-torch2.12.1-sm89-kae99c8863232-cfgf246599fbc4d@sha256:c446b3a2206ce245145d41c87f89ed96a7ac7de4c778a96e5600f94029ccebd0

# ──────────────────────────────────────────────────────────────────
# Stage 1: builder — full CUDA dev toolkit, compile + warm kernels.
# ──────────────────────────────────────────────────────────────────
# ── PER-ARCH KERNEL ARTIFACTS ────────────────────────────────────────────────
# One empty stage per arch the image could serve. scripts/build-slim.sh
# replaces each one it has an artifact for with that artifact
# (`--build-context ext-arch-sm<NN>=docker-image://...arbi-ext-kernels@<digest>`), and the
# builder's assembly reads them (arbi_serve/kernel_arch_bake.py). A stage left
# empty is an arch this build ships without. Serving a new arch is one line
# here and one entry in GPU_ARCH.
FROM scratch AS ext-arch-sm80
FROM scratch AS ext-arch-sm86
FROM scratch AS ext-arch-sm89
FROM scratch AS ext-arch-sm90
FROM scratch AS ext-arch-sm100
FROM scratch AS ext-arch-sm120
FROM scratch AS ext-arch-sm121
# The artifact job's seed: that arch's previous artifact, the same way.
FROM scratch AS ext-arch-seed

FROM scratch AS tkv-store-bake-seed-empty

FROM ${TKV_STORE_BAKE_SEED} AS tkv-store-bake-seed

# Bases are pinned by digest with the tag kept for reading: a tag is mutable
# upstream, so the same Dockerfile could otherwise build on a base nobody
# validated. They are literal FROM refs rather than version ARGs because a
# digest cannot be interpolated from a version, and because Dependabot
# (.github/dependabot.yml) refreshes only a literal FROM.
FROM nvidia/cuda:13.1.1-cudnn-devel-ubuntu24.04@sha256:ef8a68702fdc93f3acb12919cde4468b9df0aeea214c1c97bf57598dad558967 AS builder-base

ARG PYTHON_VERSION
ARG GPU_ARCH

ENV DEBIAN_FRONTEND=noninteractive \
    PIP_NO_CACHE_DIR=1 \
    PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1 \
    # Two-part peak-disk control for the torch/CUDA install (it ENOSPC'd lean
    # build runners — "failed to copy ... No space left on device"):
    #   * UV_NO_CACHE=1 — don't retain a uv download/extract cache. Otherwise uv
    #     holds each wheel TWICE (once in /root/.cache/uv/archive-v0, once in the
    #     venv), so torch peaked ~30GB. No-cache installs straight to the venv.
    #   * UV_LINK_MODE=hardlink — uv installs by HARDLINKING the unpacked wheel
    #     into the venv instead of COPYing it. The unpack dir and /opt/venv are
    #     the same overlayfs upper layer during a RUN, so the link is free and
    #     the venv shares inodes with the unpack (~1x peak) rather than holding a
    #     second physical copy (~2x). This roughly halves the remaining torch +
    #     nvidia-* install peak — the difference between fitting and ENOSPC on a
    #     small build VM. (uv silently falls back to copy if it ever can't link,
    #     e.g. a cross-device TMPDIR, so it's never worse than copy.)
    # docker layer caching still keeps unchanged RUN steps fast; mirrors the
    # PIP_NO_CACHE_DIR=1 above.
    UV_NO_CACHE=1 \
    UV_LINK_MODE=hardlink

RUN apt-get update && apt-get install -y --no-install-recommends \
        python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
        ninja-build curl ca-certificates \
        git build-essential cmake \
    && rm -rf /var/lib/apt/lists/*

# uv for fast resolves + parallel installs: one pinned, checksum-verified
# release binary, not whatever astral.sh/install.sh serves on the day of the
# build. UV_VERSION is the release CI's setup-uv ran when it was pinned. To
# bump: set UV_VERSION and copy the two `uv-<target>.tar.gz.sha256` files of
# that GitHub release into UV_SHA256 / UV_SHA256_ARM64 (tests/test_uv_pin.py).
# The ARGs sit here, after the apt layer, so a bump rebuilds from this step on
# and leaves the layers above it cached.
ARG UV_VERSION=0.13.7
ARG UV_SHA256=c5221ff41b7892bfc223543883600125f6a47fe845093e0f5bdff4f359196d83
ARG UV_SHA256_ARM64=71279f8d3c67241e807b9d8db872d1bf541579fb9f08b7d1eb1682196d1e57dc
ARG TARGETARCH
RUN target=x86_64-unknown-linux-gnu && sha="${UV_SHA256}" \
    && if [ "${TARGETARCH}" = arm64 ]; then target=aarch64-unknown-linux-gnu; sha="${UV_SHA256_ARM64}"; fi \
    && curl -fsSL --retry 3 -o /tmp/uv.tar.gz \
        "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-${target}.tar.gz" \
    && echo "${sha}  /tmp/uv.tar.gz" | sha256sum -c - \
    && tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 "uv-${target}/uv" \
    && rm /tmp/uv.tar.gz \
    && test "$(uv --version | cut -d' ' -f2)" = "${UV_VERSION}"

# Venv at /opt/venv — single dir to copy into runtime.
RUN python${PYTHON_VERSION} -m venv /opt/venv
ENV PATH="/opt/venv/bin:${PATH}" \
    VIRTUAL_ENV="/opt/venv" \
    UV_CACHE_DIR="/root/.cache/uv" \
    UV_LINK_MODE="copy"

# PyTorch and torchvision share the stable cu130 ABI. Native extensions are
# rebuilt against these headers; PyTorch 2.14 requires C++20. The CUDA 13.1.1
# build toolkit remains independent of the wheel's CUDA 13.0 runtime.
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install torch==2.14.1 --index-url https://download.pytorch.org/whl/cu130

# torchvision — multimodal (vision) input path: the bicubic-with-antialias
# resize that matches Qwen-VL's fast image processor bit-for-bit + the
# on-GPU resize/normalize/patchify pipeline. torchvision tracks the torch
# ABI, so pin to the torch-2.14 pair (0.29.x <-> torch 2.14.x) and pull from
# the SAME cu130 index — it reuses the torch installed above instead of
# re-resolving a CPU build (which would also drag in a mismatched torch).
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install torchvision==0.29.1 --index-url https://download.pytorch.org/whl/cu130

# ── The lock, read in a stage of its own ──────────────────────────────────
# LAYER ORDER (cache discipline). Everything below installs from the locked
# dependency set, and a layer is invalidated by every change above it, not only
# by the files it reads. Copying uv.lock into the dependency chain therefore
# re-ran the whole chain -- the ~20-package install, CuTeDSL, FLA, the
# causal-conv1d + mamba nvcc builds, exllamav3, the strip and the multi-GB `mv`
# -- on every turbo-attn release, because a turbo-attn release moves one line
# of uv.lock. The lock is now flattened HERE, off the chain, and the chain
# COPYs only what it reads: the constraints WITHOUT turbo-attn (unchanged by a
# turbo-attn release, so the COPY is a cache hit) and the exllamav3 reference.
# turbo-attn's own requirement line goes to its own file, installed at the END
# of the dependency chain (see "turbo-attn, last" below), so a release re-runs
# that install and the steps that need tkv, and nothing else.
#
# Pin EVERY subsequent PyPI install to the committed uv.lock so the image
# is byte-for-byte reproducible across builds. Floating constraints
# (`xgrammar>=0.2`, its declared but unused Transformers dependency, …) are the
# root cause of "green in one build, red in the next" — the resolved
# version drifts and a cross-checked oracle (transformers GELU) or a
# kernel signature (xgrammar CPU bitmask) moves underneath the tests.
# `uv export` flattens uv.lock to a constraints file; the `grep -v ' @ '`
# drops any non-PyPI (path/git/url) lines so the constraints carry only
# registry pins.
#
# A `-c` constraints file cannot express a direct reference, and a dropped
# direct reference does not fail the install that needs it -- it resolves the
# same NAME off PyPI at whatever version the index offers, which for a package
# whose fork carries the same `__version__` is indistinguishable downstream.
# The direct references are therefore split into their own `-r` file and the
# `grep` that extracts them is the gate: no match, no image. turbo-attn's
# direct reference (empty for a PyPI pin, as before: arbi_serve/_ext_cache_key.py
# reads that file, so its content is part of the extension keys) and its `==`
# PyPI pin go to two files, and one of them must hold the line.
FROM builder-base AS lock-export
COPY arbi-serve/pyproject.toml arbi-serve/uv.lock arbi-serve/README.md /workspace/arbi-serve/
RUN cd /workspace/arbi-serve && \
    uv export --frozen --extra dev --no-hashes --no-emit-project 2>/dev/null \
        > /tmp/arbi-lock-export.txt && \
    grep -v ' @ ' /tmp/arbi-lock-export.txt | grep -v '^turbo-attn==' > /etc/arbi-lock-constraints.txt && \
    (grep -E '^(einops|packaging|wheel|setuptools|psutil)==' /etc/arbi-lock-constraints.txt \
        > /etc/arbi-hybrid-build-constraints.txt || : > /etc/arbi-hybrid-build-constraints.txt) && \
    grep '^exllamav3 @ ' /tmp/arbi-lock-export.txt > /etc/arbi-exllamav3-req.txt && \
    (grep '^turbo-attn @ ' /tmp/arbi-lock-export.txt > /etc/arbi-turbo-attn-req.txt || \
        : > /etc/arbi-turbo-attn-req.txt) && \
    (grep '^turbo-attn==' /tmp/arbi-lock-export.txt > /etc/arbi-turbo-attn-pin.txt || \
        : > /etc/arbi-turbo-attn-pin.txt) && \
    test -s /etc/arbi-turbo-attn-req.txt -o -s /etc/arbi-turbo-attn-pin.txt && \
    rm -f /tmp/arbi-lock-export.txt && \
    echo "locked constraints: $(grep -c '==' /etc/arbi-lock-constraints.txt) pins (turbo-attn's is installed from its own file)" && \
    echo "exllamav3 direct reference: $(cat /etc/arbi-exllamav3-req.txt)" && \
    echo "turbo-attn direct reference: $(cat /etc/arbi-turbo-attn-req.txt)" && \
    echo "turbo-attn PyPI pin: $(cat /etc/arbi-turbo-attn-pin.txt)"

# ── causal-conv1d + mamba-ssm, compiled in a stage of their own ───────────
# These two nvcc builds read the toolchain, torch, GPU_ARCH, their pinned git
# tags and the few build packages below -- nothing else of the lock. Behind
# the COPY of the whole flattened lock (builder-deps), every uv.lock change
# other than turbo-attn's (a dev dependency, the exllamav3 pin, a new runtime
# dependency) would recompile both for every arch. This stage reads
# only arbi-hybrid-build-constraints.txt (lock-export: the pins of exactly the
# packages installed below), so its layers -- in the builder's cache and in the
# registry cache -- hold until one of those inputs moves. The compile itself is
# unchanged: same sources, same -gencode splice, same torch headers and nvcc,
# built with build isolation off against the same pinned build packages; only
# the environment's unrelated packages are absent, and the extensions do not
# read them. builder-deps installs the two wheels --no-deps.
#
# MAX_JOBS / NVCC_THREADS bound the CUDA-extension compile fan-out: causal-conv1d
# and mamba-ssm setup.py default to one ninja job per core, which saturates a
# shared build host. 8 jobs x 2 nvcc threads keeps the build fast without pinning
# every core.
#
# ARCH PIN: both setup.py files IGNORE TORCH_CUDA_ARCH_LIST — they hardcode a
# -gencode sweep over every arch the toolkit supports (9 arches on CUDA 13;
# observed compiling sm_87/100/103/110/121 cubins this image can never run,
# ~6x the nvcc time of the one arch it ships for) and even CLEAR
# TORCH_CUDA_ARCH_LIST on a torch/toolkit major mismatch. Torch's
# cpp_extension defers to explicit -gencode flags, so the env pin above never
# reaches these two builds. Patch the pinned-tag setup.py to emit ONLY the
# ${GPU_ARCH} target(s): each hardcoded pair becomes `pass` (keeps the
# version-guard if blocks syntactically valid) and cc_flag is seeded with the
# target arch LIST. ${GPU_ARCH} is a ';'-separated list ("8.9" single-arch, or
# "8.6;8.9;12.0" for the fat shipped image), so the pin emits one SASS cubin
# per arch PLUS a +PTX on the highest for forward-compat — mirroring how
# TORCH_CUDA_ARCH_LIST fans the torch-native exts (xgrammar/tkv) across the
# same list. One unified image then carries a cubin per listed arch for
# causal-conv1d + mamba, so a single fleet tag serves every listed family.
# Fail-loud asserts catch an upstream layout change at a future tag bump —
# re-verify the patch then, rather than silently sweeping again. Same
# pinned-tag surgery pattern as the mamba_ssm __init__ trim in builder-deps.
FROM builder-base AS hybrid-ext-wheels
ARG GPU_ARCH
ENV TORCH_CUDA_ARCH_LIST="${GPU_ARCH}"
# One connect timeout to pypi.org used to fail the whole image build (uv's
# default: 3 retries, 30 s). Each stage that resolves from the network
# retries longer. Set where the stage's layers already rebuild with its
# inputs, so no cached layer upstream of it changes key.
ENV UV_HTTP_TIMEOUT=120 UV_HTTP_RETRIES=10
COPY --from=lock-export /etc/arbi-hybrid-build-constraints.txt /etc/
COPY <<'PIN_ARCH_PY' /tmp/pin-cuda-arch.py
import re
import sys

# arch_spec: ';'-separated SM codes (GPU_ARCH with dots stripped), e.g. "89"
# (single-arch) or "89;120" (fat Ada+Blackwell). Emit one SASS cubin
# (-gencode arch=compute_XX,code=sm_XX) per arch PLUS a single +PTX
# (code=compute_XX) on the HIGHEST arch for forward-compat on future GPUs the
# image was not built for — the same fan-out torch applies for the exts that
# honor TORCH_CUDA_ARCH_LIST. These two setup.py files ignore that env, so we
# splice the equivalent -gencode LIST in by hand.
path, arch_spec = sys.argv[1], sys.argv[2]
archs = [a for a in arch_spec.split(";") if a]
assert archs, f"no arch codes parsed from {arch_spec!r}"
gencode: list[str] = []
for a in archs:
    gencode += ["-gencode", f"arch=compute_{a},code=sm_{a}"]
top = max(archs, key=int)
gencode += ["-gencode", f"arch=compute_{top},code=compute_{top}"]
cc_flag_init = "cc_flag = [" + ", ".join(f'"{f}"' for f in gencode) + "]"

# Both pinned upstream builds force C++17; Torch 2.14 headers need C++20.
src = open(path).read().replace("-std=c++17", "-std=c++20")
src, n = re.subn(
    r'cc_flag\.append\("-gencode"\)\n\s*cc_flag\.append\("arch=compute_\d+,code=sm_\d+"\)',
    "pass",
    src,
)
assert n >= 6, (
    f"{path}: hardcoded -gencode sweep not found (matched {n}) — "
    "upstream setup.py layout changed; re-verify this patch"
)
# Function replacement (not an f-string) so a literal '[' / '\\' in the
# assembled -gencode list is never read as a regex backreference.
src, n = re.subn(
    r"cc_flag = \[\]",
    lambda _m: cc_flag_init,
    src,
    count=1,
)
assert n == 1, (
    f"{path}: cc_flag initializer not found — "
    "upstream setup.py layout changed; re-verify this patch"
)
open(path, "w").write(src)
print(f"pinned {path} to {cc_flag_init}")
PIN_ARCH_PY
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install -c /etc/arbi-hybrid-build-constraints.txt einops packaging wheel setuptools psutil && \
    ARCH_SM=$(printf '%s' "${GPU_ARCH}" | tr -d .) && \
    mkdir -p /wheels && \
    git clone --depth 1 -b v1.6.2.post1 https://github.com/Dao-AILab/causal-conv1d.git /tmp/cc1d && \
    python /tmp/pin-cuda-arch.py /tmp/cc1d/setup.py "$ARCH_SM" && \
    MAX_JOBS=8 NVCC_THREADS=2 CAUSAL_CONV1D_FORCE_BUILD=TRUE \
        uv build --wheel --no-build-isolation --out-dir /wheels /tmp/cc1d && \
    rm -rf /tmp/cc1d && \
    git clone --depth 1 -b v2.3.2.post1 https://github.com/state-spaces/mamba.git /tmp/mamba && \
    python /tmp/pin-cuda-arch.py /tmp/mamba/setup.py "$ARCH_SM" && \
    MAX_JOBS=8 NVCC_THREADS=2 MAMBA_FORCE_BUILD=TRUE \
        uv build --wheel --no-build-isolation --out-dir /wheels /tmp/mamba && \
    rm -rf /tmp/mamba && \
    ls /wheels/causal_conv1d-*.whl /wheels/mamba_ssm-*.whl

FROM builder-base AS builder-deps
ARG PYTHON_VERSION
ARG GPU_ARCH
# Network retries: see hybrid-ext-wheels.
ENV UV_HTTP_TIMEOUT=120 UV_HTTP_RETRIES=10
# CuTeDSL ships as a FAMILY: the frontend (nvidia-cutlass-dsl) plus
# libs-base/libs-core/libs-cu12/libs-cu13. They must all be the SAME version —
# a frontend newer than libs-cu13 fails _get_baked_dso_digest() with "Missing or
# stale DSO version module", and forcing past that SIGSEGVs. Both install sites
# below read this ARG so they cannot drift.
ARG CUTLASS_DSL_VERSION=4.6.2

# arbi-serve runtime deps. Non-editable so /workspace/arbi-serve is
# also throwaway after this stage. The source tree (arbi_serve/) is copied much
# later, just before the self-install + exl3 bake that actually need it, so a
# code-only merge hits cache on the whole dependency + bake block here.
COPY --from=lock-export /etc/arbi-lock-constraints.txt /etc/arbi-exllamav3-req.txt /etc/
COPY arbi-serve/scripts/packaging/patch_xgrammar_optional_transformers.py /workspace/arbi-serve/

# arbi-serve's third-party runtime deps (NOT arbi-serve itself — that
# self-install moved below the source COPY, so this whole layer stays cached
# across code-only changes). All pinned by the lock constraints.
#
# nvidia-cutlass-dsl carries the [cu13] EXTRA on its requirement line (not just
# the lock): `-c` constraints pin the version but ignore extras, so a bare
# `nvidia-cutlass-dsl` resolves to libs-base (CUDA 12.9). [cu13] pulls
# libs-cu13 (Toolkit 13.x), which is what the pin above keeps.
#
# cutlass-dsl 4.6.x cu13: the coordinated cutlass + quack + turbo-attn upgrade is
# now unblocked. 4.6.x removed ``cute.core.ThrMma`` (moved to ``cute.ThrMma``);
# turbo-attn >=0.28.2 uses the 4.6 compat in ``tkv/kernels/cute/_fa/utils.py`` and
# quack-kernels >=0.6 dropped the ``cute.core.ThrMma`` annotation, so the FA4 cute
# import surface is green on 4.6. At 4.6.1 the [cu13] wheels read CUDA 13 cleanly.
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install -c /etc/arbi-lock-constraints.txt \
        fastapi 'uvicorn[standard]' 'pydantic>=2' \
        'orjson>=3.9' \
        'pyzmq>=25' 'msgspec>=0.19' \
        tokenizers safetensors jinja2 numpy typing-extensions \
        huggingface-hub python-multipart \
        httpx attrs python-dateutil \
        'apache-tvm-ffi>=0.1.12' \
        loguru \
        'cryptography>=41' \
        "nvidia-cutlass-dsl[cu13]==${CUTLASS_DSL_VERSION}" \
        'quack-kernels>=0.6' \
        'opentelemetry-api>=1.36.0' 'opentelemetry-sdk>=1.36.0' \
        'opentelemetry-exporter-otlp-proto-http>=1.36.0' \
        'opentelemetry-exporter-otlp-proto-grpc>=1.36.0' \
        'opentelemetry-exporter-prometheus>=0.58b0' \
        'opentelemetry-instrumentation-fastapi>=0.58b0' \
        'opentelemetry-instrumentation-logging>=0.58b0' \
        'grpcio>=1.78.1' \
        'nvidia-ml-py>=12.0' \
        'psutil>=5.9' \
        'Pillow>=10.0' && \
    uv pip install --no-deps -c /etc/arbi-lock-constraints.txt 'xgrammar>=0.2' && \
    python /workspace/arbi-serve/patch_xgrammar_optional_transformers.py --verify

# Force the cu13 CuTeDSL flavor to WIN on disk. The [cu13] extra ADDS
# nvidia-cutlass-dsl-libs-cu13 but does NOT remove nvidia-cutlass-dsl-libs-base;
# both ship a ``version_info.py`` to the SAME path, so whichever lands LAST wins.
# Reinstalling libs-cu13 LAST pins CUDA_VERSION to the Toolkit-13 build. At 4.6.1
# the cu13 wheels already read CUDA 13; this keeps that guaranteed. --no-deps:
# touch only this wheel, re-resolve nothing. The assert fails the build LOUD if
# the resolved payload is not Toolkit-13 (e.g. a stray libs-base shadow).
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install --force-reinstall --no-deps "nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION}" && \
    python -c "from cutlass.base_dsl import version_info as v; cv=v.CUDA_VERSION; assert cv.major==13, cv; print('cutlass version_info =', cv)" && \
    python -c "\
import importlib.metadata as md; \
fam={ (d.metadata['Name'] or ''): d.version for d in md.distributions() if (d.metadata['Name'] or '').startswith('nvidia-cutlass-dsl')}; \
vs=set(fam.values()); \
print('cutlass family:', fam); \
assert len(vs)==1, 'SPLIT CuTeDSL INSTALL — the frontend and the cu<N> native libs are at different versions. Compiling through that mismatch SIGSEGVs. Family: %r' % fam"

# Gated Delta Net kernels — `fla-core` provides
# the production GDN forward (chunk + fused-recurrent) used by the
# Qwen 3.5 / Qwen 3.6 hybrid models. Pure-Python wheel (no CUDA
# compile here), MIT-licensed. Install the core distribution directly:
# the `flash-linear-attention` metapackage adds Transformers, which the
# serving runtime neither imports nor needs. Add at the end so prior layers
# stay cached when only the FLA pin moves.
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install -c /etc/arbi-lock-constraints.txt 'fla-core>=0.5'

# flash-attn (FlashAttention-2) is DROPPED on the torch-2.14 stack. Source-
# building it is forbidden by project policy. It is also unnecessary: the
# DEFAULT serve path is paged_kv:bf16-turbo (FA4 CuTeDSL prefill + turbo split-K
# decode), which is FA2-free, and a measured A/B showed dropping FA2 costs nothing
# on sm_89 head_dim<=256 bf16 decode (within run-to-run noise). The exl3,
# vision-tower and embed/rerank paths were all moved off FA2; the bf16-fa /
# bf16-fa-persistent backends remain code-only and lazy-fail with a hint pointing
# at bf16-turbo. (Was: a ~30s prebuilt-wheel download here.)

# onnxscript — required by ``torch.onnx.__init__`` (it imports onnxscript
# eagerly). We never EXPORT onnx, but torch.compile's Dynamo tracer does
# ``import torch.onnx`` while building its tracing-state handler table
# (it reads ``torch.onnx.is_in_onnx_export``). Without onnxscript that
# import raises ``ModuleNotFoundError: No module named 'onnxscript'``,
# surfacing as ``InternalTorchDynamoError`` on the FIRST compiled forward
# — i.e. every served model with compile-on (the default) fails its
# first decode step. onnxscript is a small pure-Python wheel; install it
# rather than forcing every deployment onto --no-compile-on.
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install -c /etc/arbi-lock-constraints.txt onnxscript
ENV TORCH_CUDA_ARCH_LIST="${GPU_ARCH}"

# Hybrid-arch kernels (Mamba / Jamba / Qwen3-Next / LFM2 ShortConv) AND
# the EXL3 weight backend — both are FIRST-CLASS supported surfaces, so
# they install UNCONDITIONALLY (no opt-in arg). Serving a hybrid or
# EXL3-quantized model on an image that silently lacks these is a footgun:
# the runtime fails closed on multi-token Mamba prefill, LFM2/Jamba
# short-conv falls back to a slow per-row Python reference path, and EXL3
# checkpoints won't load at all. causal-conv1d + mamba-ssm are BUILT FROM SOURCE
# against the pinned Torch ABI here (the -devel builder has nvcc + the
# CUDA 13 headers); the compiled site-packages are copied into the runtime image.
#
# mamba_ssm/__init__.py is then trimmed to a version stub so it does NOT
# eagerly import ``MambaLMHeadModel`` (→ ``transformers.generation.*`` +
# ``torch._dynamo``, both stripped from slim). arbi-serve only consumes
# ``mamba_ssm.ops.{selective_scan_interface,triton.selective_state_update}``.
# The trim path is resolved via find_spec (NOT by importing mamba_ssm,
# whose un-trimmed __init__ would import transformers).
#
#   * causal-conv1d (v1.6.2.post1) / mamba-ssm (v2.3.2.post1) — BUILT FROM
#     SOURCE against torch 2.14 (CAUSAL_CONV1D_FORCE_BUILD / MAMBA_FORCE_BUILD,
#     arch pinned to ${GPU_ARCH} via the setup.py patch (hybrid-ext-wheels), bounded MAX_JOBS
#     so the compile doesn't saturate the build host). --no-deps; we add only
#     the lightweight ops deps (einops + packaging; torch brings triton) and
#     skip the heavy transformers / tilelang / tvm transitives. Source-building
#     these is permitted — only flash-attn was ever off-limits, and flash-attn
#     is now dropped entirely.
#   * exllamav3 — --no-deps weights-only trellis-kernel surface; its grammar
#     and architecture-layer dependencies are unused. Version comes from the
#     lock constraints, NEVER a literal here: a hardcoded `==` cannot be
#     reached by `uv lock`, so a pyproject/uv.lock bump silently left the image
#     behind, below the shim's EXLLAMAV3_MIN_VERSION floor, and NO EXL3
#     checkpoint could load in it. rich + marisa-trie are the two declared deps
#     we DO install and take the constraints for the same reason. The assert
#     below fails the build if any of them drifts from the lock.
# (Qwen3.5 / Qwen3.6 / Qwen3.8 GDN do NOT need causal-conv1d — they ride
# fla-core, installed above.)
# The two wheels come from their own stage (hybrid-ext-wheels, above), keyed
# on the toolchain, torch, GPU_ARCH and their build pins alone, so a uv.lock
# change elsewhere installs them instead of recompiling them.
COPY --from=hybrid-ext-wheels /wheels /tmp/hybrid-ext-wheels
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install -c /etc/arbi-lock-constraints.txt einops packaging wheel setuptools psutil && \
    uv pip install --no-deps /tmp/hybrid-ext-wheels/causal_conv1d-*.whl /tmp/hybrid-ext-wheels/mamba_ssm-*.whl && \
    rm -rf /tmp/hybrid-ext-wheels && \
    SP=$(python -c "import importlib.util as u; print(u.find_spec('mamba_ssm').submodule_search_locations[0])") && \
    printf '__version__ = "stub-for-arbi-serve"\n' > "$SP/__init__.py" && \
    EXLLAMA_NOCOMPILE=1 uv pip install --no-deps -c /etc/arbi-lock-constraints.txt -r /etc/arbi-exllamav3-req.txt && \
    uv pip install -c /etc/arbi-lock-constraints.txt rich marisa-trie

# EVERY package a version skew can make a GREEN GATE WRONG ON, asserted here
# because this is the first moment the image and the lock exist together. The
# CPU lane resolves its own venv from uv.lock and never sees an image, so
# "test in the image container" — right for GPU behaviour — does not reach it:
# the gate can be green on code the image behaves differently on, and red on
# code the image is fine with. That is not hypothetical for this set; the
# membership rule and what each one costs are in
# scripts/ci/assert_image_backend_matches_lock.py's BEHAVIOUR_CRITICAL.

# Serving has no Hugging Face model-stack dependency. xgrammar declares and
# imports Transformers for its optional from_huggingface adapter, while
# arbi-serve supplies tokenizer vocabulary + metadata directly. The image
# therefore installs xgrammar --no-deps and applies the exact-source patch
# above. Prove both halves: Transformers stayed absent and the patched xgrammar
# package really imports. An upstream source drift fails the patch step first.
RUN python -c "import importlib.util as u; assert u.find_spec('transformers') is None, 'Transformers leaked into the production runtime'; import xgrammar; print('xgrammar imports without Transformers')"

# Bake `uv` into the venv so in-container `uv pip install` works in BOTH
# the shipped slim runtime (operators add optional extras at run time) and
# the `test` stage below. Test/dev tooling itself is deliberately NOT
# installed in the builder: the runtime stage's catch-all site-packages
# COPY would sweep it into the shipped image. It lives in the `test` stage
# (FROM runtime) at the end of this file, so the slim image stays lean and
# the test/dev image is the slim image plus one thin pure-Python layer.
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install uv

# Optional kernel cache warmup. Provide --build-arg WARMUP_MODEL=...
# to bake `~/.cache/torch_extensions/` for that model's specific
# (gqa, vec_size, k_bits, v_bits) tuple. Skipped when WARMUP_MODEL is
# empty (default).
ARG WARMUP_MODEL=""
ARG WARMUP_K_BITS=4
ARG WARMUP_V_BITS=4
# Which sides serve the group-2 VQ (vq2) KV codec: auto|none|k|v|both.
# vq2 compiles a DIFFERENT kernel per side than the scalar Lloyd codec (the
# split-K .so name carries a _vqk1/_vqv1 tag), so a scalar-only bake leaves a
# vq2 deployment with no kernel to load — which under the production
# TKV_NO_JIT=1 posture is a hard boot failure, not a slow path.
#
# "auto" (default) reads each side's kind from the calibration bundle at
# TKV_CALIBRATION_FILE. That is the right answer whenever a bundle is
# readable at build time; when one is not, name the kinds explicitly. vq2
# serves widths 2, 3 and 4 only.
ARG WARMUP_VQ=auto
# Pre-baked caches land in /opt/cache-baked/, NOT /cache/, so user
# bind-mounts on /cache don't mask the pre-baked content. Two consumers:
#  * exl3 / xgrammar (torch ``cpp_extension.load``): resolved directly out
#    of /opt/cache-baked by the arbi_serve._prebake_loader short-circuit.
#  * tkv split-K decode/MTP (torch ``cpp_extension.load_inline``): NOT
#    covered by that short-circuit — it resolves under $TORCH_EXTENSIONS_DIR
#    (= /cache/torch_extensions at runtime). bootstrap._seed_prebaked_torch_
#    extensions() seeds the missing kernel dirs from here into /cache on
#    boot, so the runtime warm is a 0-JIT cache HIT.
ENV CACHE_HOME=/opt/cache-baked
RUN mkdir -p ${CACHE_HOME}/torch_extensions ${CACHE_HOME}/cute-dsl ${CACHE_HOME}/fa4-cute
# CUTE_DSL_CACHE_DIR is the production cache (tkv arbi-prefill cute_dsl_cache);
# the FLASH_ATTENTION_* pair only feeds the retired FA4 kernel, which loads in
# parity-test environments — kept so those runs stay cached too.
ENV TORCH_EXTENSIONS_DIR=${CACHE_HOME}/torch_extensions \
    CUTE_DSL_CACHE_DIR=${CACHE_HOME}/cute-dsl \
    FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED=1 \
    FLASH_ATTENTION_CUTE_DSL_CACHE_DIR=${CACHE_HOME}/fa4-cute

# xgrammar JITs apply_token_bitmask_inplace_cuda on first import via
# torch.utils.cpp_extension.load — needs ninja + g++ which the runtime
# stage drops. Bake the .so here so the runtime image hits the torch
# extension cache and skips the build path. Unconditional — without
# this, the runtime image fails on first request with
# "Ninja is required to load C++ extensions".
#
# TORCH_CUDA_ARCH_LIST is required because the build container has no
# GPU, so torch can't probe arch at runtime. Covers Turing through
# Hopper; adjust if you need older or Blackwell.
ENV TORCH_CUDA_ARCH_LIST="${GPU_ARCH}"
RUN python -c "import xgrammar"

# Triton's host module (cuda_utils: driver utilities + the generic kernel
# launcher) is otherwise compiled into TRITON_CACHE_DIR on first use and stays
# mapped, so a CRIU dump referenced a file outside the image. Bake it from this
# image's Triton against the CUDA driver stub (no GPU needed; resolves the real
# libcuda.so.1 at load); arbi_serve._triton_host_module serves it read-only.
# Model-independent. Kernel JIT caching is unchanged. Its mode comes from the
# builder's one `chmod -R a+rX ${CACHE_HOME}` after the last bake, like every
# other baked file (tests/test_baked_cache_is_world_readable.py).
COPY arbi-serve/scripts/bake_triton_host_module.py /workspace/arbi-serve/
RUN python /workspace/arbi-serve/bake_triton_host_module.py --out ${CACHE_HOME}/triton-host

# NB: the exllamav3_ext (EXL3 trellis kernel) bake needs arbi_serve installed
# (it triggers via the exl3 shim), so it lives below the source COPY at the
# end of this stage — see "EXL3 trellis kernel pre-bake" there. The xgrammar
# bake above and the tkv bakes below need NO arbi_serve source, so they stay
# here in the cached dependency block.

# ── Heavy, arbi_serve-INDEPENDENT venv post-processing — MOVED ABOVE the
#    arbi_serve source COPY so a code-only merge (the common CI trigger)
#    cache-HITS all of it. Every step below only touches torch / nvidia /
#    triton / cuda / xgrammar / tkv in the venv — never arbi_serve — so
#    ordering them here is pure cache discipline, not a behaviour change.
#    Previously they sat BELOW the source install, so a one-line code commit
#    re-ran the whole-venv strip + the ~6 GB dir `mv` AND re-EXPORTED those
#    multi-GB layers to the registry cache under zstd:19 (~20 min for a
#    code-only change — torch was cached, its DERIVED layers were not).
#    The GPU-kernel bakes genuinely need arbi_serve, so they stay in the
#    tail — but only the exl3 trellis one is keyed on the WHOLE package; see
#    the bake-inputs COPY block there. ──────────────────────────────────────

# Strip torch test trees + huge dist-info to shrink the venv copy.
RUN find /opt/venv/lib -type d \( -name 'tests' -o -name 'test' -o -name 'examples' \) \
        -prune -exec rm -rf {} + 2>/dev/null || true && \
    find /opt/venv/lib -type d -name '__pycache__' -prune -exec rm -rf {} + && \
    find /opt/venv/lib -name 'RECORD' -path '*/torch-*.dist-info/*' -delete 2>/dev/null || true

# Drop the AMD Triton backend; we run on NVIDIA only.
# Also strip the `amd` line from triton's entry_points.txt — anything
# that walks `importlib.metadata.entry_points(group='triton.backends')`
# (e.g. ``mamba_ssm.ops.triton.selective_state_update``) would otherwise
# try to import the deleted ``triton.backends.amd`` and crash.
RUN set -e; \
    SP=$(python -c 'import triton, os; print(os.path.dirname(triton.__file__))'); \
    test -n "$SP"; \
    rm -rf "$SP/backends/amd"; \
    DI=$(python -c 'import importlib.metadata as m; print(m.distribution("triton")._path)'); \
    sed -i '/^amd = triton.backends.amd$/d' "$DI/entry_points.txt"

# CUDA library prune. The cu13x torch wheel pulls ~2.6 GB of nvidia-*
# pip packages under site-packages/nvidia/ — most are eagerly opened
# at `import torch` and can't be dropped without breaking the image.
#
# DO NOT TOUCH — eagerly opened by torch._C at `import torch` (per
# `LD_DEBUG=files python -c 'import torch'` trace), or lazily dlopen'd
# in the inference hot path:
#   libcufft (278 MB)         eager NEEDED
#   libcusparse (160 MB)      eager NEEDED
#   libcusolver (150 MB)      eager NEEDED
#   libcusolverMg (115 MB)    lazy dlopen via torch.linalg.eigh path
#   libcurand, libcublas{Lt}, libnvrtc, libnvJitLink, libcudart,
#   libcudnn, libcupti, libcufile, libnvtx — all eager NEEDED
#   nvidia/cusparselt (224 MB), nvidia/nccl (187 MB),
#   nvidia/nvshmem (75 MB)   — all eager NEEDED
#
# SAFE-TO-DROP (verified via the same trace — never opened, no production
# code path lazy-dlopens them):
#   libnvperf_host   33 MB  — Nsight PerfWorks host runtime; ncu profiles
#                              from the HOST against a containerized
#                              process inject their own copy. In-container
#                              profiling would need this lib re-added.
#   libnvperf_target  6 MB  — same family, target-side
#   libnvblas        1 MB   — Fortran BLAS API; LLM stack uses cublas
#   libcufftw        1 MB   — FFTW3 bridge (libcufft is NEEDED, but
#                              libcufftw is only the FFTW3 wrapper)
#   *.a static libs  3 MB   — never dlopen'd at runtime
#
# Net saving: ~45 MB. The big nvidia-* wins are blocked by torch's
# NEEDED list — no realistic path to drop them short of source-building
# torch with USE_CUSPARSELT=0 / USE_NCCL=0 / USE_NVSHMEM=0 etc.
RUN set -e; \
    SP=$(python -c 'import torch, os; print(os.path.dirname(os.path.dirname(torch.__file__)))'); \
    test -n "$SP"; \
    if [ -d "$SP/nvidia/cu12/lib" ]; then \
        cd "$SP/nvidia/cu12/lib"; \
    elif [ -d "$SP/nvidia/cu13/lib" ]; then \
        cd "$SP/nvidia/cu13/lib"; \
    else \
        cd "$SP/nvidia"; \
    fi && \
    find . -name 'libnvperf_host.so*' -delete && \
    find . -name 'libnvperf_target.so*' -delete && \
    find . -name 'libnvblas.so*' -delete && \
    find . -name 'libcufftw.so*' -delete && \
    find . -name 'libcudadevrt.a' -delete && \
    find . -name 'libcudart_static.a' -delete

# strip --strip-unneeded every shared lib in the venv. Removes debug
# symbols + unused dynamic-export entries while keeping every external
# dlsym the runtime needs. Skip files that fail (some .so are linker
# scripts or compressed). Net: ~150-400 MB on the giant torch / triton
# blobs; verified safe across torch 2.x release wheels.
# NOT on aarch64: the image's binutils corrupt the aarch64 CUDA libraries
# torch preloads (site-packages/nvidia), and `import torch` then segfaults in
# ctypes.CDLL (torch._preload_cuda_lib). The import below fails the build on
# any platform where a strip breaks a library torch loads at import.
RUN if [ "$(uname -m)" != aarch64 ]; then \
        find /opt/venv -name '*.so' -o -name '*.so.*' 2>/dev/null \
            | xargs -r -n 1 -P "$(nproc)" strip --strip-unneeded 2>/dev/null || true; \
    fi && \
    python -X faulthandler -c "import torch; print('torch imports after the strip:', torch.__version__)"

# NB on libtorch_cuda.so: PyTorch wheels embed kernels for every (sm_70,
# sm_75, sm_80, sm_86, sm_89, sm_90, sm_100, sm_120) — about 8 GPU
# generations, 7 of 8 dead weight on a single-arch deployment. cuobjdump
# can't surgically replace fatbin sections in a .so reliably, so the
# source-build path (TORCH_CUDA_ARCH_LIST= build env) is the canonical fix;
# for prebuilt wheels we settle for the strip pass above.
#
# The runtime arch record used to be written HERE, from ${GPU_ARCH}. It is
# not any more: a record that restates the build's REQUEST is a positive
# claim nothing verified, and the step that verified it (the coverage
# assertion below) is separate, so any build that skipped it produced a
# record that could be FALSE — which is the one thing
# arbi_serve.runtime.built_arch's design cannot survive. The record is now
# DERIVED from the baked cubins, in the tail after every bake. See #2196.

# Split the big, STABLE site-packages dirs OUT of the venv so the runtime
# stage can COPY them as their own cache layers while a plain catch-all
# COPY of the remaining /opt/venv/lib does NOT re-copy them. This replaces
# a `COPY --exclude` (a dockerfile:1.7-labs feature) that silently
# regressed in newer BuildKit and duplicated ~6 GB of nvidia/torch/triton
# into the shipped image — a `mv` in the builder is robust and needs no
# labs feature. arbi_serve is deliberately NOT in this list: it is
# installed + split out in its own small mv below the source COPY, so a
# code-only change moves only its ~11 MB layer, never these multi-GB dirs.
# Runs after every builder step that imports torch/triton/xgrammar from the
# venv; the tkv steps and kernel bakes below read them via
# PYTHONPATH=/opt/venv-big, not the venv.
RUN cd /opt/venv/lib/python${PYTHON_VERSION}/site-packages && \
    mkdir -p /opt/venv-big && \
    for d in nvidia cuda torch triton xgrammar; do \
        [ -e "$d" ] && mv "$d" /opt/venv-big/ || true; \
    done

# ── turbo-attn, last ───────────────────────────────────────────────────────
# The one dependency that moves on every turbo-attn release, so it is installed
# below everything a release must not re-run: the third-party installs, the
# nvcc builds, the strip and the multi-GB `mv` above all stay cached when only
# turbo-attn's pin moves. It goes into the venv like every other package (so
# its dist-info is where pip and the lock assertion look for it) and its
# package dir joins the big dirs in /opt/venv-big below, as before. torch is
# in /opt/venv-big by now, so the tkv steps here run with PYTHONPATH on it,
# exactly as the store bake stages below always have. The test trees the venv
# strip above removes from every other package are removed from tkv here.
COPY --from=lock-export /etc/arbi-turbo-attn-req.txt /etc/arbi-turbo-attn-pin.txt /etc/
# turbo-attn (tkv codec + runtime) at the uv.lock-pinned version, from PyPI or
# from the lock's direct git reference — see the WHICHEVER note below.
# Pure-Python wheel (py3-none-any) — no compile; the per-arch CUDA kernels
# JIT at runtime (or get pre-baked below by bake_grid). --no-deps: turbo-attn's
# only base dep is torch (already pinned + installed above). We do NOT install
# turbo-attn's flash-attn extra: FA2 is retired on torch 2.14 (see the flash-attn
# note below) and the default path is FA4/turbo, so --no-deps also guarantees uv
# can't pull flash-attn from PyPI source here. Pinned via the lock constraints.
# NB: the package is "turbo-attn" (import name `tkv`); do NOT confuse with the
# unrelated upstream "turboquant" vLLM project.
# WHICHEVER the lock resolved: the direct reference when the lock has one
# (/etc/arbi-turbo-attn-req.txt), the `==` pin when it does not
# (/etc/arbi-turbo-attn-pin.txt, both from lock-export). Installing by NAME would silently take the newest PyPI release for a git-sourced
# lock — which is exactly what happened when the lock moved to a git rev: every
# image build failed the backend-match assert below with "lock pins
# 0.49.1.dev58+gcc67ca56, image carries 0.49.0".
# The direct reference resolves from a PRIVATE repo while turbo-attn's release
# lane is stalled (turbo-attn#908), so the clone needs a credential. Read from a
# BuildKit secret and applied through GIT_CONFIG_* for the length of THIS command
# only: a secret mount is not committed to a layer and the rewrite never reaches
# the image, so the token cannot be recovered from `docker history` or the
# filesystem. Same GIT_CONFIG_* form the CPU-venv CI step uses for the same repo.
# No secret is not an error -- a public source (exllamav3) and a PyPI-pinned lock
# both clone fine without one, and the install below fails loudly if it needed it.
RUN --mount=type=cache,target=/root/.cache/uv \
    --mount=type=secret,id=gh_token \
    if [ -s /run/secrets/gh_token ]; then \
        export GIT_CONFIG_COUNT=1 && \
        export GIT_CONFIG_KEY_0="url.https://x-access-token:$(cat /run/secrets/gh_token)@github.com/.insteadOf" && \
        export GIT_CONFIG_VALUE_0="https://github.com/"; \
    fi && \
    if [ -s /etc/arbi-turbo-attn-req.txt ]; then \
        uv pip install --no-deps -r /etc/arbi-turbo-attn-req.txt; \
    else \
        uv pip install --no-deps -r /etc/arbi-turbo-attn-pin.txt; \
    fi && \
    SP=/opt/venv/lib/python${PYTHON_VERSION}/site-packages && \
    find "$SP/tkv" -type d \( -name 'tests' -o -name 'test' -o -name 'examples' \) \
        -prune -exec rm -rf {} +

# The image's turbo-attn MUST equal the uv.lock pin, asserted where it is
# installed. arbi_serve.runtime.backend_version_guard raises on the same
# mismatch at engine boot, i.e. in every consumer of a published image.
# The EXL3 family MUST equal the uv.lock pins, for the same reason turbo-attn
# does: below the shim's version floor every EXL3 load is refused outright, and
# a native family split across versions ends in SIGSEGV rather than a clean
# import error. Transformers is deliberately absent from the runtime image;
# reference/parity tests install it in the test stage below. Both read uv.lock,
# so both sit here, below every layer a lock move must not invalidate.
COPY arbi-serve/uv.lock arbi-serve/scripts/ci/assert_image_backend_matches_lock.py /workspace/arbi-serve/
RUN python /workspace/arbi-serve/assert_image_backend_matches_lock.py \
        --lock /workspace/arbi-serve/uv.lock --image "(builder)" --exact \
        --from-env turbo-attn && \
    python /workspace/arbi-serve/assert_image_backend_matches_lock.py \
        --lock /workspace/arbi-serve/uv.lock --image "(builder)" --exact \
        --from-env marisa-trie --from-env rich \
        --behaviour-critical runtime


# --vq is passed ONLY when it departs from the default. turbo-attn installs
# from the uv.lock PyPI pin, and the flag postdates some pinned versions — an
# unconditional flag would break every default build against an older wheel
# for a knob that build is not using. Omitting it also keeps the default
# build's command line byte-identical to before the flag existed.
RUN if [ -n "${WARMUP_MODEL}" ]; then \
        vq_arg=""; \
        [ "${WARMUP_VQ}" = "auto" ] || vq_arg="--vq ${WARMUP_VQ}"; \
        PYTHONPATH=/opt/venv-big python -m tkv.kernels.precompile \
            --model "${WARMUP_MODEL}" \
            --k-bits ${WARMUP_K_BITS} \
            --v-bits ${WARMUP_V_BITS} \
            ${vq_arg}; \
    else \
        echo "WARMUP_MODEL not set; skipping tkv kernel pre-bake. First request will JIT tkv kernels (~30s)."; \
    fi

# ── Model-agnostic tkv decode kernel pre-bake (GPU-free) ──────────────
# Bake the EXACT split-K decode + compress-store kernels the served config
# requests at warm_tkv_decode_autotune, so the runtime warm is a 0-JIT
# cache HIT instead of minutes of boot-time nvcc (py-spy caught a 27B TP2
# boot stalled in warm_autotune_layers -> compile_decode_and_cs ->
# _compile_unified_splitk). Unlike the WARMUP_MODEL path this needs NO
# model and NO GPU: TORCH_CUDA_ARCH_LIST drives -gencode so nvcc emits
# sm_<arch> cubins device-less. The arch is known (${GPU_ARCH}) and the
# serving shapes are known, so a precise (not a giant grid) bake is right.
#
# TKV_BAKE_SHAPES is head_dim:num_kv_heads:num_q_heads in PER-RANK terms
# (bake_grid derives gqa = num_q_heads // num_kv_heads directly — no TP
# awareness), so for a TP=N deployment divide the model's total head counts
# by N. Default 256:2:12 = Qwen3.x-27B-AWQ (head_dim 256, total n_kv 4 /
# n_q 24) at TP2 → per-rank n_kv 2, n_q 12 → gqa 6 (NOT the 256:2:8 gqa-4
# single-GPU lead shape baked in turbo-attn's own image).
#
# MTP-verify / LSE-emitting decode kernels are NOT baked here: the PyPI
# bake_grid CLI exposes only --shapes / --bits (plus --vq on newer
# turbo-attn) and deliberately leaves MTP + smart-mix to lazy JIT into the persistent
# /cache/tkv volume (its docstring: "MTP falls back to lazy JIT … to keep
# the build short"). So an MTP deployment JITs the verify kernel on its
# first spec step (~30 s), then it's a warm cache hit on the mounted volume
# for every later boot. WARMUP_MODEL still pre-bakes a specific model's full
# (incl. MTP) kernel set when given. Set TKV_BAKE=0 to skip (pure lazy-JIT
# image).
ARG TKV_BAKE=1
ARG TKV_BAKE_SHAPES=256:2:12
ARG TKV_BAKE_BITS
# Codec family for the model-agnostic bake: none|k|v|both. Unlike the
# WARMUP_* bake above there is no bundle to read here (this stage runs with
# no model and no calibration mounted), so the family is stated, not derived
# — hence "none" rather than "auto". vq2 serves widths 2, 3 and 4 only, and
# the bake refuses a width outside that set rather than emitting a kernel
# whose pair index cannot address its codebook.
#
# A vq2 image wants BOTH families: this grid's scalar kernels still serve the
# bypass arm and any scalar-calibrated layer, so set TKV_BAKE_VQ=both and
# keep a second build (or a second bits list) for the scalar rung rather than
# trading one for the other.
ARG TKV_BAKE_VQ
# --vq only when non-default, for the same PyPI-pin reason as the WARMUP bake.
# --ring bakes the recent-ring build (_rg1) of every pick the decode table can
# make, because ARBI_TKV_RECENT_RING attaches a ring on every TKV boot; passed
# only when the installed bake_grid knows the flag.
# THE TKV KERNEL SHARE. tkv (from turbo-attn 0.63, tkv/kernels/_jit_share.py)
# looks every content-keyed build up in TKV_KERNEL_SHARE_DIR before compiling
# it, copies a hit into TORCH_EXTENSIONS_DIR, and publishes what it compiles.
# The share here is a BuildKit cache mount on the image builder, so a
# turbo-attn bump whose kernels did not move copies every module instead of
# recompiling it (the two tkv bakes were 1058 s + 843 s of a pin-only build).
# The key is tkv's own (generated sources, every flag but the archs, the bytes
# of every header, the toolchain), so a changed kernel is a miss and compiles
# as before. Against a tkv without the share the variable is unread.
RUN --mount=type=cache,id=arbi-tkv-kernel-share,target=/var/cache/tkv-kernel-share,sharing=locked \
    if [ "${TKV_BAKE}" = "1" ]; then \
        vq_arg=""; \
        [ "${TKV_BAKE_VQ}" = "none" ] || vq_arg="--vq ${TKV_BAKE_VQ}"; \
        ring_arg=""; if PYTHONPATH=/opt/venv-big python -m tkv.kernels.bake_grid --help 2>/dev/null | grep -q -- "--ring"; then ring_arg=--ring; fi; \
        TORCH_CUDA_ARCH_LIST="${GPU_ARCH}" PYTHONPATH=/opt/venv-big \
        TKV_KERNEL_SHARE_DIR=/var/cache/tkv-kernel-share \
        python -m tkv.kernels.bake_grid \
            --shapes "${TKV_BAKE_SHAPES}" \
            --bits "${TKV_BAKE_BITS}" \
            ${vq_arg} ${ring_arg}; \
    else \
        echo "TKV_BAKE=0 — skipping model-agnostic tkv kernel bake."; \
    fi

# tkv's package dir joins the other big dirs (its dist-info stays in the venv,
# as before).
RUN mv /opt/venv/lib/python${PYTHON_VERSION}/site-packages/tkv /opt/venv-big/

# ── bake inputs: the kernel sources, and the package surface the bakes
#    IMPORT to reach them, put in place BEFORE the whole-package COPY below.
#
# WHY. Every kernel pre-bake below is an nvcc run whose output is a function of
# its own sources, ${GPU_ARCH} and the pinned torch — and of nothing else in
# this package. Sitting after a COPY of the whole tree they were keyed on the
# whole tree instead, so a docstring edit anywhere in arbi_serve re-ran all of
# them. They are therefore given exactly what they import, and nothing else.
#
# INTO SITE-PACKAGES, at the path the installed package will occupy, NOT a tree
# of their own. ``Path(__file__).resolve().parent`` is how the vendored csrc
# dirs are found, so the source path reaches nvcc as ``-I`` and is embedded in
# the debug line tables the kernels compile with: baking from any other
# directory yields a different .so for the same sources. The install below
# overwrites every one of these files with byte-identical content (this set is
# a subset of that one), so nothing here survives it but the layer boundary.
#
# The set is a CLOSURE, not a guess, and tests/test_dockerfile_bake_inputs.py
# is what keeps it one: it materialises this very COPY list, runs each bake's
# entry point against it with ``cpp_extension.load`` stubbed, and fails when a
# module resolves from outside the list or when the recorded compile — name,
# source digests, every flag — differs from the whole-tree one. An import added
# to a bake-reachable module therefore fails on the PR gate, not thirty minutes
# into a merge-to-main image build. Widen the list and it goes green.
#
# The exl3 TRELLIS bake is deliberately NOT fed from here: reaching it runs
# ``runtime.built_arch``'s device refusal, which imports ``arbi_serve.runtime``
# — the engine core. It stays below the full COPY, where its inputs honestly
# are the whole package.
ENV SITE_PACKAGES=/opt/venv/lib/python${PYTHON_VERSION}/site-packages
# The tkv bake FIRST, and keyed on ONE file, because it needs exactly one:
# tkv_prebake.py imports nothing from ``arbi_serve`` (its own docstring says
# so, and the contract test holds it to that) and is run as a script, so the
# package surface the three bakes below need must not sit above it — a layer
# is invalidated by every COPY before it, not only by the ones it reads.
# Store-derived tkv kernel bake. The production store (staged by CI into
# arbi-serve/tkvbake/: store.json and the model's config.json) fixes which
# kernels a boot on it can request, in the rotation states and ranges the store
# and config derive. `tkv.kernels.precompile --from-store --served` compiles
# the combos the declared TKV_BITS (4, below) resolves to on that store, with
# nvcc for GPU_ARCH (no GPU needed): every deployment declares TKV_BITS=4, so
# a served boot compiles nothing. A boot at another width or budget pays its
# own compile. A tkv without --served bakes every uniform width and recorded
# verdict instead (scripts/ci/tkv_store_selectors.sh decides).
#
# THE BAKE IS ITS OWN IMAGE, reused across builds by content key. As a layer
# here it re-ran on any change above it -- a uv.lock move, a turbo-attn pin
# whose kernels did not change -- and for the production store that is hours
# of nvcc. `tkv-store-bake-key` holds the key: tkv.kernels.bake_manifest over
# the kernel sources, toolchain, binding flags, arch and the cache names the
# selector requests on the staged store, per MTP depth. scripts/build-slim.sh
# reuses arbi-serve-kernel-bake:tkv-<key> when the registry has it, builds and
# pushes it from `tkv-store-bake` when not, and passes it as
# TKV_STORE_KERNEL_BAKE. A local build leaves that at the stage name and
# bakes inline.
#
# The bake starts from an EMPTY extension dir at the path the image serves it
# from, so the bake image is the store's whole served set and nothing the
# stages above baked (build records name absolute paths, so the path is
# fixed). The builder re-runs the selector against what it received and fails
# by module name when a bake does not match this image's tkv or toolchain.
#
# An empty tkvbake/ (a local build with no store staged) skips the bake.
# TKV_STORE_BAKE_SPEC lists MTP depths to bake besides 0; each depth is its own
# verify-kernel set, so name only the depths production serves. Ahead of the
# tkv_prebake stamp below, which stamps these dirs too.
#
# `--ring` bakes the recent-ring decode builds (`_rg1`) beside the ring-off
# ones, because ARBI_TKV_RECENT_RING attaches a ring on every TKV boot and a
# ring-on decode loads those. Every --from-store call below passes the same
# selector arguments, so the key, the seed, the bake and each verify name one
# module set. Passed only when the installed precompile knows the flag, so a
# build against a turbo-attn without the ring keeps its selector unchanged.
FROM builder-deps AS tkv-store-bake-key-run
ARG GPU_ARCH
ARG TKV_STORE_BAKE_TP
ARG TKV_STORE_BAKE_SPEC
ARG TKV_STORE_BAKE_SHAPES
ARG TKV_STORE_BAKE_SHAPE_SPEC
ARG TKV_STORE_BAKE_ARCHS
RUN --mount=type=bind,source=arbi-serve/tkvbake,target=/opt/tkvbake \
    --mount=type=bind,source=arbi-serve/scripts/ci/tkv_store_selectors.sh,target=/opt/tkv-store-selectors.sh \
    if [ -f /opt/tkvbake/store.json ] && [ -f /opt/tkvbake/model/config.json ]; then \
        set -e; keys=""; \
        ring_arg=""; if PYTHONPATH=/opt/venv-big python -m tkv.kernels.precompile --help 2>/dev/null | grep -q -- "--ring"; then ring_arg=--ring; fi; \
        sh /opt/tkv-store-selectors.sh "env PYTHONPATH=/opt/venv-big python" "${ring_arg}" > /tmp/tkv-selectors.txt; \
        echo "tkv store bake selectors:"; sed 's/^/  /' /tmp/tkv-selectors.txt; \
        for arch in $(echo "${TKV_STORE_BAKE_ARCHS:-${GPU_ARCH}}" | tr ';' ' '); do while IFS= read -r sel <&3; do \
            keys="${keys}$(TORCH_CUDA_ARCH_LIST="${arch}" PYTHONPATH=/opt/venv-big \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            python -m tkv.kernels.bake_manifest key -- --from-store ${sel})-"; \
        done 3< /tmp/tkv-selectors.txt; done; \
        printf '%s' "${keys}" | sha256sum | cut -c1-24 > /tkv-store-bake.key; \
    else \
        echo "tkv store bake: no store staged in arbi-serve/tkvbake/ -- skipped"; \
        echo none > /tkv-store-bake.key; \
    fi; \
    echo "tkv store bake key: $(cat /tkv-store-bake.key)"

FROM scratch AS tkv-store-bake-key
COPY --from=tkv-store-bake-key-run /tkv-store-bake.key /tkv-store-bake.key

FROM builder-deps AS tkv-store-bake-build
ARG GPU_ARCH
ARG TKV_STORE_BAKE_TP
ARG TKV_STORE_BAKE_SPEC
ARG TKV_STORE_BAKE_SHAPES
ARG TKV_STORE_BAKE_SHAPE_SPEC
ARG TKV_STORE_BAKE_ARCHS
# Compiled STRAIGHT INTO a cache mount at the served path, so every module a
# bake finishes is kept even when the build is interrupted (the job timeout,
# a full disk) and the next attempt resumes from it; the mount starts empty of
# anything the stages above baked. A mount is not part of the layer, so the
# result is copied out for the image.
#
# SEEDED FROM THE PREVIOUS BAKE (TKV_STORE_BAKE_SEED, the image
# `arbi-serve-kernel-bake:tkv-latest` points at). tkv names each module's
# directory by its own inputs, so a module the previous bake already holds is
# copied in, not compiled, and a new key costs only the modules that changed.
# `bake_manifest seed` copies exactly the modules this store's selector plans
# and records the generation's toolchain beside them (`.tkv-toolchain`, what
# a serving stage accepts the bake by).
#
# The image is the manifest and nothing else: the finished set is seeded out
# of the cache mount into an empty directory, so an earlier key's modules that
# the mount still holds never ship, and `bake_manifest verify` runs over that
# directory BEFORE the image is pushed under its key.
#
# A tkv without `bake_manifest seed` (turbo-attn before #1257) takes the
# generation-level seed and prune of scripts/ci/tkv_bake_seed.py instead.
#
# THE WORKER COUNT IS MEASURED, NOT CAPPED. The seed only helps while the
# kernel tree is unchanged: every module's key digests the whole tkv/kernels
# include tree, so a turbo-attn release that edits a header recompiles the
# full set. tkv's default ceiling of 4 workers protects a serving host's boot.
# This stage does nothing but compile, so the ceiling is the core count and
# tkv's own caps bind: RAM from MemAvailable (cgroup-aware) and CPU from free
# cores (tkv/kernels/_parallel_bake.py). The build log's `[tkv precompile] N
# worker(s) ... [<reason>; avail= ram_cap= cpu_cap=]` line reports the choice.
COPY arbi-serve/scripts/ci/tkv_bake_seed.py /opt/tkv-bake-seed.py
RUN --mount=type=bind,source=arbi-serve/tkvbake,target=/opt/tkvbake \
    --mount=type=bind,source=arbi-serve/scripts/ci/tkv_store_selectors.sh,target=/opt/tkv-store-selectors.sh \
    --mount=type=cache,id=arbi-tkv-store-bake,target=/opt/cache-baked/torch_extensions \
    --mount=type=bind,from=tkv-store-bake-seed,target=/opt/tkv-store-bake-seed \
    mkdir -p /tkv-store-bake-out; \
    if [ -f /opt/tkvbake/store.json ] && [ -f /opt/tkvbake/model/config.json ]; then \
        set -e; t0=$(date +%s); out=/tkv-store-bake-out; seedroot=/opt/tkv-store-bake-seed/torch_extensions; \
        ring_arg=""; if PYTHONPATH=/opt/venv-big python -m tkv.kernels.precompile --help 2>/dev/null | grep -q -- "--ring"; then ring_arg=--ring; fi; \
        sh /opt/tkv-store-selectors.sh "env PYTHONPATH=/opt/venv-big python" "${ring_arg}" > /tmp/tkv-selectors.txt; \
        echo "tkv store bake selectors:"; sed 's/^/  /' /tmp/tkv-selectors.txt; \
        if PYTHONPATH=/opt/venv-big python -c 'import tkv.kernels.bake_manifest as m; assert hasattr(m, "seed")' 2>/dev/null; \
        then manifest_seed=1; else manifest_seed=0; fi; \
        if [ "${manifest_seed}" = 0 ]; then \
            gen="$(PYTHONPATH=/opt/venv-big python -c 'from tkv.kernels._jit_common import kernel_generation; print(kernel_generation())')"; \
            python /opt/tkv-bake-seed.py seed --from "${seedroot}" --into "${TORCH_EXTENSIONS_DIR}" --generation "${gen}"; \
        fi; \
        : > /tmp/tkv-served.txt; \
        for arch in $(echo "${TKV_STORE_BAKE_ARCHS:-${GPU_ARCH}}" | tr ';' ' '); do while IFS= read -r sel <&3; do \
            if [ "${manifest_seed}" = 1 ] && [ -d "${seedroot}" ]; then \
            TORCH_CUDA_ARCH_LIST="${arch}" PYTHONPATH=/opt/venv-big \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            python -m tkv.kernels.bake_manifest seed --from "${seedroot}" -- --from-store ${sel}; \
            fi; \
            if ! TORCH_CUDA_ARCH_LIST="${arch}" PYTHONPATH=/opt/venv-big \
                TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
                python -m tkv.kernels.bake_manifest verify -- --from-store ${sel} > /tmp/tkv-gaps.txt 2>&1; then \
                echo "tkv store bake: the seed lacks $(grep -c '^  ' /tmp/tkv-gaps.txt) module(s); each is COMPILED HERE (a prebuilt tkv artifact that covered them would not be):"; \
                grep '^  ' /tmp/tkv-gaps.txt | sed 's/^ */tkv store bake: COMPILING /'; \
            else \
                echo "tkv store bake: the seed covers every served module (${sel}); nothing to compile"; \
            fi; \
            TKV_BAKE_MAX_WORKERS="$(nproc)" \
            TORCH_CUDA_ARCH_LIST="${arch}" PYTHONPATH=/opt/venv-big \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            python -m tkv.kernels.precompile --from-store ${sel}; \
            if [ "${manifest_seed}" = 1 ]; then \
            TORCH_EXTENSIONS_DIR="${out}" TORCH_CUDA_ARCH_LIST="${arch}" PYTHONPATH=/opt/venv-big \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            python -m tkv.kernels.bake_manifest seed --from "${TORCH_EXTENSIONS_DIR}" -- --from-store ${sel}; \
            else \
            TORCH_CUDA_ARCH_LIST="${arch}" PYTHONPATH=/opt/venv-big \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            python -m tkv.kernels.bake_manifest names -- --from-store ${sel} >> /tmp/tkv-served.txt; \
            fi; \
        done 3< /tmp/tkv-selectors.txt; done; \
        if [ "${manifest_seed}" = 0 ]; then \
            python /opt/tkv-bake-seed.py prune --root "${TORCH_EXTENSIONS_DIR}" \
                --generation "${gen}" --names /tmp/tkv-served.txt; \
            cp -a "${TORCH_EXTENSIONS_DIR}/." "${out}/"; \
        fi; \
        for arch in $(echo "${TKV_STORE_BAKE_ARCHS:-${GPU_ARCH}}" | tr ';' ' '); do while IFS= read -r sel <&3; do \
            TORCH_EXTENSIONS_DIR="${out}" TORCH_CUDA_ARCH_LIST="${arch}" PYTHONPATH=/opt/venv-big \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            python -m tkv.kernels.bake_manifest verify -- --from-store ${sel}; \
        done 3< /tmp/tkv-selectors.txt; done; \
        echo "tkv store bake: $(find "${out}" -name '*.so' | wc -l) .so, $(du -sh "${out}" | cut -f1), $(( $(date +%s) - t0 ))s"; \
    else \
        echo "tkv store bake: no store staged in arbi-serve/tkvbake/ -- skipped"; \
    fi

FROM scratch AS tkv-store-bake
COPY --from=tkv-store-bake-build /tkv-store-bake-out /torch_extensions

FROM ${TKV_STORE_KERNEL_BAKE} AS tkv-store-kernels

# The cross-baked tkv prefill modules of an Arm image assembled under emulation
# (`tkv-prefill-xbake`, passed by build-slim.sh as a build context); empty here.
FROM scratch AS tkv-prefill-prebuilt

FROM builder-deps AS builder
ARG PYTHON_VERSION
ARG GPU_ARCH
ARG TKV_BAKE_BITS
ARG TKV_BAKE_VQ
ARG TKV_STORE_BAKE_TP
ARG TKV_STORE_BAKE_SPEC
ARG TKV_STORE_BAKE_SHAPES
ARG TKV_STORE_BAKE_SHAPE_SPEC
ARG TKV_STORE_BAKE_ARCHS
COPY --from=tkv-store-kernels /torch_extensions ${CACHE_HOME}/torch_extensions
# The store bake image carries each module's whole build directory -- its
# objects and ninja state are what let an interrupted bake resume -- but a
# served boot imports a published module by its .so alone, without asking
# ninja (tkv/kernels/_jit_publish.py, `import_published`). The intermediates
# are pruned here, as every other bake in this stage prunes its own, so the
# runtime's COPY of /opt/cache-baked ships only what a boot reads. Pruned
# BEFORE `bake_manifest verify`, so verify judges the tree that ships. Here
# and not in the bake stage: a bake is reused by its content key, so a prune
# there would reach the image only once the key next moved.
#
# The same holds for tkv's generated sources (cuda.cu, main.cpp): a module's
# key is computed from the sources the selector generates in memory, and a
# published directory is read for its .so alone (`published`). Each module's
# .so is also stripped of its static symbol table, which is most of a
# module's bytes: the device code is a small fatbin beside the host code, and
# the store bake carries one module per spec per arch. --strip-unneeded keeps
# the dynamic symbols the import resolves (PyInit_*) and the .nv_fatbin
# section, as the venv strip above relies on.
RUN find "${CACHE_HOME}/torch_extensions" \
        \( -name '*.o' -o -name 'build.ninja' -o -name '.ninja_log' -o -name '.ninja_deps' \) \
        -delete; \
    gen="${CACHE_HOME}/torch_extensions/_tkv_gen"; \
    if [ -d "${gen}" ]; then \
        set -e; \
        find "${gen}" -type f \( -name 'cuda.cu' -o -name 'main.cpp' \) -delete; \
        find "${gen}" -type f -name '*.so' -print0 \
            | xargs -0 -r -n 64 -P "$(nproc)" strip --strip-unneeded; \
        echo "tkv store bake as shipped: $(find "${gen}" -name '*.so' | wc -l) .so, $(du -sh "${gen}" | cut -f1)"; \
    fi
# Each arch the store bake verified here leaves its manifest in
# tkv-store-arch/sm<NN>, written only after that arch's verify passed. The arch
# record reads it (scripts/ci/assert_baked_arch_coverage.py): an arch with no
# verified store bake -- an added arch build-slim.sh left out -- is not claimed,
# so a boot on it is refused by name instead of compiling its kernels.
#
# ONE EXCEPTION, the Arm image assembled under emulation on an x86_64 builder
# (ARBI_NO_COMPILE=1, an arm64 target, this RUN executing as aarch64:
# scripts/ci/xbuild/assemble_arm64_image.sh): every verify there is a torch
# import under qemu, minutes each, and this store bake was already verified
# natively at the served path. build-slim.sh refuses that path unless the bake
# is the one a native verification recorded (ARBI_TKV_VERIFIED_NATIVELY names
# it), and the Arm smoke re-runs the verify natively on the image. The marks
# then list what the bake holds for each arch, which the arch record still
# checks against every module's cubins. Never on x86_64 or a native build.
ARG ARBI_NO_COMPILE=
ARG TARGETARCH
ARG ARBI_TKV_VERIFIED_NATIVELY=
RUN --mount=type=bind,source=arbi-serve/tkvbake,target=/opt/tkvbake \
    --mount=type=bind,source=arbi-serve/scripts/ci/tkv_store_selectors.sh,target=/opt/tkv-store-selectors.sh \
    if [ -f /opt/tkvbake/store.json ] && [ -f /opt/tkvbake/model/config.json ] && [ "${ARBI_NO_COMPILE:-}" = 1 ] && [ "${TARGETARCH:-}" = arm64 ] && [ "$(uname -m)" = aarch64 ] && [ -n "${ARBI_TKV_VERIFIED_NATIVELY:-}" ]; then \
        set -e; echo "tkv store bake: NOT re-verified under emulation; verified natively: ${ARBI_TKV_VERIFIED_NATIVELY}"; \
        marks="${CACHE_HOME}/torch_extensions/tkv-store-arch"; rm -rf "${marks}"; mkdir -p "${marks}"; \
        for arch in $(printf '%s' "${TKV_STORE_BAKE_ARCHS:-${GPU_ARCH}}" | tr ';' ' '); do \
            sm="sm$(echo "${arch}" | tr -d .)"; \
            (cd "${CACHE_HOME}/torch_extensions/_tkv_gen" && find . -mindepth 2 -maxdepth 2 -type d \( -name "*-${sm}" -o -name "*-noarch" \) | sed 's#^\./##') > "${marks}/${sm}"; \
            echo "  ${sm}: $(wc -l < "${marks}/${sm}") module(s) in the natively verified bake"; \
        done; \
    elif [ -f /opt/tkvbake/store.json ] && [ -f /opt/tkvbake/model/config.json ]; then \
        set -e; \
        ring_arg=""; if PYTHONPATH=/opt/venv-big python -m tkv.kernels.precompile --help 2>/dev/null | grep -q -- "--ring"; then ring_arg=--ring; fi; \
        sh /opt/tkv-store-selectors.sh "env PYTHONPATH=/opt/venv-big python" "${ring_arg}" > /tmp/tkv-selectors.txt; \
        echo "tkv store bake selectors:"; sed 's/^/  /' /tmp/tkv-selectors.txt; \
        marks="${CACHE_HOME}/torch_extensions/tkv-store-arch"; rm -rf "${marks}"; mkdir -p "${marks}"; \
        for arch in $(echo "${TKV_STORE_BAKE_ARCHS:-${GPU_ARCH}}" | tr ';' ' '); do while IFS= read -r sel <&3; do \
            TORCH_CUDA_ARCH_LIST="${arch}" PYTHONPATH=/opt/venv-big \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            python -m tkv.kernels.bake_manifest verify -- --from-store ${sel}; \
            TORCH_CUDA_ARCH_LIST="${arch}" PYTHONPATH=/opt/venv-big \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            python -m tkv.kernels.bake_manifest manifest -- --from-store ${sel} \
                >> "${marks}/sm$(echo "${arch}" | tr -d .)"; \
        done 3< /tmp/tkv-selectors.txt; done; \
    else \
        echo "tkv store bake: no store staged -- nothing to verify"; \
    fi
COPY arbi-serve/arbi_serve/tkv_prebake.py ${SITE_PACKAGES}/arbi_serve/
# tkv turbo_attn_int8 prefill kernel pre-bake + tkv source fingerprint.
# TKV_PREFILL_INT8 routes the codec prefill through a hand-written
# CUDA kernel that tkv JITs with ``cpp_extension.load_inline``. A cold build
# is nvcc, minutes long, and it holds the extension baton while it runs —
# and unlike the split-K decode grid above, no bake_grid entry reaches it.
# So it is asked for exactly the way a serving dispatch asks: through the
# launcher's own ``_module``, which owns the cache name, the tile shape and
# the -D define list. GPU-free like the other bakes (TORCH_CUDA_ARCH_LIST
# drives -gencode). Run as the SCRIPT its own ``__main__`` guard already
# makes it: the module's docstring records that it imports nothing from
# ``arbi_serve`` so the image can bake before the install, and ``python -m``
# would nonetheless execute the package ``__init__`` and key this bake on
# the whole package.
#
# The SAME step then stamps EVERY tkv extension dir under ${CACHE_HOME}
# with a digest of the tkv sources it was built from — the decode /
# MTP-verify / compress-store dirs baked earlier included. That record is
# what lets the boot seed (`cli.bootstrap._seed_prebaked_torch_extensions`)
# refuse to carry a bake across to a differently-sourced tkv: on that path
# the extension NAME is the entire cache key, and a TKV_NO_JIT=1 boot will
# not rebuild over a wrong hit. So the stamp runs even when the bake does
# not, which is the case for every turbo-attn pin that has no such kernel
# (it prints why and leaves the exit code 0).
#
# NEVER FATAL, for the same reason the exl3 int8 GEMM bake below is not: a
# failed bake costs back the JIT this removes and nothing else, and a boot
# that pays that nvcc says so on its own timeline. Failing every image build
# over it would be the larger outage. A
# failure prints differently from an absence, so the two are never confused,
# and the stamp runs either way.
#
# Widths track the model-agnostic bake above (TKV_BAKE_BITS / TKV_BAKE_VQ)
# and are resolved through tkv's own bw-combo resolver, so a deployment that
# widens one widens both. TKV_TURBO_ATTN_INT8_BAKE=0 stamps without baking.
ARG TKV_TURBO_ATTN_INT8_BAKE=1
# ARBI_NO_COMPILE=1: a build that must compile nothing (an Arm image assembled
# under emulation, scripts/ci/xbuild/assemble_arm64_image.sh). The bake then
# runs with TKV_NO_JIT=1: every module the store bake holds is used, a module
# it lacks is refused (and JITs at its first boot, named), and nothing compiles.
ARG ARBI_NO_COMPILE=
# The same tkv kernel share as the model-agnostic bake (see there).
# TKV_PREFILL_PREBUILT (build-slim.sh): the cross-baked device-only modules
# (`tkv-prefill-xbake` above), copied in first. Empty on every other build.
RUN --mount=type=cache,id=arbi-tkv-kernel-share,target=/var/cache/tkv-kernel-share,sharing=locked \
    --mount=type=bind,from=tkv-prefill-prebuilt,target=/opt/tkv-prefill-prebuilt \
    if [ -d /opt/tkv-prefill-prebuilt/_tkv_gen ]; then \
        mkdir -p "${TORCH_EXTENSIONS_DIR}/_tkv_gen" \
        && cp -an /opt/tkv-prefill-prebuilt/_tkv_gen/. "${TORCH_EXTENSIONS_DIR}/_tkv_gen/" \
        && echo "tkv prefill: $(find /opt/tkv-prefill-prebuilt/_tkv_gen -name '*.fatbin' | wc -l) cross-baked device module(s) copied in"; \
    fi; \
    TORCH_CUDA_ARCH_LIST="${GPU_ARCH}" PYTHONDONTWRITEBYTECODE=1 \
    PYTHONPATH=/opt/venv-big TKV_KERNEL_SHARE_DIR=/var/cache/tkv-kernel-share \
    env ${ARBI_NO_COMPILE:+TKV_NO_JIT=1} \
    python -c "import runpy, sys; sys.argv = ['tkv_prebake'] + sys.argv[1:]; \
runpy.run_path('${SITE_PACKAGES}/arbi_serve/tkv_prebake.py', run_name='__main__')" \
        --bits "${TKV_BAKE_BITS}" \
        --vq "${TKV_BAKE_VQ}" \
        $([ "${TKV_TURBO_ATTN_INT8_BAKE}" = "1" ] || echo --stamp-only)

# THE ARBI CUDA EXTENSIONS ARE NOT COMPILED IN THIS BUILD. AWQ-Marlin, fp4-Marlin,
# exl3 a_prep, the decode GEMV, the exl3 int8 GEMM and exllamav3_ext are each
# compiled per arch, as a versioned artifact built by its own CI job at the
# same time as the others (arbi_serve/kernel_arch_bake.py, `ext-arch-bake`
# below, ci.yml `kernel-artifacts`). This build pulls those artifacts and
# verifies them against its own tree, after the install below.

# ── arbi_serve source: copied after the bakes that do NOT depend on it, so a
#    code-only change re-runs only this short tail (self-install + the exl3
#    trellis bake + the arch record + an ~11 MB arbi_serve mv) and cache-HITS
#    the entire dependency block, the xgrammar/tkv bakes, the whole-venv
#    strip/prune/big-dir mv, AND the four nvcc bakes above. Non-editable
#    install so /workspace/arbi-serve stays throwaway after this stage. It
#    overwrites the bake-input subset already in site-packages with the same
#    bytes, and adds the rest of the package. ─────────────────────────────────
# The project metadata the self-install below reads (it left the dependency
# chain with the lock, see lock-export).
COPY arbi-serve/pyproject.toml arbi-serve/README.md /workspace/arbi-serve/
COPY arbi-serve/arbi_serve /workspace/arbi-serve/arbi_serve
# The `arbi-serve` console script is `_arbi_serve_launcher:main`, a third top-level
# package of the wheel. hatchling skips a `packages` entry whose directory is absent
# without an error, so leaving it out ships the script and none of its code
# (tests/test_dockerfile_installs_every_wheel_package.py).
COPY arbi-serve/_arbi_serve_launcher /workspace/arbi-serve/_arbi_serve_launcher
# The generated admin client is a second top-level package in the arbi-serve
# wheel. Include it in the build context so hatchling vendors it alongside
# arbi_serve; `--no-deps` is intentional because this stage already has the
# locked runtime environment.
COPY arbi-serve/client/arbi_serve_client /workspace/arbi-serve/client/arbi_serve_client
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install --no-deps /workspace/arbi-serve

# THE PER-ARCH KERNEL ARTIFACTS, PULLED AND VERIFIED -- NOT COMPILED. Each arch
# the image serves has an artifact (arbi-ext-kernels:<source>-<toolchain>-sm<NN>)
# that build-slim.sh hands this build as the named context `ext-arch-sm<NN>`,
# overriding the empty stage of that name above. `assemble` runs every
# extension's own loader once per arch with compiling FORBIDDEN
# (EXT_BAKE_NO_COMPILE): each must resolve to an artifact entry under the key
# THIS tree computes -- sources, toolchain, flags, arch -- and import. An arch
# whose artifact is missing or does not match is not shipped and is named in
# the log; the arch record below then leaves it out, so its cards are refused
# by name at boot. Only an arch in REQUIRED_GPU_ARCH fails this build: an arch
# being added can never block the image. One root per arch, under
# ${CACHE_HOME}/torch_extensions/arch/sm<NN> (arbi_serve/_ext_arch_roots.py).
ARG REQUIRED_GPU_ARCH=
COPY arbi-serve/scripts/aot_compile_triton.py arbi-serve/scripts/aot_arch_payload.py /workspace/arbi-serve/scripts/
RUN --mount=type=bind,from=ext-arch-sm80,target=/opt/ext-arch/sm80 \
    --mount=type=bind,from=ext-arch-sm86,target=/opt/ext-arch/sm86 \
    --mount=type=bind,from=ext-arch-sm89,target=/opt/ext-arch/sm89 \
    --mount=type=bind,from=ext-arch-sm90,target=/opt/ext-arch/sm90 \
    --mount=type=bind,from=ext-arch-sm100,target=/opt/ext-arch/sm100 \
    --mount=type=bind,from=ext-arch-sm120,target=/opt/ext-arch/sm120 \
    --mount=type=bind,from=ext-arch-sm121,target=/opt/ext-arch/sm121 \
    PYTHONPATH=/opt/venv-big python -m arbi_serve.kernel_arch_bake assemble \
        --archs "${GPU_ARCH}" --required "${REQUIRED_GPU_ARCH:-${GPU_ARCH}}" \
        --seeds /opt/ext-arch --prebake "${CACHE_HOME}/torch_extensions" \
    && PYTHONPATH=/opt/venv-big python /workspace/arbi-serve/scripts/aot_arch_payload.py assemble \
        --archs "${GPU_ARCH}" --required "${REQUIRED_GPU_ARCH:-${GPU_ARCH}}" \
        --seed /opt/ext-arch --out "/opt/venv/lib/python${PYTHON_VERSION}/site-packages/arbi_serve/kernels/_aot"

# ARCH COVERAGE is the assembly's above (a REQUIRED arch missing fails the
# build) and the record's below (a root marked complete whose binaries lack its
# own arch's cubin fails the build); an arch that was only requested and did
# not arrive is left out of the record, not faked into it.
COPY arbi-serve/scripts/ci/assert_baked_arch_coverage.py /tmp/assert-arch-coverage.py

# The EXL3 MoE expert-parallel dispatch hands exl3_mgemm this rank's expert
# range; the exllamav3 revision uv.lock pins is the one whose filtering path
# keeps slot positions. The registry wheel reports the SAME __version__, so
# the lock-vs-image version assertion above cannot tell the two apart -- this
# reads the capability out of the header the extension compiles from, and
# fails the BUILD rather than the first EP boot.
RUN PYTHONPATH=/opt/venv-big python -c "from arbi_serve.weight_quant.exl3.shim import assert_mgemm_expert_range_supported as a; print('mgemm expert-range slots:', a(context='image build'))"

# RUNTIME ARCH RECORD, derived from the artifact. arbi_serve.runtime.built_arch
# refuses a card whose arch this record excludes, before a weight is read, so
# the record's whole worth is that a POSITIVE entry is true. Writing it from
# ${GPU_ARCH} made it a restatement of the build's REQUEST, true only when the
# coverage assertion above had also run — a separate step, and nothing at
# runtime knew whether it had (#2196). So the record is read off the binaries
# instead, by cuobjdump: the archs EVERY fat arch-fanned bake carries, narrowed
# to the archs whose per-arch root the assembly completed and whose every .so
# carries that arch (scripts/ci/assert_baked_arch_coverage.py). No ${GPU_ARCH}
# on this command line, by design — the record cannot restate what it is never
# told.
#
# LAST, after every bake above, because it describes all of them; the
# extensions baked for an arch set other than GPU_ARCH (nvfp4-marlin,
# exl3-i8-gemm) are excluded
# inside the script, where the reason for excluding them is written down.
# FATAL: a build that cannot derive a record must not ship a silent one.
# On the emulated Arm assembly only (see the store-bake verify above), the tkv
# store-bake marks are taken as verified natively: their cubin check is a
# cuobjdump under qemu per module, thousands of them, and the Arm smoke
# re-runs this whole record natively on the image.
# TKV_LEFT_OUT_REFS (`<arch>=<tkv-kernels ref> ...`, from build-slim.sh) names the
# artifact each left-out arch lacked, beside the record, so a boot on that card
# says which artifact to publish instead of "rebuild with GPU_ARCH". Declared
# here, after every bake, so a change in it re-runs only this tail.
ARG TKV_LEFT_OUT_REFS=
RUN if [ "${ARBI_NO_COMPILE:-}" = 1 ] && [ "${TARGETARCH:-}" = arm64 ] && [ "$(uname -m)" = aarch64 ] && [ -n "${ARBI_TKV_VERIFIED_NATIVELY:-}" ]; then \
        export ARBI_TKV_MARKS_VERIFIED_NATIVELY="${ARBI_TKV_VERIFIED_NATIVELY}"; \
    fi; \
    TKV_LEFT_OUT_REFS="${TKV_LEFT_OUT_REFS}" python /tmp/assert-arch-coverage.py \
        --record /opt/venv/share/arbi-serve.gpu-arch \
        "${CACHE_HOME}/torch_extensions"

# THE BAKED CACHE IS READ-ONLY CONTENT, so every uid that can run this image
# has to be able to READ it. The bakes above run as root and one of them
# publishes a PRIVATE directory: tkv's content-addressed include snapshot is
# staged with `tempfile.mkdtemp` (0700 by construction, umask-independent) and
# renamed into place, so the leaf lands 0700 root:root beneath 0755 parents —
# a tree whose root passes a permission check while the boot still dies, in
# ninja, with "Permission denied" on a header inside it. Only a deployment
# whose /cache is already warm escapes it, because then no boot reaches for
# the bake at all; every FRESH one does.
#
# MODE, NOT OWNERSHIP. A chown to ${ARBI_UID} would fix the default account
# and break `docker run --user "$(id -u):$(id -g)"`, which
# scripts/docker-entrypoint.sh supports on purpose. `a+rX` is readable by
# whoever runs it, and `X` grants search on directories without making a
# non-executable file executable.
#
# HERE: after the last step that writes under ${CACHE_HOME}, and in the
# BUILDER — `COPY --from` carries modes, so the ship image gains no second
# copy of the ~1 GB of baked kernels.
# tests/test_baked_cache_is_world_readable.py holds both halves of this.
RUN chmod -R a+rX ${CACHE_HOME}

# Split arbi_serve (+ its dist-info) into its OWN small /opt/venv-big layer,
# mirroring the stable-dir mv above. LAST + separate so a code-only change
# re-exports only this ~11 MB layer, never the multi-GB torch/nvidia dirs.
# The dist-info is moved out too — its RECORD carries per-file hashes that
# change on every code edit, so leaving it in site-packages would bust the
# 665 MB catch-all /opt/venv/lib COPY in the runtime stage on every commit.
# The runtime stage COPYs this pkg dir's contents back into site-packages.
RUN cd /opt/venv/lib/python${PYTHON_VERSION}/site-packages && \
    mkdir -p /opt/venv-big/arbi_serve_pkg && \
    mv arbi_serve /opt/venv-big/arbi_serve_pkg/ && \
    mv arbi_serve-*.dist-info /opt/venv-big/arbi_serve_pkg/

# ── One arch's kernel artifact (arbi_serve/kernel_arch_bake.py `compile`) ─────
# Built by scripts/ci/kernel_arch_artifact.sh, one CI job per arch, each on its
# own runner at the same time, and pushed as
# arbi-ext-kernels:<source>-<toolchain>-sm<NN> (scripts/ci/kernel_artifact.py). FROM builder-deps with this commit's
# package installed exactly as the builder installs it, so the toolchain and
# the installed sources -- everything the key reads -- are the image's. Seeded
# from that arch's previous artifact (`ext-arch-seed`, a named build context),
# so only what changed compiles, one translation unit at a time.
FROM builder-deps AS kernel-src
# The project metadata the self-install below reads (it left the dependency
# chain with the lock, see lock-export).
COPY arbi-serve/pyproject.toml arbi-serve/README.md /workspace/arbi-serve/
COPY arbi-serve/arbi_serve /workspace/arbi-serve/arbi_serve
COPY arbi-serve/_arbi_serve_launcher /workspace/arbi-serve/_arbi_serve_launcher
COPY arbi-serve/client/arbi_serve_client /workspace/arbi-serve/client/arbi_serve_client
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install --no-deps /workspace/arbi-serve

COPY arbi-serve/scripts/aot_compile_triton.py arbi-serve/scripts/aot_arch_payload.py /workspace/arbi-serve/scripts/

# ── The same bake, cross-compiled for aarch64 on an x86_64 builder ───────────
# nvcc compiles the device code exactly as a native build does; the host code
# is compiled by the aarch64 cross g++ against the aarch64 torch wheel's
# headers and libraries, CUDA's SBSA target libraries and the aarch64 Python
# headers (scripts/ci/xbuild/arbi_xbuild_hooks.py says what is redirected and why
# the modules' keys are a native build's). Pinned to the builder's own
# toolchain: the torch wheel is this venv's torch version for aarch64, checked
# against the index's sha256; the CUDA cross packages are this toolkit's
# release; the Python headers are this Python package's version.
FROM kernel-src AS xbuild-arm64
ARG PYTHON_VERSION
RUN set -eu; \
    . /etc/os-release; \
    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${VERSION_ID%%.*}${VERSION_ID#*.}/cross-linux-sbsa /" \
        > /etc/apt/sources.list.d/cuda-cross-sbsa.list; \
    sed -i '/^Types:/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources; \
    printf 'Types: deb\nURIs: http://ports.ubuntu.com/ubuntu-ports\nSuites: %s %s-updates %s-security\nComponents: main universe\nArchitectures: arm64\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n' \
        "$VERSION_CODENAME" "$VERSION_CODENAME" "$VERSION_CODENAME" > /etc/apt/sources.list.d/ubuntu-ports.sources; \
    dpkg --add-architecture arm64; \
    apt-get update; \
    rel="$(nvcc --version | sed -n 's/.*release \([0-9]*\)\.\([0-9]*\).*/\1-\2/p')"; \
    pkgs="g++-aarch64-linux-gnu"; \
    for pair in "cuda-nvcc-${rel}:cuda-nvcc-cross-sbsa-${rel}" "cuda-cudart-dev-${rel}:cuda-cudart-cross-sbsa-${rel}" \
                "cuda-cccl-${rel}:cuda-cccl-cross-sbsa-${rel}" "cuda-crt-${rel}:cuda-crt-cross-sbsa-${rel}" \
                "cuda-driver-dev-${rel}:cuda-driver-cross-sbsa-${rel}" "libcublas-dev-${rel}:libcublas-cross-sbsa-${rel}"; do \
        v="$(dpkg-query -W -f '${Version}' "${pair%%:*}" 2>/dev/null || true)"; \
        pkgs="${pkgs} ${pair#*:}${v:+=${v}}"; \
    done; \
    py_dev="$(dpkg-query -W -f '${Version}' "libpython${PYTHON_VERSION}-dev")"; \
    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ${pkgs}; \
    cd /tmp && apt-get download "libpython${PYTHON_VERSION}-dev:arm64=${py_dev}"; \
    dpkg -x libpython*-dev_*_arm64.deb /tmp/pydev; \
    cp -a /tmp/pydev/usr/include/aarch64-linux-gnu /usr/include/; \
    rm -rf /tmp/pydev /tmp/*.deb /var/lib/apt/lists/*; \
    test -f "/usr/include/aarch64-linux-gnu/python${PYTHON_VERSION}/pyconfig.h"; \
    test -d /usr/local/cuda/targets/sbsa-linux/lib
RUN set -eu; \
    tv="$(PYTHONPATH=/opt/venv-big python -c 'import torch; print(torch.__version__)')"; \
    cu="${tv#*+}"; py="cp$(echo "${PYTHON_VERSION}" | tr -d .)"; \
    whl="torch-${tv%%+*}%2B${cu}-${py}-${py}-manylinux_2_28_aarch64.whl"; \
    sha="$(curl -fsSL "https://download.pytorch.org/whl/${cu}/torch/" | grep -o "${whl}#sha256=[0-9a-f]*" | head -1 | cut -d= -f2)"; \
    test -n "$sha"; \
    curl -fsSL -o /tmp/torch.whl "https://download.pytorch.org/whl/${cu}/${whl}"; \
    echo "${sha}  /tmp/torch.whl" | sha256sum -c -; \
    mkdir -p /opt/xbuild-arm64; \
    python -c "import sys, zipfile; z = zipfile.ZipFile(sys.argv[1]); [z.extract(n, sys.argv[2]) for n in z.namelist() if n.startswith(('torch/include/', 'torch/lib/'))]" \
        /tmp/torch.whl /opt/xbuild-arm64; \
    rm /tmp/torch.whl; \
    test -f /opt/xbuild-arm64/torch/lib/libtorch.so
COPY arbi-serve/scripts/ci/xbuild/arbi_xbuild_hooks.py /opt/venv/lib/python${PYTHON_VERSION}/site-packages/arbi_xbuild_hooks.py
RUN echo "import arbi_xbuild_hooks" > /opt/venv/lib/python${PYTHON_VERSION}/site-packages/arbi_xbuild.pth
ENV ARBI_XBUILD_TARGET=aarch64 \
    ARBI_XBUILD_TORCH=/opt/xbuild-arm64/torch \
    CC=aarch64-linux-gnu-gcc \
    CXX=aarch64-linux-gnu-g++ \
    NVCC_APPEND_FLAGS="-target-dir sbsa-linux"

# The cuMem allocator shim (arbi_serve/runtime/_cumem_shim.py), cross-compiled
# for an Arm image that is assembled under emulation on an x86_64 builder: its
# host C++ is the only compile the image build's own stages hold, and nothing
# compiles under qemu (scripts/ci/xbuild/assemble_arm64_image.sh). Baked under
# the key a native aarch64 build computes (cpp_ext_cache_key: torch, CUDA,
# Python, the exllamav3 and turbo-attn revisions), which `cumem-shim-bake`
# re-derives natively and checks before it ships the library.
FROM xbuild-arm64 AS cumem-shim-xbuild
# The key names exllamav3 by the rev stamp the runtime stage writes from the
# installed dist-info (below, "ASSERTED, not merely recorded"); a builder stage
# has only the pinned requirement line, which keys the same install under a
# different name. Stamped here the same way, so the runtime finds this bake.
RUN installed="$(python -c 'import glob, json; h = sorted(glob.glob("/opt/venv*/lib/python*/site-packages/exllamav3-*.dist-info/direct_url.json")); print(json.load(open(h[0]))["vcs_info"]["commit_id"] if h else "")')" \
    && test -n "$installed" && echo "$installed" > /etc/arbi-exllamav3-rev
RUN CUDA_VISIBLE_DEVICES= LIBRARY_PATH=/usr/local/cuda/targets/sbsa-linux/lib/stubs \
    PYTHONPATH=/opt/venv-big ARBI_BOOT_CACHE_DIR=/tmp/cumem-bake XDG_CACHE_HOME=/tmp/cumem-bake/xdg \
    python -c "import arbi_serve.runtime._cumem_shim as s; print('cumem shim cross-baked at', s.bake('/out/cumem_ext'))" \
    && python -c "import glob, sys; so = glob.glob('/out/cumem_ext/*/arbi_serve_cumem_ext.so'); m = open(so[0], 'rb').read(20)[18:20] if so else b''; sys.exit(0 if m == bytes([0xb7, 0]) else 'cumem shim: not one aarch64 library: %r' % so)"

FROM scratch AS cumem-shim-xbake
COPY --from=cumem-shim-xbuild /out/cumem_ext /cumem_ext

# tkv's generic int8 prefill modules (the builder's `tkv_prebake` step: the
# D128 grid at TKV_BAKE_BITS and the fresh-quantized build) for an Arm image
# assembled under emulation, which compiles nothing, so its prebake would
# otherwise leave them to a cold nvcc at a GB10/GH200/GB200's first request.
# They are DEVICE-ONLY modules (the `dev-` generation: a fatbin, no host
# code), keyed without the host, so the x86_64 nvcc builds exactly what an
# Arm builder would; only that generation is kept, so no x86_64 host object
# can reach the image. The builder copies them in before its prebake, which
# then finds every one under TKV_NO_JIT=1.
FROM kernel-src AS tkv-prefill-xbuild
ARG TKV_BAKE_BITS
ARG TKV_BAKE_VQ
ARG XBAKE_GPU_ARCH
COPY arbi-serve/arbi_serve/tkv_prebake.py /tmp/tkv_prebake.py
RUN test -n "${XBAKE_GPU_ARCH}" \
    && CUDA_VISIBLE_DEVICES= TORCH_CUDA_ARCH_LIST="${XBAKE_GPU_ARCH}" PYTHONDONTWRITEBYTECODE=1 \
       PYTHONPATH=/opt/venv-big TORCH_EXTENSIONS_DIR=/tmp/tkv-prefill TKV_KERNEL_SHARE_DIR= \
       python /tmp/tkv_prebake.py --bits "${TKV_BAKE_BITS}" --vq "${TKV_BAKE_VQ}" \
    && mkdir -p /out/_tkv_gen \
    && find /tmp/tkv-prefill/_tkv_gen -mindepth 1 -maxdepth 1 -type d -name 'dev-*' -exec cp -a {} /out/_tkv_gen/ \; \
    && find /out/_tkv_gen -name '*.o' -delete \
    && n="$(find /out/_tkv_gen -name '*.fatbin' | wc -l)" && echo "tkv prefill modules cross-baked: ${n} fatbin(s)" \
    && test "$n" -gt 0

FROM scratch AS tkv-prefill-xbake
COPY --from=tkv-prefill-xbuild /out/_tkv_gen /_tkv_gen

FROM ${EXT_BAKE_BASE} AS ext-arch-bake-run
ARG BAKE_ARCH
# A failed REQUIRED bake (compile exits 1) does not fail this step: a failed
# step keeps none of its output, and the output is the only place the modules
# that DID build exist. The exit status goes to /ext-arch/STATUS (shipped as
# /arbi-ext-kernels/STATUS), and
# scripts/ci/kernel_arch_artifact.sh publishes such an artifact only as
# <ref>-partial, the next bake's seed, and fails the job.
RUN --mount=type=bind,from=ext-arch-seed,target=/opt/ext-arch-seed \
    test -n "${BAKE_ARCH}" \
    && mkdir -p /ext-arch \
    && { PYTHONPATH=/opt/venv-big python -m arbi_serve.kernel_arch_bake compile \
            --arch "${BAKE_ARCH}" --out /ext-arch --seed /opt/ext-arch-seed/reuse \
         && echo ok > /ext-arch/STATUS \
         || echo "required-bake-failed" > /ext-arch/STATUS; } \
    && test -f /ext-arch/MANIFEST && test -f /ext-arch/report.json \
    && PYTHONPATH=/opt/venv-big python /workspace/arbi-serve/scripts/aot_arch_payload.py bake \
        --archs "${BAKE_ARCH}" --out /ext-arch/aot --seed /opt/ext-arch-seed/aot

FROM scratch AS ext-arch-bake
COPY --from=ext-arch-bake-run /ext-arch/reuse /reuse
COPY --from=ext-arch-bake-run /ext-arch/aot /aot
COPY --from=ext-arch-bake-run /ext-arch/MANIFEST /arbi-ext-kernels/MANIFEST
COPY --from=ext-arch-bake-run /ext-arch/report.json /arbi-ext-kernels/report.json
COPY --from=ext-arch-bake-run /ext-arch/STATUS /arbi-ext-kernels/STATUS

# ──────────────────────────────────────────────────────────────────
# Stage 2: runtime — CUDA *base* image (libcudart only). The PyTorch
# wheel bundles libcublas / libcudnn / libcusparse / libcufft inside
# its own torch/lib/ tree; the cuda-runtime image's copies are
# duplicates we never resolve. Net: ~800 MB - 1.5 GB cut vs the
# -runtime base image.
# ──────────────────────────────────────────────────────────────────
# The exact CUDA 13.1 package versions the builder's -devel base carries. The runtime stage installs
# the same ones, so the JIT toolchain a boot fingerprints (nvcc's `Build cuda_13.1.r13.1/compiler.N`)
# is the one the image's kernels were baked with: tkv's local generation is then the baked one and
# the baked tier is read as the process's own, not as a foreign generation. An unpinned
# `cuda-nvcc-13-1` would follow the apt repo's newest 13.1 patch and drift from the bake again.
# A package the -devel base does not carry is listed unpinned.
FROM nvidia/cuda:13.1.1-cudnn-devel-ubuntu24.04@sha256:ef8a68702fdc93f3acb12919cde4468b9df0aeea214c1c97bf57598dad558967 AS cuda-toolkit-pins
RUN for p in cuda-nvcc-13-1 cuda-cudart-dev-13-1 cuda-cccl-13-1 cuda-cuobjdump-13-1 \
             libcusparse-dev-13-1 libcublas-dev-13-1 libcusolver-dev-13-1 libcurand-dev-13-1; do \
        v=$(dpkg-query -W -f='${Version}' "$p" 2>/dev/null || true); \
        if [ -n "$v" ]; then echo "$p=$v"; else echo "$p"; fi; \
    done > /cuda-toolkit-pins.txt && cat /cuda-toolkit-pins.txt

# The ffmpeg every image decodes uploads with (arbi_serve/audio/ffmpeg_decode.py runs it as a separate program). ONE pinned,
# checksum-verified LGPL build for the runtime AND the test stages, so the format tests exercise the binary that ships.
# Only the `ffmpeg` binary and its licence text are kept. To bump: pick a BtbN/FFmpeg-Builds release that publishes
# `*-linux64-lgpl-<series>.tar.xz`, set FFMPEG_BUILD_TAG and FFMPEG_ASSET, and copy that release's line for the asset from its
# `checksums.sha256` into FFMPEG_SHA256 (and the linuxarm64 asset's line into FFMPEG_SHA256_ARM64) (tests/test_ffmpeg_pin.py checks the four agree). The dated tag is immutable;
# `latest` is not. Latest stable release: https://ffmpeg.org/download.html
FROM nvidia/cuda:13.1.1-base-ubuntu24.04@sha256:e8c8679ccd042249d4c4080a3fab5a6bb52ab6e771addffa2e6e4eafea797bd2 AS ffmpeg-pinned
ARG FFMPEG_BUILD_TAG=autobuild-2026-11-01-12-49
ARG FFMPEG_ASSET=ffmpeg-n9.0.2-42-gd7643a0fe3-linux64-lgpl-9.0
ARG FFMPEG_SHA256=9f0b46b253bc352aeb63b7829dd518f4d879e10f51bc135496b7e266ceea0bb3
# The same release's linuxarm64 asset (FFMPEG_ASSET with linux64 -> linuxarm64), for an arm64 build.
ARG FFMPEG_SHA256_ARM64=88c3d8901ca5938aac402c9c4e44b149745df554f8e2bf4ba2444179f7f46562
ARG TARGETARCH
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl xz-utils \
    && rm -rf /var/lib/apt/lists/* \
    && asset="${FFMPEG_ASSET}" && sha="${FFMPEG_SHA256}" \
    && if [ "${TARGETARCH}" = arm64 ]; then \
           asset="$(printf '%s' "${FFMPEG_ASSET}" | sed 's/-linux64-/-linuxarm64-/')"; sha="${FFMPEG_SHA256_ARM64}"; \
       fi \
    && curl -fsSL --retry 3 -o /tmp/ffmpeg.tar.xz \
        "https://github.com/BtbN/FFmpeg-Builds/releases/download/${FFMPEG_BUILD_TAG}/${asset}.tar.xz" \
    && echo "${sha}  /tmp/ffmpeg.tar.xz" | sha256sum -c - \
    && mkdir -p /out \
    && tar -xJf /tmp/ffmpeg.tar.xz -C /out --strip-components=1 \
        "${asset}/bin/ffmpeg" "${asset}/LICENSE.txt" \
    && /out/bin/ffmpeg -hide_banner -version | head -1

FROM nvidia/cuda:13.1.1-base-ubuntu24.04@sha256:e8c8679ccd042249d4c4080a3fab5a6bb52ab6e771addffa2e6e4eafea797bd2 AS runtime-toolchain

ARG PYTHON_VERSION
ENV DEBIAN_FRONTEND=noninteractive \
    PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1 \
    PATH="/opt/venv/bin:${PATH}" \
    VIRTUAL_ENV="/opt/venv"

# Runtime stage carries:
#   * python3.12      — interpreter
#   * cuda-nvcc       — needed by tkv's torch.cpp_extension.load_inline
#                        (per-arch CUDA module compile on first request
#                        for unfamiliar (head_dim, num_kv_heads) shapes
#                        the WARMUP_MODEL builder didn't pre-bake)
#   * cuda-cudart-dev — cudart.h headers nvcc needs at compile
#   * cuda-cccl       — Thrust / CUB headers nvcc needs (~150 MB)
#   * cuda-cuobjdump  — SASS disassembly of a locally compiled cubin, so an
#                        if-constexpr gate's "excluded arms produce identical
#                        code" claim can be checked INSIDE the serving image
#                        instead of via ncu's source page or a second image
#   * g++             — host compiler nvcc invokes
#   * ninja-build     — build system torch.cpp_extension uses
#
# CUDA math-library dev headers. AWQ Marlin AND exllamav3 (EXL3) both
# JIT-compile their kernels on first cold load, and every torch CUDA
# extension pulls ``ATen/cuda/CUDAContext.h`` → ``cusparse.h`` (+ cublas
# / cusolver). Without these the cold JIT dies with
# ``fatal error: cusparse.h: No such file or directory`` — so a fresh
# container can't serve any AWQ/EXL3 checkpoint. Both are first-class
# surfaces, so the headers stay in (~3.5 GB). A warmed kernel cache
# avoids the cold JIT at runtime, but the image must still be able to
# build it. The CUDA packages are the 13.1 ones at the builder's exact versions
# (cuda-toolkit-pins above): the toolkit a boot JIT-compiles with is the one the bake used.
COPY --from=cuda-toolkit-pins /cuda-toolkit-pins.txt /tmp/cuda-toolkit-pins.txt
RUN apt-get update && apt-get install -y --no-install-recommends \
        python${PYTHON_VERSION} \
        python${PYTHON_VERSION}-dev \
        $(cat /tmp/cuda-toolkit-pins.txt) \
        g++ \
        ninja-build \
        curl ca-certificates \
        iproute2 \
    && rm -rf /var/lib/apt/lists/* \
    && mkdir -p /cache/cute-dsl /cache/fa4-cute /cache/torch_extensions /cache/triton \
                /cache/hf /cache/xdg /cache/tkv \
                /cache/arbi-serve/xgrammar /cache/xgrammar

# ``/cache/arbi-serve/xgrammar`` AND ``/cache/xgrammar``: the grammar cache is
# the one managed cache with no env var of its own. It derives from the BOOT
# CACHE (``boot_cache_subdir("xgrammar")`` via ``_ARBI_CACHE_SUBDIRS``), so it
# moved when nine ``*_dir`` flags collapsed into one root — and the image was
# still creating only the OLD path, which ``ARBI_ASSERT_WARM_CACHE`` then read
# as a warm→cold regression on any host that had served structured output.
# The old path stays beside the new one so a volume written by a pre-collapse
# image still resolves: the entries are content-hash-named grammars carrying no
# build key (unlike their ``inductor-<key>`` siblings), so they remain usable
# across the move.

# Cap the CUDA-extension parallel compile at 2 jobs by default —
# nvcc + g++ from torch.cpp_extension.load fans out unbounded
# (one job per .cu translation unit) which OOMs / overloads
# small dev boxes on first model load. Operators with bigger
# build hosts can override at run time.
# CUDA_HOME is the 13.1 toolkit installed above (headers, nvcc, math-library
# dev headers), the same release as the -devel builder that bakes the image's
# kernels. With it at a 13.0 tree, a boot fingerprinted its toolchain as
# cuda_13.0 while every baked kernel recorded cuda_13.1, so tkv read the bake as
# a foreign generation and anything it JIT-built landed in a second generation.
# torch stays the cu130 wheel (no cu131 wheel exists); it runs on the 13.1
# runtime, and its CUDA build is the ABI tail of the toolchain identity, which
# is unchanged.
ENV CUDA_HOME=/usr/local/cuda-13.1 \
    PATH="/usr/local/cuda/bin:${PATH}" \
    MAX_JOBS=2

# Persistent kernel + grammar caches (mount as a docker volume).
# CUTE_DSL_CACHE_DIR = production arbi-prefill CuTeDSL JIT cache; the
# FLASH_ATTENTION_* pair only feeds the retired FA4 kernel (parity tests).
ENV CUTE_DSL_CACHE_DIR=/cache/cute-dsl \
    FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED=1 \
    FLASH_ATTENTION_CUTE_DSL_CACHE_DIR=/cache/fa4-cute \
    TORCH_EXTENSIONS_DIR=/cache/torch_extensions \
    TKV_BAKED_KERNEL_DIR=/opt/cache-baked/torch_extensions \
    TRITON_HOME=/cache/triton \
    HF_HOME=/cache/hf \
    XDG_CACHE_HOME=/cache/xdg \
    TKV_CACHE_DIR=/cache/tkv
# ^ The xgrammar cache used to be baked here too. It is not any more: every
# cache a BOOT writes derives from ONE root (arbi_serve/cache_paths.py),
# which is /cache/arbi-serve whenever /cache is writable — exactly what this
# image's volume makes it. A baked per-cache path is a second place for the
# tree to be half-configured.
# ^ TKV_CACHE_DIR is the tkv split-K autotune table dir (the ~2-3 min cold
# sweep). It MUST be baked here, not left to tkv's default. tkv defaults its
# cache to ``$XDG_CACHE_HOME/tkv`` (= /cache/xdg/tkv above) — already on the
# volume — BUT docker/compose.yaml separately sets TKV_CACHE_DIR=/cache/tkv,
# so a compose-launched server warmed /cache/tkv while a bare ``docker run``
# (benchmarks, ad-hoc) fell back to /cache/xdg/tkv. Same volume, DIFFERENT
# dir → the autotune table was written twice and the two copies diverged, so a
# bench run re-ran the autotune the compose boot had already cached. Pinning it
# here makes EVERY launch path share ONE table → autotune is paid once per
# shape, ever, across compose / docker run / serve_bench. (Compose's identical
# override is now a harmless no-op.)

# Universal serving defaults: offline HF (models live on local disk), and
# loopback gloo so torch.distributed doesn't bind a routable interface.
# NOTE: TORCHINDUCTOR_COMPILE_THREADS is intentionally NOT pinned here.
# The old value 1 ("idle-thread hygiene") serialized the cold Inductor
# compile/codegen passes (~2-4x slower one-time cache population). Inductor's
# compile workers only spawn DURING compile and go idle right after — that
# idle cost is acceptable for a much faster cold build. We honor the code
# default (8, set in arbi_serve/compile/runtime.py::configure_inductor_caches),
# which operators can still override with their own env value.
ENV HF_HUB_OFFLINE=1 \
    GLOO_SOCKET_IFNAME=lo

# CUDA caching-allocator: expandable_segments MUST be OFF — it is
# mutually exclusive with torch.cuda.MemPool (pytorch#147851), and the
# engine routes weights/KV/recurrent/arena + the serving hot path
# through ~8 cuMem-backed MemPools. ON throws "MemPool doesn't support
# expandable_segments" on any unwrapped pool use. Set at the image layer
# (before first cuda init) on BOTH the new (torch>=2.9) and legacy var
# names. Mirrors arbi_serve/__init__.py. (Was True for a since-removed
# slack-reclaim scheme; cuMem pools own VA directly so that's moot.)
ENV PYTORCH_ALLOC_CONF=expandable_segments:False \
    PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False

# ── The store bake checked at the served path, before the builder runs ──────
# The runtime stage below runs this same check (`bake_manifest accepts`, then
# `verify` per shipped arch) on the finished image, and that stays the gate.
# It runs last, though: behind the builder and its nvcc bakes, so a toolchain
# skew between the bake and the serving stage was found an hour into a build.
# This stage asks the same question from the same serving toolchain
# (`runtime-toolchain`) with the same torch and tkv at the same site-packages
# path, the builder-deps copies the runtime stage also receives, over the bake
# image itself. scripts/build-slim.sh builds it right after the bake resolves,
# so the skew fails the build in minutes. The two RUN blocks are identical,
# and tests/test_dockerfile_tkv_store_bake.py holds them so.
FROM runtime-toolchain AS tkv-served-path-verify
ARG PYTHON_VERSION
RUN mkdir -p /opt/venv/lib/python${PYTHON_VERSION}/site-packages
COPY --from=builder-deps /opt/venv-big/nvidia \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/nvidia
COPY --from=builder-deps /opt/venv-big/cuda \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/cuda
COPY --from=builder-deps /opt/venv-big/torch \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/torch
COPY --from=builder-deps /opt/venv-big/triton \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/triton
COPY --from=builder-deps /opt/venv-big/tkv \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/tkv
COPY --from=builder-deps /opt/venv/bin /opt/venv/bin
COPY --from=builder-deps /opt/venv/lib /opt/venv/lib
COPY --from=builder-deps /opt/venv/pyvenv.cfg /opt/venv/pyvenv.cfg
COPY --from=tkv-store-kernels /torch_extensions /opt/cache-baked/torch_extensions
ARG GPU_ARCH
ARG TKV_STORE_BAKE_TP
ARG TKV_STORE_BAKE_SPEC
ARG TKV_STORE_BAKE_SHAPES
ARG TKV_STORE_BAKE_SHAPE_SPEC
ARG TKV_STORE_BAKE_ARCHS
# The Arm image assembled under emulation: see the builder's store-bake verify.
ARG ARBI_NO_COMPILE=
ARG TARGETARCH
ARG ARBI_TKV_VERIFIED_NATIVELY=
RUN --mount=type=bind,source=arbi-serve/tkvbake,target=/opt/tkvbake \
    --mount=type=bind,source=arbi-serve/scripts/ci/tkv_store_selectors.sh,target=/opt/tkv-store-selectors.sh \
    --mount=type=tmpfs,target=/tmp/tkv-runtime-verify \
    if [ -f /opt/tkvbake/store.json ] && [ -f /opt/tkvbake/model/config.json ] && [ "${ARBI_NO_COMPILE:-}" = 1 ] && [ "${TARGETARCH:-}" = arm64 ] && [ "$(uname -m)" = aarch64 ] && [ -n "${ARBI_TKV_VERIFIED_NATIVELY:-}" ]; then \
        echo "tkv store bake at the served path: NOT re-verified under emulation; verified natively: ${ARBI_TKV_VERIFIED_NATIVELY}"; \
    elif [ -f /opt/tkvbake/store.json ] && [ -f /opt/tkvbake/model/config.json ]; then \
        set -e; ext=/tmp/tkv-runtime-verify/torch_extensions; mkdir -p "$ext"; \
        ring_arg=""; if /opt/venv/bin/python -m tkv.kernels.precompile --help 2>/dev/null | grep -q -- "--ring"; then ring_arg=--ring; fi; \
        sh /opt/tkv-store-selectors.sh "/opt/venv/bin/python" "${ring_arg}" > /tmp/tkv-selectors.txt; \
        TORCH_EXTENSIONS_DIR="$ext" /opt/venv/bin/python -m tkv.kernels.bake_manifest accepts; \
        /opt/venv/bin/python -c 'import os, sys; from tkv.kernels import _jit_common as jc; g = jc.kernel_generation(); r = os.path.join(os.environ["TKV_BAKED_KERNEL_DIR"], "_tkv_gen"); have = sorted(n for n in os.listdir(r) if not n.startswith("dev-")) if os.path.isdir(r) else []; ok = not have or g in have; print("tkv generation", g, "baked", have, "(own)" if ok else "FOREIGN: serving toolchain %r is not the bake toolchain" % jc._toolchain_identity()); sys.exit(0 if ok else 1)'; \
        for arch in $(echo "${TKV_STORE_BAKE_ARCHS:-${GPU_ARCH}}" | tr ';' ' '); do \
        while IFS= read -r sel <&3; do \
            echo "tkv store bake at the served path: sm ${arch}, ${sel}"; \
            TORCH_EXTENSIONS_DIR="$ext" TORCH_CUDA_ARCH_LIST="${arch}" \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            /opt/venv/bin/python -m tkv.kernels.bake_manifest verify -- --from-store ${sel}; \
        done 3< /tmp/tkv-selectors.txt; done; \
    else \
        echo "tkv store bake: no store staged -- nothing to verify at the served path"; \
    fi

# ── The runtime image, layered by how often each part changes ─────────────
# `runtime-core` is everything but arbi_serve: the venv's big dirs, the rest
# of the venv, the baked kernels, then tkv (every turbo-attn release) and the
# checks that read it. `runtime` and `test` each put the arbi_serve package on
# top of their own base as the LAST layers, so a code-only merge rebuilds a
# few MB in each and re-runs nothing below them: no venv re-copy, no
# whole-venv compileall, no store-bake verify, and no test-tooling install.
#
# Every COPY of builder output is `--link`: a link layer's cache key and blob
# are its own content, independent of the layers under it, so an unchanged
# dir is a cache hit (and the same blob in the registry) even when something
# below it moved -- a turbo-attn release re-copies tkv, not the 1.5 GB venv
# and the 2 GB baked-kernel dir above and below it.
FROM runtime-toolchain AS runtime-core
ARG PYTHON_VERSION

# LAYERED COPY — ordered from least- to most-frequently-changed so
# incremental pulls only re-download the diff. A code-only update only
# refreshes ~50-100 MB (arbi_serve layer); the ~3 GB nvidia-* + torch
# layers stay cached on the puller.
#
# Ordering rationale (estimated change frequency):
#   1. nvidia/*           — torch wheel updates (rare, weeks-months)
#   2. cuda/, cuda_*       — same cadence as nvidia
#   3. torch/             — torch wheel updates (rare)
#   4. triton/            — torch upgrade or independent triton bump
#   5. xgrammar/          — independent SDK version bump (rare)
#   6. /opt/venv/{bin,lib/python3.12/site-packages-rest, …, share}
#                         — small Python stdlib + venv plumbing (rare)
#   7. /opt/cache-baked/  — only changes when xgrammar / tkv kernel
#                            sources change OR GPU_ARCH bumps (rare)
#   8. tkv/              — turbo-attn releases (frequent — daily-weekly)
#   9. arbi_serve/        — every commit (most frequent — hourly-daily),
#                            in `runtime` / `test` below
#
# An empty /opt/venv mkdir comes first so subsequent paths land where
# torch / Python expects.
RUN mkdir -p /opt/venv/lib/python${PYTHON_VERSION}/site-packages

# Big, stable dirs first (rarely change → cached on pullers). These were
# `mv`d to /opt/venv-big in the builder's last step, so copying them here
# and the catch-all /opt/venv/lib below cannot double-store them.
COPY --link --from=builder /opt/venv-big/nvidia \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/nvidia
COPY --link --from=builder /opt/venv-big/cuda \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/cuda
COPY --link --from=builder /opt/venv-big/torch \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/torch
COPY --link --from=builder /opt/venv-big/triton \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/triton
COPY --link --from=builder /opt/venv-big/xgrammar \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/xgrammar

# Everything else in the venv — Python interpreter (bin/, lib/python3.12/
# {config-*, lib-dynload}), small site-packages (fastapi, pydantic,
# uvicorn, tokenizers, safetensors, jinja2, loguru, opentelemetry-*,
# numpy, sympy, networkx, hf_xet, etc), pyvenv.cfg, share/.
# The big dirs above (and tkv, and arbi_serve + its dist-info) were moved out
# of /opt/venv/lib in the builder, so this catch-all copies the remainder with
# NO overlap — no `--exclude` (labs feature) and no double-storage.
COPY --link --from=builder /opt/venv/bin /opt/venv/bin
COPY --link --from=builder /opt/venv/lib /opt/venv/lib
COPY --link --from=builder /opt/venv/pyvenv.cfg /opt/venv/pyvenv.cfg
COPY --link --from=builder /opt/venv/share /opt/venv/share

# The unified wheel's client package is part of the production artifact too.
# Exercise it only after the manually assembled dependency closure and all
# split site-packages layers have landed, so no transitive dependency from the
# test stage can mask an omitted base requirement.
RUN python -c "import importlib.util as u; assert u.find_spec('transformers') is None; import arbi_serve_client, xgrammar; print('runtime client + structured generation imports OK')"

# The `ui` extra's own packages this stage does not already carry as base
# deps (fastapi / uvicorn[standard] / pydantic / opentelemetry-api+sdk are
# installed above and cover most of the extra already).
#
# WHY THIS IS HERE AND NOT LEFT TO THE `ui` EXTRA ALONE: compose.serve.yaml /
# compose.dflash.yaml run the admin console from `${ARBI_IMAGE}` — this same
# `runtime`/`slim` image — deliberately, to rule out the console and the
# engine ever drifting to different sources (see that recipe's comment on why
# a separate `ARBI_CONSOLE_IMAGE` was removed). But `arbi_serve/admin_console`
# imports `openai` (its OpenAI-compat Playground client) at module scope, and
# the `slim` ship image never installed the `ui` extra pyproject.toml
# declares it under — so a default `--profile ui` bring-up of the committed
# recipe crashed the moment the console imported:
#   ModuleNotFoundError: No module named 'openai'
# Confirmed live on a `--target slim` build: the console container CrashLoops
# on this import, is unreachable, and every request the admin console would
# proxy (the Playground tab, the flag-truth panel, the A/B harness) is dead —
# discovered while deploying a PP2 co-admission + DFlash campaign, which needs
# the console for its workload/fidelity tooling.
#
# `openai` and `websockets` are per-copy of pyproject.toml's `ui` comment
# ("pulled in transitively via uvicorn[standard]... but the console imports
# it directly now, so it needs to be a real declared dependency") — the same
# reasoning extends to actually installing it, not just declaring it.
# Network retries: see hybrid-ext-wheels.
ENV UV_HTTP_TIMEOUT=120 UV_HTTP_RETRIES=10
COPY --from=builder /etc/arbi-lock-constraints.txt /etc/arbi-lock-constraints.txt
RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install -c /etc/arbi-lock-constraints.txt \
        'openai>=1.0' 'websockets>=13' pyyaml prometheus-client

# Bytecode for everything the boot imports. PYTHONDONTWRITEBYTECODE=1 above
# stops the RUNTIME from writing .pyc files but not from reading them, and a
# venv installed under it ships none — so every boot compiled every imported
# source (torch alone is over two thousand modules) before it could start.
# One layer per copy it covers, after that copy, so each is rebuilt with the
# code it covers (the venv here, tkv and arbi_serve with their own layers
# below); a stale .pyc is ignored by source mtime, never served.
RUN python -m compileall -q -j 0 /opt/venv/lib/python${PYTHON_VERSION}/site-packages

# Pre-baked extension .so files load via torch.ops.load_library through
# the arbi_serve._prebake_loader monkey-patch.
COPY --link --from=builder /opt/cache-baked /opt/cache-baked

# Put the baked exllamav3 kernel on the IMPORT PATH, not only behind the
# prebake short-circuit.
#
# exllamav3's own `ext.py` decides whether to JIT with
# `try: import exllamav3_ext / except ModuleNotFoundError: build_jit = True`,
# and it imports torch first, so a module it can import is one it will not
# compile. The `_prebake_loader` monkey-patch only redirects a caller that
# imported arbi_serve FIRST; anything else — a test probe, a fresh
# subprocess, any tool that imports exllamav3 on its own — misses it and
# runs ninja + nvcc over a dozen compilation units. Cold that is ~20
# minutes, spent silently with the calling python near-idle, which reads as
# a hang rather than a build. The CPU lane, which has no GPU and wants no
# nvcc, paid it on every run.
#
# One symlink makes the guard find the binary the image already built:
# `tests/test_exl3_fork_surface.py` goes 362s (2 of 4 probes skipped on the
# timeout) -> 23s, all 4 passing. The prebake short-circuit stays as it is;
# this only removes the reason anything else has to compile.
RUN set -eu; \
    sp="$(/opt/venv/bin/python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"; \
    n="$(find /opt/cache-baked/torch_extensions/arch -path '*/exllamav3_ext/exllamav3_ext.so' | wc -l)"; \
    if [ "$n" -eq 0 ]; then echo "no baked exllamav3_ext in any arch root - importers will JIT-compile it" >&2; exit 1; fi; \
    printf '%s\n' \
        '"""exllamav3_ext from the per-arch root that carries this card (arbi_serve._ext_arch_roots)."""' \
        'import sys' \
        'from arbi_serve._ext_arch_roots import import_baked_module' \
        'sys.modules[__name__] = import_baked_module(__name__)' \
        > "$sp/exllamav3_ext.py"; \
    echo "exllamav3_ext on the import path: $sp/exllamav3_ext.py -> $n arch root(s)"

# tkv, the package every turbo-attn release moves (its dist-info is in the
# /opt/venv/lib copy above). Above the checks that read it, below everything
# that does not.
COPY --link --from=builder /opt/venv-big/tkv \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/tkv
RUN python -m compileall -q -j 0 /opt/venv/lib/python${PYTHON_VERSION}/site-packages/tkv

# THE STORE BAKE, CHECKED WHERE IT IS SERVED. The builder verifies the bake
# with tkv at /opt/venv-big, but this stage serves tkv from site-packages and
# seeds the kernels into a different extension dir at boot. A module key that
# depended on either one would pass the builder's verify and miss every kernel
# at boot; that happened with turbo-attn 0.60.1's keys (840 of 1008 missed).
# So the check runs again here, from the runtime tkv, with an EMPTY writable
# extension dir, as a user boots with a fresh cache mounted over /cache: tkv
# reads the bake in place from TKV_BAKED_KERNEL_DIR (set above), and nothing
# else may be needed. Once per shipped arch, each alone: a boot compiles for
# its own card's arch only, so each must find a bake that covers it.
#
# The bake is compiled in the builder, whose nvcc is the -devel base's; this
# stage installs its own nvcc (cuda-nvcc-13-1 above, the builder's exact version). tkv keys a baked
# generation on the toolchain, and reads a bake by another nvcc only when its
# recorded toolchain has this stage's torch and kernel tree. `accepts` runs
# first so a skew fails this build by name (both toolchains printed), not as
# a verify listing every served module as missing. The generation check after
# it holds the stricter claim: the serving nvcc IS the bake's (same build), so
# the bake is read as this process's own generation, never as a foreign one.
ARG GPU_ARCH
ARG TKV_STORE_BAKE_TP
ARG TKV_STORE_BAKE_SPEC
ARG TKV_STORE_BAKE_SHAPES
ARG TKV_STORE_BAKE_SHAPE_SPEC
ARG TKV_STORE_BAKE_ARCHS
# The Arm image assembled under emulation: see the builder's store-bake verify.
ARG ARBI_NO_COMPILE=
ARG TARGETARCH
ARG ARBI_TKV_VERIFIED_NATIVELY=
RUN --mount=type=bind,source=arbi-serve/tkvbake,target=/opt/tkvbake \
    --mount=type=bind,source=arbi-serve/scripts/ci/tkv_store_selectors.sh,target=/opt/tkv-store-selectors.sh \
    --mount=type=tmpfs,target=/tmp/tkv-runtime-verify \
    if [ -f /opt/tkvbake/store.json ] && [ -f /opt/tkvbake/model/config.json ] && [ "${ARBI_NO_COMPILE:-}" = 1 ] && [ "${TARGETARCH:-}" = arm64 ] && [ "$(uname -m)" = aarch64 ] && [ -n "${ARBI_TKV_VERIFIED_NATIVELY:-}" ]; then \
        echo "tkv store bake at the served path: NOT re-verified under emulation; verified natively: ${ARBI_TKV_VERIFIED_NATIVELY}"; \
    elif [ -f /opt/tkvbake/store.json ] && [ -f /opt/tkvbake/model/config.json ]; then \
        set -e; ext=/tmp/tkv-runtime-verify/torch_extensions; mkdir -p "$ext"; \
        ring_arg=""; if /opt/venv/bin/python -m tkv.kernels.precompile --help 2>/dev/null | grep -q -- "--ring"; then ring_arg=--ring; fi; \
        sh /opt/tkv-store-selectors.sh "/opt/venv/bin/python" "${ring_arg}" > /tmp/tkv-selectors.txt; \
        TORCH_EXTENSIONS_DIR="$ext" /opt/venv/bin/python -m tkv.kernels.bake_manifest accepts; \
        /opt/venv/bin/python -c 'import os, sys; from tkv.kernels import _jit_common as jc; g = jc.kernel_generation(); r = os.path.join(os.environ["TKV_BAKED_KERNEL_DIR"], "_tkv_gen"); have = sorted(n for n in os.listdir(r) if not n.startswith("dev-")) if os.path.isdir(r) else []; ok = not have or g in have; print("tkv generation", g, "baked", have, "(own)" if ok else "FOREIGN: serving toolchain %r is not the bake toolchain" % jc._toolchain_identity()); sys.exit(0 if ok else 1)'; \
        for arch in $(echo "${TKV_STORE_BAKE_ARCHS:-${GPU_ARCH}}" | tr ';' ' '); do \
        while IFS= read -r sel <&3; do \
            echo "tkv store bake at the served path: sm ${arch}, ${sel}"; \
            TORCH_EXTENSIONS_DIR="$ext" TORCH_CUDA_ARCH_LIST="${arch}" \
            TKV_CALIBRATION_FILE=/opt/tkvbake/store.json TKV_BITS=4 \
            /opt/venv/bin/python -m tkv.kernels.bake_manifest verify -- --from-store ${sel}; \
        done 3< /tmp/tkv-selectors.txt; done; \
    else \
        echo "tkv store bake: no store staged -- nothing to verify at the served path"; \
    fi

# Container launcher shim. Single-GPU: exec's `python -m arbi_serve`
# (unchanged). ARBI_TP_SIZE>1: exec's `torchrun --nproc-per-node=N -m
# arbi_serve --tp-size N` so the container serves tensor-parallel through
# `docker compose up` (the server reads WORLD_SIZE/RANK from torchrun —
# a bare python launch can't do TP>1). See scripts/docker-entrypoint.sh.
COPY arbi-serve/scripts/docker-entrypoint.sh /usr/local/bin/arbi-entrypoint.sh
RUN chmod +x /usr/local/bin/arbi-entrypoint.sh

# The exllamav3 fork revision this image resolved. It is provenance the
# arbi-serve SHA cannot stand in for: the fork carries the EXL3 GEMM kernel
# map, so the pin decides which shape table (and which reconstruct surface)
# the image actually holds, and two different pins both report version
# `1.4.2` — a version-keyed staleness check is blind by construction.
#
# ASSERTED, not merely recorded: a label that can disagree with the venv is
# worse than none, because it is trusted. uv/pip write the resolved commit
# into the dist-info `direct_url.json`, which is the installed truth; the
# build fails if the declared ARG does not match it.
ARG ARBI_EXL3_REV=unknown
RUN set -eu; \
    installed="$(/opt/venv/bin/python -c 'import glob, json; h = sorted(glob.glob("/opt/venv/lib/python*/site-packages/exllamav3-*.dist-info/direct_url.json")); print(json.load(open(h[0]))["vcs_info"]["commit_id"] if h else "")')"; \
    if [ -z "$installed" ]; then \
        echo "provenance: exllamav3 has no dist-info direct_url.json - cannot record a rev" >&2; \
        exit 1; \
    fi; \
    if [ "$ARBI_EXL3_REV" != "unknown" ] && [ "$ARBI_EXL3_REV" != "$installed" ]; then \
        echo "provenance: declared ARBI_EXL3_REV=$ARBI_EXL3_REV != installed $installed" >&2; \
        exit 1; \
    fi; \
    echo "$installed" > /etc/arbi-exllamav3-rev

# ── The serving uid ───────────────────────────────────────────────────
# The engine runs as `arbi` (uid/gid 10001), NOT root.
#
# WHAT ROOT WAS ACTUALLY BUYING, MEASURED, NOT ASSUMED. Docker's default
# capability set does not include CAP_IPC_LOCK, CAP_SYS_ADMIN,
# CAP_SYS_RESOURCE or CAP_SYS_PTRACE — a uid-0 container here holds
# CapEff=0x00000000a80425fb, and inside it `mlock(64 MiB)` fails ENOMEM
# against an 8 MiB RLIMIT_MEMLOCK exactly as it does for any other uid.
# So the gigabytes this engine page-locks were never reaching the host
# through `mlock`: `cudaHostAlloc` pins through the NVIDIA driver's own
# get_user_pages path, which RLIMIT_MEMLOCK does not govern. Root bought
# no pinned bytes. It bought CAP_DAC_OVERRIDE — the ability to write a
# directory it does not own — and that is a permissions bug wearing a
# privilege costume. Correct ownership replaces it below.
#
# The device nodes need nothing either: the nvidia runtime exposes
# /dev/nvidia* as crw-rw-rw-, so any uid opens them (nvidia-caps, the
# 0600 MIG surface, is unused on these cards).
#
# A REAL user, not a bare numeric uid. `getpass.getuser()` and
# `os.path.expanduser("~")` both consult the passwd database, and torch
# names its Inductor scratch `/tmp/torchinductor_<username>` — a uid with
# no passwd entry turns those into a KeyError at import time, which reads
# as a torch bug rather than a missing account.
#
# 10001 is above the distro's dynamic range and clear of the base image's
# own `ubuntu` (1000), so a rebuild on a newer base cannot silently alias
# the serving account onto a pre-existing one.
ARG ARBI_UID=10001
ARG ARBI_GID=10001
RUN set -eu; \
    groupadd -g "${ARBI_GID}" arbi; \
    useradd -u "${ARBI_UID}" -g "${ARBI_GID}" -m -d /home/arbi -s /bin/bash arbi

# HOME is the engine's own, not root's. Every `~`-rooted cache in the
# tree (`_cpp_ext_locks`, `_compile_cache_env`, `cli/kv_resolve`,
# `runtime/_cumem_shim`, the flat-weight and shape-pin caches) resolves
# through this, so leaving it at /root would point a non-root process at
# a directory it cannot write and send each of those to its fallback —
# silently, since they all degrade rather than refuse.
ENV HOME=/home/arbi

# THE MOUNT POINTS ARE CREATED AND OWNED HERE, and that is what makes the
# non-root recipe work with no privileged step on a fresh deployment:
# Docker seeds an EMPTY named volume from the image path it is mounted
# over, and it copies the ownership and mode along with the content. So a
# first `docker compose up` on a clean host gets /cache and the two home
# caches already owned by uid 10001. (Volumes that a PRIOR root container
# already populated keep their root ownership — Docker only seeds an
# empty one — which is the one-time migration
# `scripts/migrate-cache-volumes-to-nonroot.sh` performs, and which the
# entrypoint's write probe refuses to boot without.)
#
# The request-path tiers are included by name. They are derived at runtime
# under the RUNTIME CACHE (`_blob_vault.runtime_cache_tier`, called from
# engine/active.py for `savepoints` and `prefix_tier`), which
# resolves to the boot cache's local `runtime-cache` leaf here; created 0700 with
# 0600 blobs, and a tier that cannot create its directory answers None
# and switches ITSELF OFF with a log line — a silent capability loss, not
# a crash. Pre-creating them owned means that path is never taken.
RUN set -eu; \
    mkdir -p \
        /cache/cute-dsl /cache/fa4-cute /cache/torch_extensions /cache/triton \
        /cache/hf /cache/xdg /cache/xgrammar /cache/tkv /cache/pycache \
        /cache/arbi-serve/budget-cache \
        /cache/arbi-serve/runtime-cache/savepoints \
        /cache/arbi-serve/runtime-cache/prefix_tier \
        /home/arbi/.cache/arbi-serve /home/arbi/.cache/torch_extensions; \
    chown -R "${ARBI_UID}:${ARBI_GID}" /cache /home/arbi; \
    chmod 0700 /cache/arbi-serve/runtime-cache/savepoints \
        /cache/arbi-serve/runtime-cache/prefix_tier

EXPOSE 8000
ENTRYPOINT ["/usr/local/bin/arbi-entrypoint.sh"]
CMD ["--help"]

# ── arbi_serve, last ──────────────────────────────────────────────────────
# arbi_serve + its dist-info (moved together into /opt/venv-big/arbi_serve_pkg
# in the builder). Copying the dir's CONTENTS lands both back in site-packages,
# exactly where the catch-all /opt/venv/lib COPY used to carry them — as the
# top layers of `runtime` and of `test`, so a code-only change refreshes these
# and nothing under them. The SHA stamp and the provenance LABEL change with
# every commit, so they ride here too.
FROM ${BOOT_STORE} AS boot-store

# ── The cuMem allocator shim, baked once for every image that serves ─────
# Every engine boot loads arbi_serve_cumem_ext (a host-C++ torch extension over the CUDA
# driver API). Unbaked, a fresh volume compiled it with g++/ninja at boot. Built here from the
# serving toolchain and the arbi_serve package the images ship, against the driver stub (a
# build has no driver); `runtime` and `test` copy the result. The loader imports it as built
# when its recorded source digest is this package's (arbi_serve/runtime/_cumem_shim.py).
# Empty unless a build passes the cross-baked shim (`cumem-shim-xbake`) as this
# named context: an Arm image assembled under emulation compiles nothing.
FROM scratch AS cumem-shim-prebuilt

FROM runtime-core AS cumem-shim-bake
ARG PYTHON_VERSION
USER root
COPY --link --from=builder /opt/venv-big/arbi_serve_pkg/ \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/
# A prebuilt shim ships only under the key THIS image computes for it, read
# here by this image's own Python: one baked elsewhere under another key is
# refused rather than shipped where no boot would find it.
RUN --mount=type=bind,from=cumem-shim-prebuilt,target=/opt/cumem-prebuilt \
    CUDA_VISIBLE_DEVICES= LIBRARY_PATH=/usr/local/cuda-13.1/lib64/stubs \
    ARBI_BOOT_CACHE_DIR=/tmp/cumem-bake XDG_CACHE_HOME=/tmp/cumem-bake/xdg \
    sh -euc 'if [ -d /opt/cumem-prebuilt/cumem_ext ]; then \
        key="$(python -c "from arbi_serve._ext_cache_key import cpp_ext_cache_key; print(cpp_ext_cache_key())")"; \
        test -f "/opt/cumem-prebuilt/cumem_ext/${key}/arbi_serve_cumem_ext.so" \
            || { echo "cumem shim: the prebuilt one is not under this image key ${key}: $(ls /opt/cumem-prebuilt/cumem_ext)" >&2; exit 1; }; \
        mkdir -p /opt/cache-baked/cumem_ext; cp -a "/opt/cumem-prebuilt/cumem_ext/${key}" /opt/cache-baked/cumem_ext/; \
        echo "cumem shim: prebuilt (cross-compiled) under ${key}"; \
    else \
        echo "cumem shim baked at $(python -c "import arbi_serve.runtime._cumem_shim as s; print(s.bake())")"; \
    fi' && \
    chmod -R a+rX /opt/cache-baked/cumem_ext

FROM runtime-core AS runtime
ARG PYTHON_VERSION
# compute-sanitizer, so a launch fault is diagnosable inside the serving image
# (`sanitizer=memcheck` in the Blackwell qualification refuses by name without it). It is
# installed here and not in runtime-toolchain with nvcc: a package in that stage invalidates
# every layer after it, while this one sits ahead of the arbi_serve copy and so is built once
# and reused by every later commit. It is a debugger, not a compiler, so it does not take
# part in the toolchain pin that keeps the JIT fingerprint equal to the bake's.
USER root
RUN apt-get update && apt-get install -y --no-install-recommends cuda-sanitizer-13-1 \
    && rm -rf /var/lib/apt/lists/*
# The Transcribe, voice-chat and Step-Audio2 preprocessors decode every upload that is not 16-bit PCM WAV by running
# ffmpeg as a separate program; the test stage copies the same pinned binary.
COPY --from=ffmpeg-pinned /out/bin/ffmpeg /usr/local/bin/ffmpeg
COPY --from=ffmpeg-pinned /out/LICENSE.txt /usr/share/doc/ffmpeg/LICENSE.txt
COPY --link --from=builder /opt/venv-big/arbi_serve_pkg/ \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/
RUN python -m compileall -q -j 0 /opt/venv/lib/python${PYTHON_VERSION}/site-packages/arbi_serve
COPY --link --from=cumem-shim-bake /opt/cache-baked/cumem_ext /opt/cache-baked/cumem_ext
COPY --link --from=boot-store /boot-store/ /opt/arbi-serve/boot-store/

# Stamp the git SHA this image was built from. A stale image (baked
# arbi_serve older than the repo tip) is the single most expensive bench
# footgun — see scripts/build-slim.sh (passes --build-arg ARBI_GIT_SHA)
# and the bench harness preflight, which reads this and FAILS LOUD if it
# != the repo HEAD. The engine also logs it at boot.
ARG ARBI_GIT_SHA=unknown
RUN echo "${ARBI_GIT_SHA}" > /etc/arbi-build-sha

ARG ARBI_EXL3_REV=unknown
# The archs whose tkv kernels the store bake carries (build-slim.sh: an added
# arch whose tkv-kernels artifact was not published at build time is left
# out). prod's promotion reads it from the registry, without pulling, and
# waits while a fleet arch is missing (scripts/ci/prod_fleet_archs.py).
ARG TKV_STORE_BAKE_ARCHS=
# Provenance as LABELs so `docker inspect` answers "what is in this image"
# without pulling and running it — the question a stale-image investigation
# has to answer first, and the one a registry tag alone cannot.
LABEL org.opencontainers.image.source="https://github.com/arbicity/arbi-serve" \
      org.opencontainers.image.revision="${ARBI_GIT_SHA}" \
      city.arbi.exllamav3.revision="${ARBI_EXL3_REV}" \
      city.arbi.tkv.store-archs="${TKV_STORE_BAKE_ARCHS}"

USER arbi

# ── OCR image (`--target runtime-ocr`, :ocr-<sha>) ──────
# The ship image plus the OCR models (docs/ocr.md): the locked `ocr` extra
# (OpenCV; the models are arbi-serve's own, arbi_serve/models/paddle), so `python -m arbi_serve
# ocr` boots without compiling. Like `runtime` it carries no transformers, and the import
# check below holds that. The boot-store packs come from `runtime` (BOOT_STORE, declared at the
# top), which carries the OCR config's beside the LLM's.

# The `ocr` extra's locked pins, in a stage of its own so that `lock-export`, which every
# runtime layer descends from, is unchanged by this image and keeps its cache.
FROM builder-base AS ocr-lock
COPY arbi-serve/pyproject.toml arbi-serve/uv.lock arbi-serve/README.md /workspace/arbi-serve/
RUN cd /workspace/arbi-serve && \
    uv export --frozen --extra ocr --no-hashes --no-emit-project 2>/dev/null \
        | grep -v ' @ ' | grep -v '^turbo-attn==' > /etc/arbi-ocr-req.txt && \
    ! grep -q '^transformers==' /etc/arbi-ocr-req.txt && \
    echo "ocr pins: $(grep -c '==' /etc/arbi-ocr-req.txt)"

# The OCR image's calloc, preloaded before jemalloc (scripts/packaging/ocr_calloc_memset.c says
# why), built where there is a compiler.
FROM builder-base AS ocr-calloc
COPY arbi-serve/scripts/packaging/ocr_calloc_memset.c /tmp/ocr_calloc_memset.c
RUN gcc -O2 -fno-builtin -fPIC -shared -Wall -Werror \
        -o /opt/libarbi-ocr-calloc.so /tmp/ocr_calloc_memset.c && \
    ! nm -D --undefined-only /opt/libarbi-ocr-calloc.so | grep -qw calloc

FROM runtime AS runtime-ocr
USER root
COPY --from=ocr-lock /etc/arbi-ocr-req.txt /etc/
# /usr/local/lib is on the loader's default path on every platform, so LD_PRELOAD below names both
# libraries bare and the loader finds each in its own platform's directory (x86_64-linux-gnu or
# aarch64-linux-gnu). A preload that does not resolve is only a warning from the loader, so the
# libjemalloc2 step checks that both load.
COPY --from=ocr-calloc /opt/libarbi-ocr-calloc.so /usr/local/lib/libarbi-ocr-calloc.so
# The image sets XDG_CACHE_HOME=/cache/xdg, so uv is pointed at the cache mount explicitly, and
# the import check writes its boot-cache side files to a scratch root: this stage leaves /cache,
# which a fresh volume is seeded from, as `runtime` has it.
RUN --mount=type=cache,target=/root/.cache/uv \
    UV_CACHE_DIR=/root/.cache/uv UV_LINK_MODE=copy \
    uv pip install --python /opt/venv/bin/python --no-deps -r /etc/arbi-ocr-req.txt && \
    ARBI_BOOT_CACHE_DIR=/tmp/dm-bake XDG_CACHE_HOME=/tmp/dm-bake/xdg \
    python -c "import importlib.util as u, sys, cv2, arbi_serve.ocr.server, arbi_serve.ocr.routes, arbi_serve.ocr.tasks, arbi_serve.models.paddle.pp_doclayout_v3, arbi_serve.models.paddle.ppocr_det, arbi_serve.models.paddle.ppocr_rec, arbi_serve.models.paddle.pp_lcnet, arbi_serve.models.paddle.slanext, arbi_serve.models.paddle.rt_detr; assert u.find_spec('transformers') is None and 'transformers' not in sys.modules, 'transformers in the OCR image'; print('ocr imports OK')" && \
    rm -rf /tmp/dm-bake
# The unit sets both for its engines (ocr/unit.py); an engine started by hand from this
# image gets the same. Process mode keeps the request pipelines in the API child, off the engine GIL.
ENV ARBI_USE_RADIX_PAGETABLE=0 ARBI_ENGINE_PROC=1
# Every process of the unit (its proxy, the engine, the API processes) allocates through
# jemalloc. A process returns freed memory at once while it boots (decay 0 here) and keeps it
# resident once it serves, so the multi-MB buffers of each page reuse resident memory instead of
# being faulted in again; arbi_serve/ocr/malloc_retention.py switches it, bounds what stays
# resident and says why each option is set. glibc's MALLOC_* settings (ocr/unit.py) are ignored
# under it. The calloc preloaded before it zeroes in place what jemalloc would hand back to the
# kernel to zero.
RUN apt-get update && apt-get install -y --no-install-recommends libjemalloc2 && \
    rm -rf /var/lib/apt/lists/* && ldconfig && \
    out="$(LD_PRELOAD="libarbi-ocr-calloc.so libjemalloc.so.2" /bin/true 2>&1)" && \
    if [ -n "$out" ]; then echo "OCR preload does not resolve: $out" >&2; exit 1; fi
ENV LD_PRELOAD="libarbi-ocr-calloc.so libjemalloc.so.2" \
    MALLOC_CONF=narenas:1,oversize_threshold:0,dirty_decay_ms:0,muzzy_decay_ms:0,lg_extent_max_active_fit:20
USER arbi
CMD ["ocr", "--root", "/models", "--port", "18093"]

# ── Test / dev image ──────────────────────────────────────────────────
# The slim `runtime` stage above is what we SHIP (`docker build --target
# runtime`). This `test` stage is the slim image plus the run-the-suite +
# dev tooling in one thin pure-Python layer. CI tests THIS image and ships
# `runtime`; because `test` is `FROM runtime`, every byte of the shipped
# image is exercised under test — the only difference is additive,
# server-inert test packages. Use this image for local development and to
# run the suite / benches inside the container.
#
# It is the LAST stage, so a bare `docker build` (no --target) yields this
# fuller dev/test image; pass `--target runtime` to build the slim ship
# artifact.
#
#   * pytest + pytest-asyncio (asyncio_mode=auto) + pytest-xdist (CPU lane
#     runs `-n auto`) + pytest-timeout — the test runner.
#   * httpx — FastAPI TestClient (httpx is also used by the serving stack).
#   * peft — test-only LoRA *reference oracle* (cross-checks our BGMV in
#     test_lora_adapter.py). The runtime never imports it; baked in here so
#     the test image is self-sufficient (no gate-time install). Pinned via
#     the lock constraints so it cannot drag in a different transformers.
#   * node (apt) — the admin console's JS tests EXECUTE the shipped module
#     rather than asserting over its text; without the binary they skip and a
#     green run covers none of the console. See the RUN line below.
#   * opencv-python-headless — the `ocr` extra (and the `dev` extra the CPU lane
#     installs): tests/ocr's server contract and geometry tests import cv2 and
#     skip without it, so a local run in this image would cover less than CI.
#   * openai — tests/test_admin_console.py et al. exercise the standalone
#     admin console (arbi_serve/admin_console/), whose OpenAI-compat proxy
#     path imports the official `openai` SDK directly (the `ui` extra in
#     pyproject.toml). Baked in here for the same self-sufficiency reason
#     as peft.
# Transformers is a reference oracle for tests only. It is intentionally
# installed here and absent from the runtime/slim stage.
#
# LAYERED LIKE `runtime`: the test tooling sits on `runtime-core`, and the
# arbi_serve layers `runtime` ends with are repeated on top (the same --link
# copy, so the same blob), so a code-only change does not re-run the tooling
# install below. Every layer of `runtime` is in `test`; `test` adds only the
# tooling layers between.
# Test-only extras must be constrained too: exporting just `dev` leaves
# torchaudio and the other audio/video packages free to drift off uv.lock.
# Keep this export separate so test-only pins do not invalidate runtime layers.
FROM lock-export AS test-lock
RUN cd /workspace/arbi-serve && \
    uv export --frozen --extra dev --extra audio --extra video --no-hashes --no-emit-project \
        > /tmp/arbi-test-lock-export.txt && \
    grep -v ' @ ' /tmp/arbi-test-lock-export.txt | grep -v '^turbo-attn==' > /etc/arbi-lock-constraints.txt && \
    rm /tmp/arbi-test-lock-export.txt

FROM runtime-core AS test-core
# Network retries: see hybrid-ext-wheels (inherited from runtime-core).
# `runtime` ends as uid 10001; the layers below install packages into
# /opt/venv, which that uid does not own. Root for the BUILD only — the
# stage hands the image back to `arbi` at its end, so `dev` (an alias of
# this stage) serves under the same uid as `slim`. A test image that ran
# as root would test a permission surface the ship image does not have.
USER root
# `node` — the admin console's JavaScript is TESTED by executing it.
#
# The console ships as an ES module the server serves verbatim, so the only
# way to assert what `renderConfig` (and its hundred siblings) DO, rather than
# what their source text says, is to run them. ~101 tests shell out to `node`
# for exactly that. Without the binary every one of them SKIPS, the suite
# exits 0, and a green run says nothing whatsoever about the console — a
# runtime fault there ships under a passing check. tests/_node_gate.py turns
# that silence into a failure; this line is what keeps the failure from being
# the normal state of the image.
#
# In the `test` stage ONLY: the shipped `runtime`/`slim` image never executes
# the console (a browser does), so it pays nothing. Cost here, measured on
# :test-latest: +120 MiB uncompressed on a 15.0 GiB image (+0.8 %), +30 MB
# compressed to pull. Ubuntu's `nodejs` (v18) is enough — the harness runs
# plain scripts and `--input-type=module`, no npm, no package.json, no
# registry fetch at test time.
#
# `git` — tests read the tree they run in (`git ls-files`, merge-base diffs,
# the backend version guard's work-tree probe). Without the binary those
# tests fail with FileNotFoundError, which reads as a code fault on every
# local run in this image and trains people to ignore red. Test stage only.
# A checkout is bind-mounted in owned by the host user, not uid 10001, so
# git's ownership check would refuse it; the image only ever reads trees.
RUN apt-get update && apt-get install -y --no-install-recommends \
        nodejs \
        git \
    && git config --system --add safe.directory '*' \
    && rm -rf /var/lib/apt/lists/*
# The same pinned ffmpeg the ship image carries: the format tests (tests/test_ffmpeg_decode.py) must run what ships.
COPY --from=ffmpeg-pinned /out/bin/ffmpeg /usr/local/bin/ffmpeg
# The test lock adds audio/video pins to the builder's dev constraints without
# depending on `builder`: naming it here would make every test-core build,
# including `test-overlay`, solve the whole kernel bake first.
COPY --from=test-lock /etc/arbi-lock-constraints.txt /etc/arbi-lock-constraints.txt
RUN --mount=type=cache,target=/root/.cache/uv \
    UV_LINK_MODE=copy uv pip install -c /etc/arbi-lock-constraints.txt \
        pytest pytest-asyncio pytest-xdist pytest-timeout httpx pytest-httpx docker peft openai attrs transformers \
        opencv-python-headless \
    && python -c "import cv2"
# Transformers is the reference oracle a parity test compares against, so a
# version the CPU gate did not resolve is a gate comparing against a different
# oracle. It cannot be asserted in `runtime` — it is deliberately absent there
# — so it is asserted in the stage that has it.
COPY arbi-serve/scripts/ci/assert_image_backend_matches_lock.py arbi-serve/uv.lock /tmp/lockcheck/
RUN python /tmp/lockcheck/assert_image_backend_matches_lock.py \
        --lock /tmp/lockcheck/uv.lock --image "(test)" --exact \
        --behaviour-critical test
# attrs is the runtime dependency of the vendored arbi_serve_client package.
# The `audio` extra (silero-vad, s3tokenizer, onnxruntime, torchaudio) is
# deliberately left OUT of `runtime`/slim to keep the ship image minimal —
# but this IS the comprehensive test/dev image, so it belongs here: without
# it, every audio-extra-gated test (VAD backends beyond energy, Smart Turn
# v3 EOU detection, the diarizer, s3tokenizer voice cloning) only ever
# verifies its own clean-skip path in CI, never real dependency-backed
# behavior. Same `-c` lock-pinned pattern as the line above.
RUN --mount=type=cache,target=/root/.cache/uv \
    UV_LINK_MODE=copy uv pip install -c /etc/arbi-lock-constraints.txt \
        silero-vad s3tokenizer onnxruntime torchaudio && \
    python -c "import torch, torchaudio; torchaudio.functional.resample(torch.ones(160), 16000, 8000); print('torchaudio native ops OK')"
# The `video` extra (PyAV, which bundles FFmpeg) is left out of `runtime`/slim for the same reason as
# `audio`, and belongs here for the same one: without it the container-decode path
# (multimodal/video_decode.py, tests/test_video_decode_container.py) only ever verifies its refusal.
RUN --mount=type=cache,target=/root/.cache/uv \
    UV_LINK_MODE=copy uv pip install -c /etc/arbi-lock-constraints.txt av
# The test/dev image adds packages the runtime precompile never saw.
RUN python -m compileall -q -j 0 /opt/venv/lib/python${PYTHON_VERSION}/site-packages

FROM test-core AS test
ARG PYTHON_VERSION
COPY --link --from=builder /opt/venv-big/arbi_serve_pkg/ \
    /opt/venv/lib/python${PYTHON_VERSION}/site-packages/
RUN python -m compileall -q -j 0 /opt/venv/lib/python${PYTHON_VERSION}/site-packages/arbi_serve
COPY --link --from=cumem-shim-bake /opt/cache-baked/cumem_ext /opt/cache-baked/cumem_ext
COPY --link --from=boot-store /boot-store/ /opt/arbi-serve/boot-store/

# Stamp the git SHA this image was built from. A stale image (baked
# arbi_serve older than the repo tip) is the single most expensive bench
# footgun — see scripts/build-slim.sh (passes --build-arg ARBI_GIT_SHA)
# and the bench harness preflight, which reads this and FAILS LOUD if it
# != the repo HEAD. The engine also logs it at boot.
ARG ARBI_GIT_SHA=unknown
RUN echo "${ARBI_GIT_SHA}" > /etc/arbi-build-sha

ARG ARBI_EXL3_REV=unknown
# Provenance as LABELs so `docker inspect` answers "what is in this image"
# without pulling and running it — the question a stale-image investigation
# has to answer first, and the one a registry tag alone cannot.
LABEL org.opencontainers.image.source="https://github.com/arbicity/arbi-serve" \
      org.opencontainers.image.revision="${ARBI_GIT_SHA}" \
      city.arbi.exllamav3.revision="${ARBI_EXL3_REV}"

# Back to the serving uid — see the `USER root` note at the top of `test-core`.
USER arbi

# ── CI's test image: the SHIPPED image plus the test tooling ──────────
# `docker build --target test-overlay --build-context runtime-core=docker-image://
# <the pushed slim image>@sha256:...` makes `test-core` start from the exact
# bytes of the runtime image that was just built and pushed, so the test image
# is those layers plus the tooling layers above, and nothing is compiled or
# baked a second time. The `test` stage above arrives at the same content from
# source and re-runs `builder` and `runtime-core` whenever their cache is not in
# the local builder, which put a second full build behind the runtime one; this
# does not. Built from source, `test` is still what `dev` and a local
# `docker build` use.
FROM test-core AS test-overlay
USER arbi

# ─────────────────────────────────────────────────────────────────────
# User-facing build targets — call from `docker build --target …` or compose:
#   --target slim  → the minimal production SHIP image (alias of `runtime`).
#   --target dev   → slim + test/dev tooling for local iteration (alias of
#                    `test`); compose's `dev` profile bind-mounts live source
#                    over the baked package (PYTHONPATH=/work) so edits apply
#                    without a rebuild.
# `runtime` / `test` are retained as the internal/CI stage names
# (scripts/build-slim.sh + .github/workflows/ci.yml reference them — keep in
# sync). `slim`/`dev` are pure aliases (no extra layers, no size cost).
# ─────────────────────────────────────────────────────────────────────
FROM runtime AS slim
FROM test AS dev

# ──────────────────────────────────────────────────────────────────
# console — the admin UI as a thin HTTP client
# ──────────────────────────────────────────────────────────────────
# The console talks to arbi-serve over the NETWORK and imports only the
# pure-Python parts of this package (runtime_flags, playground_workloads).
# No CUDA, no engine, no model ever loaded here: it does not belong in the
# serving image, which carries the whole GPU stack to serve a web page.
#
# arbi_serve_client is NOT optional. Every /proxy/* admin route imports it, so
# the console copies the same generated source vendored into the unified wheel.
# That keeps the thin console image aligned with the admin routes in this exact
# checkout without installing the local GPU server dependency set.
# Its Python matches PYTHON_VERSION (tests/test_supply_chain_pins.py).
FROM python:3.12-slim@sha256:f77ac9e44ae96ef2c90b8053ea08c31f8be030f824196b0ae4db6d462c84e51f AS console

WORKDIR /app

COPY arbi-serve/client/arbi_serve_client /app/arbi_serve_client
RUN pip install --no-cache-dir \
        fastapi "uvicorn[standard]" httpx websockets openai attrs \
        opentelemetry-api opentelemetry-sdk prometheus-client pydantic pyyaml \
 && rm -rf /root/.cache/pip

# The Evals tab's harness (the `evals` extra: lm-eval). Installed here rather
# than left to the operator so the committed recipe produces a console whose
# every tab works; without it the tab renders the catalogue and reports the
# install line, which is correct but is not a product.
#
# THE TRADEOFF, plainly: lm-eval depends on torch, and torch plus the dataset
# stack (datasets / pyarrow / pandas / scipy / scikit-learn) is by far the
# largest thing in this image — it dominates the ~200 MB the console would
# otherwise be. `scripts/build-slim.sh`-style size checks should expect that.
# torch comes from PyTorch's CPU wheel index because the default PyPI wheel
# bundles the CUDA runtime (several more GB of nvidia-* wheels) for a process
# that never touches a GPU: the console is an HTTP client, and the harness it
# drives scores a REMOTE endpoint, so nothing here runs a model locally.
# torch is installed FIRST so lm-eval's own resolution finds it already
# satisfied and never reaches for the CUDA build.
RUN pip install --no-cache-dir --index-url https://download.pytorch.org/whl/cpu torch \
 && pip install --no-cache-dir lm-eval \
 && rm -rf /root/.cache/pip

COPY arbi-serve/arbi_serve /app/arbi_serve

# Same non-root account, SAME uid as the serving image — not merely
# "also not root". The console shares the `arbi-serve-compile-cache`
# volume with the engine (it is where its server registry and its
# self-signed key land, the key written 0600), so a console on a
# different uid could not read back what the engine's uid wrote, and
# vice versa. Port 8899 is unprivileged, so nothing here wants uid 0.
ARG ARBI_UID=10001
ARG ARBI_GID=10001
RUN set -eu; \
    groupadd -g "${ARBI_GID}" arbi; \
    useradd -u "${ARBI_UID}" -g "${ARBI_GID}" -m -d /home/arbi -s /bin/bash arbi; \
    mkdir -p /cache; \
    chown -R "${ARBI_UID}:${ARBI_GID}" /cache /home/arbi /app
ENV HOME=/home/arbi
USER arbi

EXPOSE 8899

# --host 0.0.0.0: the console's own default is 127.0.0.1, unreachable from
# outside the container. Add --self-signed-tls to serve HTTPS (the Audio
# tab's mic capture needs a secure context).
#
# NO --engine and NO --admin-token-file here: a console that runs beside an
# engine is told BOTH by its deployment (the committed recipes pass them), and
# a default for either would be a default that is right on one deployment and
# silently wrong on the next -- an address that connects to nothing, or a
# credential file that is not the engine's. Unset, the console dials loopback
# with no credential, which is correct for a console on the engine's own host
# and is the state a `pip install arbi-serve[ui]` console starts in.
ENTRYPOINT ["python", "-m", "arbi_serve.admin_console", "--host", "0.0.0.0", "--port", "8899"]
