"""Diagnostic-only instrumentation for the seed-step stall (never committed)."""
import sys, pathlib
root = pathlib.Path(sys.argv[1])
p = root / "arbi_serve/engine/step_phase_probe.py"
s = p.read_text()
a = '''    now = time.perf_counter_ns()
    if _last_name is not None:
        delta = now - _last_ns
'''
b = '''    now = time.perf_counter_ns()
    _dx_mark(name, now)
    if _last_name is not None:
        delta = now - _last_ns
        _tt = time.thread_time_ns()
        _tid = _threading.get_ident()
        _prev = _DIAG_TT.get(_tid, _tt)
        _DIAG_TT[_tid] = _tt
        _DIAG_STEP.append((name, delta, _tt - _prev))
'''
assert s.count(a) == 1; s = s.replace(a, b)
a = '''    if name == "step_begin":
        if _cur:'''
b = '''    if name == "step_begin":
        tot = sum(d for _, d, _c in _DIAG_STEP)
        if tot > 50_000_000:
            logger.warning(
                "DIAGSTEP t=%.3f total=%.1fms %s | %s",
                time.time(),
                tot / 1e6,
                " ".join(f"{n}={d / 1e6:.1f}/{c / 1e6:.1f}" for n, d, c in _DIAG_STEP),
                _dx_report(now),
            )
        _DIAG_STEP.clear()
        _DX["step_t0"] = now
        if _cur:'''
assert s.count(a) == 1; s = s.replace(a, b)
a = '''_cur: list[str] = []'''
b = '''_cur: list[str] = []
_DIAG_STEP: list = []
_DIAG_TT: dict = {}
import gc as _gc  # noqa: E402
import os as _os  # noqa: E402
import threading as _threading  # noqa: E402

_DX: dict = {"gc": [], "gc_start": {}, "step_t0": 0, "ev": {}, "clk": None, "proc0": None, "alloc0": None}


def _dx_gc(phase, info):
    now = time.perf_counter_ns()
    tid = _threading.get_ident()
    if phase == "start":
        _DX["gc_start"][tid] = now
        return
    t0 = _DX["gc_start"].pop(tid, None)
    if t0 is None:
        return
    _DX["gc"].append((t0, now, int(info.get("generation", -1)), _threading.current_thread().name, int(info.get("collected", 0))))
    if len(_DX["gc"]) > 4096:
        del _DX["gc"][:2048]


_gc.callbacks.append(_dx_gc)


def _dx_proc_cpu():
    try:
        with open("/proc/self/stat") as f:
            parts = f.read().rsplit(")", 1)[1].split()
        return (int(parts[11]) + int(parts[12])) * 1000.0 / _os.sysconf("SC_CLK_TCK")
    except Exception:  # noqa: BLE001
        return 0.0


def _dx_alloc():
    try:
        import torch
        st = torch.cuda.memory_stats()
        return (st.get("num_device_alloc", 0), st.get("num_device_free", 0), st.get("num_alloc_retries", 0), st.get("num_sync_all_streams", 0))
    except Exception:  # noqa: BLE001
        return (0, 0, 0, 0)


_NVML = [None]


def _dx_clock():
    try:
        import pynvml
        if _NVML[0] is None:
            pynvml.nvmlInit()
            idx = int((_os.environ.get("CUDA_VISIBLE_DEVICES") or "0").split(",")[0]) if (_os.environ.get("CUDA_VISIBLE_DEVICES") or "0").split(",")[0].isdigit() else 0
            _NVML[0] = pynvml.nvmlDeviceGetHandleByIndex(0 if _os.environ.get("CUDA_VISIBLE_DEVICES") else idx)
        h = _NVML[0]
        return (pynvml.nvmlDeviceGetClockInfo(h, pynvml.NVML_CLOCK_SM), pynvml.nvmlDeviceGetPerformanceState(h))
    except Exception as e:  # noqa: BLE001
        return (repr(e)[:40], -1)


def _dx_event(key):
    try:
        import torch
        ev = torch.cuda.Event(enable_timing=True)
        ev.record()
        _DX["ev"][key] = ev
    except Exception:  # noqa: BLE001
        pass


def _dx_mark(name, now):
    if name == "seed_batch_built":
        _DX["ev"] = {}
        _DX["clk"] = _dx_clock()
        _DX["proc0"] = _dx_proc_cpu()
        _DX["alloc0"] = _dx_alloc()
        _DX["gccount0"] = _gc.get_count()
        _DX["fwd_t0"] = now
        _dx_event("a")
    elif name in ("dseed_fwd_launched", "dseed_sampled", "seed_device_drained"):
        _dx_event(name)


def _dx_report(now):
    t0 = _DX["step_t0"]
    gcs = [g for g in _DX["gc"] if g[1] >= t0 and g[0] <= now]
    gtxt = ",".join(f"g{g[2]}@{(g[0] - t0) / 1e6:.1f}+{(g[1] - g[0]) / 1e6:.1f}ms[{g[3][:14]}]" for g in gcs) or "none"
    gpu = ""
    ev = _DX.get("ev") or {}
    try:
        if "a" in ev:
            for k in ("dseed_fwd_launched", "dseed_sampled", "seed_device_drained"):
                if k in ev:
                    ev[k].synchronize()
                    gpu += f" gpu_a->{k}={ev['a'].elapsed_time(ev[k]):.1f}"
    except Exception as e:  # noqa: BLE001
        gpu += f" gpu_err={e!r}"[:60]
    extra = ""
    if _DX.get("proc0") is not None:
        a1 = _dx_alloc()
        a0 = _DX["alloc0"]
        extra = (f" proc_cpu={_dx_proc_cpu() - _DX['proc0']:.0f}ms clk={_DX['clk']} gccount0={_DX.get('gccount0')}"
                 f" alloc_d={[x - y for x, y in zip(a1, a0)]} since_fwd={(now - _DX['fwd_t0']) / 1e6:.1f}ms")
        _DX["proc0"] = None
    stats = _gc.get_stats()
    return f"gc={gtxt}{gpu}{extra} gcstats={[s['collections'] for s in stats]} frozen={_gc.get_freeze_count()}"'''
assert s.count(a) == 1; s = s.replace(a, b, 1)
p.write_text(s)

p = root / "arbi_serve/engine/run_step/__init__.py"
s = p.read_text()
a = '''    return _mtp_seed_post_forward(eng, slate, batch, mtp_opted_rows, logits, hidden_last)'''
b = '''    from arbi_serve.engine.step_phase_probe import mark as _dmark

    _dmark("dseed_fwd_launched")
    return _mtp_seed_post_forward(eng, slate, batch, mtp_opted_rows, logits, hidden_last)'''
assert s.count(a) == 1; s = s.replace(a, b)
a = '''        sampled = eng.model_runner._sample(slate, logits)
'''
b = '''        sampled = eng.model_runner._sample(slate, logits)
        from arbi_serve.engine.step_phase_probe import mark as _dmark

        _dmark("dseed_sampled")
'''
assert s.count(a) == 1; s = s.replace(a, b)
p.write_text(s)
print("patched")
