# oracle dump: pie::metal::pipeline::emit_grouped_fused_region_msl
# source of truth: compiler/codegen/src/metal/ (the C++ oracle this was dumped from is deleted)
# emitter_version: 33
# @runtime: bytes=39611 fnv1a64=0x310a2f7d72f22fd5
# @grouped: bytes=1043 fnv1a64=0x429904221e8e963f
# @grouped was bytes=970 fnv1a64=0xa8ab9e1ebf8714d9 through emitter 41. The lane
#   record grew `attn_score_base` and `attn_score_row_stride` (+73 bytes of struct
#   text, no field before them moved), which is the grouped form's route to the
#   attention-score rectangle: that form binds no per-intrinsic buffer, so a
#   rectangle it must reach arrives as an address on the record. Unlike @runtime,
#   this prefix is not kernel-side text an `oracle-inputs/` copy could stand in
#   for -- `codegen::layout` declares the table and prints it, so the bytes are
#   the emitter's and the move belongs in the diff.
# cases: 192
# re-blessed: the C++ oracle (crates/engine-metal/csrc/src/pipeline/m1_codegen.cpp) is
#   deleted, so these bytes can no longer be re-derived from it. Three cases
#   below were re-blessed from Rust in the change that made the emitted corpus
#   compile: `crates/engine-metal/csrc/tests/msl_compile_test.cpp` put every kernel here
#   through Apple's shader compiler for the first time and found two rejections
#   the byte comparison could never see, because the oracle had them too.
#     * readiness/commit `wide_30` emitted `[[buffer(31)]]`, one past Metal's
#       highest buffer index; `METAL_M1_MAX_CHANNELS` now rejects it by name.
#     * three grouped fused regions passed the `const device bfloat*` logits
#       pointer into `ptir_m1_execute`'s `const device uchar*` parameter, which
#       MSL will not convert. They now cast.
#   Nothing else in this directory moved, and `msl_compile_test` is the standing
#   check that took over from the oracle for the question it could not answer.
#   re-blessed again (emitter 27): `m1_reduce_argmax` no longer materializes one
#   16-byte candidate per element into device `temporary` and tree-reduces it in
#   passes -- `m1_argmax_combine` is a strict total order, so a fold gives the
#   same answer -- and the bf16 logits intrinsic copies unrolled. Both loops run
#   on the single thread that owns a lane, so at a 248320 vocab they were ~250ms
#   per decode step against a ~4ms model forward. Only @runtime bytes moved.
#   re-blessed again (emitter 29): a generated region now gets a threadgroup per
#   lane instead of a thread, and the three things that walked a vocabulary-wide
#   row -- the logits gather, argmax, and the typed copy behind a reshape -- are
#   split across it. m1_copy_typed also moved from one device access per BYTE to
#   one per word. Between them a decode step's GPU time went 89ms -> 6.5ms.
#   re-blessed again (emitter 31): a same-footprint reshape is a view, so the
#   grouped emitter elides its copy and points the consumers at the source
#   value's offset. The runtime executed it as a byte-for-byte copy of the whole
#   value, which in the sampler's graph was a vocabulary-wide round trip through
#   device memory for nothing.
# REGENERATED: the body below is this compiler's output, not the recorded output of the source of truth above
#   reason: metal emitter 41->42, and the shared lane record is what moved. The GROUPED (M3) form binds no per-intrinsic buffer -- every rectangle it reads arrives as an address on the lane record -- so `attn_score` was the one id its table refused: the observability slab is the shell's own reservation (engine_metal::scores), not the readout arena, and no row displacement off lane.logits_base reaches it. LaneRecord now carries attn_score_base and attn_score_row_stride (appended, so every field before them keeps its offset; the record is 96 -> 112 bytes), emit_grouped_fused_region gathers the F32 planes from that base, and m3_intrinsic_bindable stops declining the id. What moved in these dumps is exactly the +73 bytes of struct text the two fields add to the M1 and M3 preambles, plus the emitter version spelled into ptir_m3_generic_{ready,commit}_v42. No emitted logic changed for any corpus case: the base corpus reads no score rectangle, so no kernel body here grew an arm. The bump is load-bearing rather than hygienic -- a library cached under 41 reads the 96-byte record, so lane 1 of any grouped table would start sixteen bytes early -- and CUDA bumps 24 -> 25 for the same reason on its own copy of the struct. The reason the record grew at all is the M2 ceiling: the score rectangle costs two of the twelve argument slots the channels grow into, so a ten-channel guest reading scores (trackb-h2o) was refused by a limit that had nothing to do with what it asked for.
=== beam_epilogue#0 singleton#0
function_name: ptir_m3s_b4fadd8446eb58dd_r0
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=51513 prefix=@runtime+@grouped tail_bytes=9971
kernel void ptir_m3s_b4fadd8446eb58dd_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#1
function_name: ptir_m3s_b4fadd8446eb58dd_r1
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=52566 prefix=@runtime+@grouped tail_bytes=11024
kernel void ptir_m3s_b4fadd8446eb58dd_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[1];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[1]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== beam_epilogue#0 singleton#2
function_name: ptir_m3s_b4fadd8446eb58dd_r2
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=51523 prefix=@runtime+@grouped tail_bytes=9981
kernel void ptir_m3s_b4fadd8446eb58dd_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 2, scratch + offsets[1], scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#3
function_name: ptir_m3s_b4fadd8446eb58dd_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=51523 prefix=@runtime+@grouped tail_bytes=9981
kernel void ptir_m3s_b4fadd8446eb58dd_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#4
function_name: ptir_m3s_b4fadd8446eb58dd_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51536 prefix=@runtime+@grouped tail_bytes=9994
kernel void ptir_m3s_b4fadd8446eb58dd_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 4, scratch + offsets[1], scratch + offsets[3], scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#5
function_name: ptir_m3s_b4fadd8446eb58dd_r5
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=51522 prefix=@runtime+@grouped tail_bytes=9980
kernel void ptir_m3s_b4fadd8446eb58dd_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(1u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#6
function_name: ptir_m3s_b4fadd8446eb58dd_r6
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=51523 prefix=@runtime+@grouped tail_bytes=9981
kernel void ptir_m3s_b4fadd8446eb58dd_r6(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 6, scratch + offsets[5], scratch, scratch, scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#7
function_name: ptir_m3s_b4fadd8446eb58dd_r7
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=51522 prefix=@runtime+@grouped tail_bytes=9980
kernel void ptir_m3s_b4fadd8446eb58dd_r7(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(2u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#10
function_name: ptir_m3s_b4fadd8446eb58dd_r10
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=51526 prefix=@runtime+@grouped tail_bytes=9984
kernel void ptir_m3s_b4fadd8446eb58dd_r10(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 10, scratch + offsets[0], scratch, scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#12
function_name: ptir_m3s_b4fadd8446eb58dd_r12
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51540 prefix=@runtime+@grouped tail_bytes=9998
kernel void ptir_m3s_b4fadd8446eb58dd_r12(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 12, scratch + offsets[11], scratch + offsets[9], scratch, scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#14
function_name: ptir_m3s_b4fadd8446eb58dd_r14
region: library=1 library_op=1 schedule=3 nodes=1 inputs=1 outputs=2 sinks=0
ok: true
source: bytes=51541 prefix=@runtime+@grouped tail_bytes=9999
kernel void ptir_m3s_b4fadd8446eb58dd_r14(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(81u, status, descriptors, lane_params + 14, scratch + offsets[13], scratch, scratch, scratch + offsets[14], scratch + offsets[15], temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#15
function_name: ptir_m3s_b4fadd8446eb58dd_r15
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=51514 prefix=@runtime+@grouped tail_bytes=9972
kernel void ptir_m3s_b4fadd8446eb58dd_r15(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 15, scratch, scratch, scratch, scratch + offsets[16], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#16
function_name: ptir_m3s_b4fadd8446eb58dd_r16
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51541 prefix=@runtime+@grouped tail_bytes=9999
kernel void ptir_m3s_b4fadd8446eb58dd_r16(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(19u, status, descriptors, lane_params + 16, scratch + offsets[15], scratch + offsets[16], scratch, scratch + offsets[17], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#18
function_name: ptir_m3s_b4fadd8446eb58dd_r18
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51541 prefix=@runtime+@grouped tail_bytes=9999
kernel void ptir_m3s_b4fadd8446eb58dd_r18(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 18, scratch + offsets[18], scratch + offsets[17], scratch, scratch + offsets[19], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#25
function_name: ptir_m3s_b4fadd8446eb58dd_r25
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=51514 prefix=@runtime+@grouped tail_bytes=9972
kernel void ptir_m3s_b4fadd8446eb58dd_r25(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 25, scratch, scratch, scratch, scratch + offsets[26], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#26
function_name: ptir_m3s_b4fadd8446eb58dd_r26
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51555 prefix=@runtime+@grouped tail_bytes=10013
kernel void ptir_m3s_b4fadd8446eb58dd_r26(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(99u, status, descriptors, lane_params + 26, scratch + offsets[26], scratch + offsets[17], scratch + offsets[26], scratch + offsets[27], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#28
function_name: ptir_m3s_b4fadd8446eb58dd_r28
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51541 prefix=@runtime+@grouped tail_bytes=9999
kernel void ptir_m3s_b4fadd8446eb58dd_r28(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(24u, status, descriptors, lane_params + 28, scratch + offsets[26], scratch + offsets[28], scratch, scratch + offsets[29], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#30
function_name: ptir_m3s_b4fadd8446eb58dd_r30
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51541 prefix=@runtime+@grouped tail_bytes=9999
kernel void ptir_m3s_b4fadd8446eb58dd_r30(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(26u, status, descriptors, lane_params + 30, scratch + offsets[25], scratch + offsets[30], scratch, scratch + offsets[31], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#31
function_name: ptir_m3s_b4fadd8446eb58dd_r31
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51541 prefix=@runtime+@grouped tail_bytes=9999
kernel void ptir_m3s_b4fadd8446eb58dd_r31(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(28u, status, descriptors, lane_params + 31, scratch + offsets[29], scratch + offsets[31], scratch, scratch + offsets[32], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#35
function_name: ptir_m3s_b4fadd8446eb58dd_r35
region: library=0 library_op=0 schedule=1 nodes=1 inputs=3 outputs=1 sinks=0
ok: true
source: bytes=51555 prefix=@runtime+@grouped tail_bytes=10013
kernel void ptir_m3s_b4fadd8446eb58dd_r35(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 35, scratch + offsets[32], scratch + offsets[34], scratch + offsets[35], scratch + offsets[36], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#43
function_name: ptir_m3s_b4fadd8446eb58dd_r43
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51541 prefix=@runtime+@grouped tail_bytes=9999
kernel void ptir_m3s_b4fadd8446eb58dd_r43(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(18u, status, descriptors, lane_params + 43, scratch + offsets[26], scratch + offsets[42], scratch, scratch + offsets[44], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#48
function_name: ptir_m3s_b4fadd8446eb58dd_r48
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=51671 prefix=@runtime+@grouped tail_bytes=10129
kernel void ptir_m3s_b4fadd8446eb58dd_r48(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 48, scratch + offsets[48], scratch, scratch, pending_1, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_1 = pending_1;
  if (m3_tid == 0) pending_flags[pending_index_1] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== beam_epilogue#0 singleton#75
function_name: ptir_m3s_b4fadd8446eb58dd_r75
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=51541 prefix=@runtime+@grouped tail_bytes=9999
kernel void ptir_m3s_b4fadd8446eb58dd_r75(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(31u, status, descriptors, lane_params + 75, scratch + offsets[15], scratch + offsets[16], scratch, scratch + offsets[66], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 singleton#76
function_name: ptir_m3s_b4fadd8446eb58dd_r76
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=51526 prefix=@runtime+@grouped tail_bytes=9984
kernel void ptir_m3s_b4fadd8446eb58dd_r76(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 76, scratch + offsets[66], scratch, scratch, scratch + offsets[67], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 fused#0
function_name: ptir_m3_b4fadd8446eb58dd_r0
region: library=0 library_op=0 schedule=1 nodes=14 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=56032 prefix=@runtime+@grouped tail_bytes=14490
kernel void ptir_m3_b4fadd8446eb58dd_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[1];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[1]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 2, scratch + offsets[1], scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 4, scratch + offsets[1], scratch + offsets[3], scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(1u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 6, scratch + offsets[5], scratch, scratch, scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(2u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 8, scratch + offsets[7], scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 9, scratch + offsets[4], scratch + offsets[8], scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 11, scratch + offsets[0], scratch, scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 12, scratch + offsets[11], scratch + offsets[9], scratch, scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 13, scratch + offsets[12], scratch, scratch, scratch + offsets[13], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 fused#1
function_name: ptir_m3_b4fadd8446eb58dd_r1
region: library=1 library_op=1 schedule=3 nodes=1 inputs=1 outputs=2 sinks=0
ok: true
source: bytes=51539 prefix=@runtime+@grouped tail_bytes=9997
kernel void ptir_m3_b4fadd8446eb58dd_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(81u, status, descriptors, lane_params + 14, scratch + offsets[13], scratch, scratch, scratch + offsets[14], scratch + offsets[15], temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== beam_epilogue#0 fused#2
function_name: ptir_m3_b4fadd8446eb58dd_r2
region: library=0 library_op=0 schedule=1 nodes=67 inputs=2 outputs=0 sinks=15
ok: true
source: bytes=71884 prefix=@runtime+@grouped tail_bytes=30342
kernel void ptir_m3_b4fadd8446eb58dd_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  const uint dense_7 = channel_bindings[dispatch_lane * layout->reserved0 + 7];
  const M3LaneChannelSlot channel_7 = slots[lane.channel_slot_offset + dense_7];
  const uint pending_index_7 = lane.channel_slot_offset + dense_7;
  const device uchar* current_7 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_7] != 0 ? channel_7.pending_cell : channel_7.committed_cell);
  device uchar* pending_7 = reinterpret_cast<device uchar*>(channel_7.pending_cell);
  const uint dense_8 = channel_bindings[dispatch_lane * layout->reserved0 + 8];
  const M3LaneChannelSlot channel_8 = slots[lane.channel_slot_offset + dense_8];
  const uint pending_index_8 = lane.channel_slot_offset + dense_8;
  const device uchar* current_8 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_8] != 0 ? channel_8.pending_cell : channel_8.committed_cell);
  device uchar* pending_8 = reinterpret_cast<device uchar*>(channel_8.pending_cell);
  const uint dense_9 = channel_bindings[dispatch_lane * layout->reserved0 + 9];
  const M3LaneChannelSlot channel_9 = slots[lane.channel_slot_offset + dense_9];
  const uint pending_index_9 = lane.channel_slot_offset + dense_9;
  const device uchar* current_9 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_9] != 0 ? channel_9.pending_cell : channel_9.committed_cell);
  device uchar* pending_9 = reinterpret_cast<device uchar*>(channel_9.pending_cell);
  const uint dense_10 = channel_bindings[dispatch_lane * layout->reserved0 + 10];
  const M3LaneChannelSlot channel_10 = slots[lane.channel_slot_offset + dense_10];
  const uint pending_index_10 = lane.channel_slot_offset + dense_10;
  const device uchar* current_10 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_10] != 0 ? channel_10.pending_cell : channel_10.committed_cell);
  device uchar* pending_10 = reinterpret_cast<device uchar*>(channel_10.pending_cell);
  const uint dense_11 = channel_bindings[dispatch_lane * layout->reserved0 + 11];
  const M3LaneChannelSlot channel_11 = slots[lane.channel_slot_offset + dense_11];
  const uint pending_index_11 = lane.channel_slot_offset + dense_11;
  const device uchar* current_11 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_11] != 0 ? channel_11.pending_cell : channel_11.committed_cell);
  device uchar* pending_11 = reinterpret_cast<device uchar*>(channel_11.pending_cell);
  const uint dense_12 = channel_bindings[dispatch_lane * layout->reserved0 + 12];
  const M3LaneChannelSlot channel_12 = slots[lane.channel_slot_offset + dense_12];
  const uint pending_index_12 = lane.channel_slot_offset + dense_12;
  const device uchar* current_12 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_12] != 0 ? channel_12.pending_cell : channel_12.committed_cell);
  device uchar* pending_12 = reinterpret_cast<device uchar*>(channel_12.pending_cell);
  const uint dense_13 = channel_bindings[dispatch_lane * layout->reserved0 + 13];
  const M3LaneChannelSlot channel_13 = slots[lane.channel_slot_offset + dense_13];
  const uint pending_index_13 = lane.channel_slot_offset + dense_13;
  const device uchar* current_13 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_13] != 0 ? channel_13.pending_cell : channel_13.committed_cell);
  device uchar* pending_13 = reinterpret_cast<device uchar*>(channel_13.pending_cell);
  const uint dense_14 = channel_bindings[dispatch_lane * layout->reserved0 + 14];
  const M3LaneChannelSlot channel_14 = slots[lane.channel_slot_offset + dense_14];
  const uint pending_index_14 = lane.channel_slot_offset + dense_14;
  const device uchar* current_14 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_14] != 0 ? channel_14.pending_cell : channel_14.committed_cell);
  device uchar* pending_14 = reinterpret_cast<device uchar*>(channel_14.pending_cell);
  const uint dense_15 = channel_bindings[dispatch_lane * layout->reserved0 + 15];
  const M3LaneChannelSlot channel_15 = slots[lane.channel_slot_offset + dense_15];
  const uint pending_index_15 = lane.channel_slot_offset + dense_15;
  const device uchar* current_15 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_15] != 0 ? channel_15.pending_cell : channel_15.committed_cell);
  device uchar* pending_15 = reinterpret_cast<device uchar*>(channel_15.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 15, scratch, scratch, scratch, scratch + offsets[16], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(19u, status, descriptors, lane_params + 16, scratch + offsets[15], scratch + offsets[16], scratch, scratch + offsets[17], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 17, current_1, scratch, scratch, scratch + offsets[18], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 18, scratch + offsets[18], scratch + offsets[17], scratch, scratch + offsets[19], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 19, current_2, scratch, scratch, scratch + offsets[20], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 20, scratch + offsets[20], scratch + offsets[17], scratch, scratch + offsets[21], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 21, current_3, scratch, scratch, scratch + offsets[22], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 22, scratch + offsets[22], scratch + offsets[17], scratch, scratch + offsets[23], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 23, current_4, scratch, scratch, scratch + offsets[24], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 24, scratch + offsets[24], scratch + offsets[17], scratch, scratch + offsets[25], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 25, scratch, scratch, scratch, scratch + offsets[26], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(99u, status, descriptors, lane_params + 26, scratch + offsets[26], scratch + offsets[17], scratch + offsets[26], scratch + offsets[27], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 27, scratch + offsets[27], scratch + offsets[17], scratch, scratch + offsets[28], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(24u, status, descriptors, lane_params + 28, scratch + offsets[26], scratch + offsets[28], scratch, scratch + offsets[29], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 29, scratch, scratch, scratch, scratch + offsets[30], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(26u, status, descriptors, lane_params + 30, scratch + offsets[25], scratch + offsets[30], scratch, scratch + offsets[31], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(28u, status, descriptors, lane_params + 31, scratch + offsets[29], scratch + offsets[31], scratch, scratch + offsets[32], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 32, current_5, scratch, scratch, scratch + offsets[33], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 33, scratch + offsets[33], scratch + offsets[17], scratch, scratch + offsets[34], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 34, current_6, scratch, scratch, scratch + offsets[35], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 35, scratch + offsets[32], scratch + offsets[34], scratch + offsets[35], scratch + offsets[36], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 36, scratch, scratch, scratch, scratch + offsets[37], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 37, scratch + offsets[32], scratch + offsets[25], scratch + offsets[37], scratch + offsets[38], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 38, scratch, scratch, scratch, scratch + offsets[39], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 39, scratch + offsets[23], scratch + offsets[39], scratch, scratch + offsets[40], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 40, scratch + offsets[32], scratch + offsets[23], scratch + offsets[40], scratch + offsets[41], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 41, scratch, scratch, scratch, scratch + offsets[42], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 42, scratch + offsets[41], scratch + offsets[39], scratch, scratch + offsets[43], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(18u, status, descriptors, lane_params + 43, scratch + offsets[26], scratch + offsets[42], scratch, scratch + offsets[44], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 44, scratch + offsets[43], scratch + offsets[44], scratch, scratch + offsets[45], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(99u, status, descriptors, lane_params + 46, scratch + offsets[19], scratch + offsets[45], scratch + offsets[36], scratch + offsets[47], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 47, scratch + offsets[47], scratch, scratch, scratch + offsets[48], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 48, scratch + offsets[48], scratch, scratch, pending_1, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_1 = pending_1;
  if (m3_tid == 0) pending_flags[pending_index_1] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 49, scratch + offsets[38], scratch + offsets[39], scratch, scratch + offsets[49], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(99u, status, descriptors, lane_params + 51, scratch + offsets[21], scratch + offsets[45], scratch + offsets[49], scratch + offsets[51], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 52, scratch + offsets[51], scratch, scratch, scratch + offsets[52], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 53, scratch + offsets[52], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(18u, status, descriptors, lane_params + 54, scratch + offsets[30], scratch + offsets[43], scratch, scratch + offsets[53], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 55, scratch + offsets[49], scratch + offsets[53], scratch, scratch + offsets[54], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 56, current_7, scratch, scratch, scratch + offsets[55], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 57, scratch + offsets[54], scratch, scratch, pending_7, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_7 = pending_7;
  if (m3_tid == 0) pending_flags[pending_index_7] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 58, scratch, scratch, scratch, scratch + offsets[56], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 60, scratch + offsets[56], scratch, scratch, scratch + offsets[58], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 62, scratch + offsets[52], scratch, scratch, scratch + offsets[60], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(26u, status, descriptors, lane_params + 63, scratch + offsets[58], scratch + offsets[60], scratch, scratch + offsets[61], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 64, scratch + offsets[61], scratch, scratch, scratch + offsets[62], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 65, current_8, scratch, scratch, scratch + offsets[63], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 66, scratch + offsets[62], scratch, scratch, pending_8, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_8 = pending_8;
  if (m3_tid == 0) pending_flags[pending_index_8] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 67, current_9, scratch, scratch, scratch + offsets[64], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 68, scratch + offsets[39], scratch + offsets[64], scratch, scratch + offsets[65], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 69, scratch + offsets[65], scratch, scratch, pending_9, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_9 = pending_9;
  if (m3_tid == 0) pending_flags[pending_index_9] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 70, scratch + offsets[41], scratch, scratch, pending_3, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_3 = pending_3;
  if (m3_tid == 0) pending_flags[pending_index_3] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 71, scratch + offsets[36], scratch, scratch, pending_5, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_5 = pending_5;
  if (m3_tid == 0) pending_flags[pending_index_5] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 72, scratch + offsets[49], scratch, scratch, pending_4, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_4 = pending_4;
  if (m3_tid == 0) pending_flags[pending_index_4] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 73, scratch + offsets[36], scratch, scratch, pending_10, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_10 = pending_10;
  if (m3_tid == 0) pending_flags[pending_index_10] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 74, scratch + offsets[38], scratch, scratch, pending_11, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_11 = pending_11;
  if (m3_tid == 0) pending_flags[pending_index_11] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(31u, status, descriptors, lane_params + 75, scratch + offsets[15], scratch + offsets[16], scratch, scratch + offsets[66], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 76, scratch + offsets[66], scratch, scratch, scratch + offsets[67], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 77, scratch + offsets[67], scratch, scratch, pending_12, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_12 = pending_12;
  if (m3_tid == 0) pending_flags[pending_index_12] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 78, scratch + offsets[14], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 79, scratch + offsets[67], scratch, scratch, pending_13, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_13 = pending_13;
  if (m3_tid == 0) pending_flags[pending_index_13] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 80, scratch + offsets[17], scratch, scratch, pending_14, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_14 = pending_14;
  if (m3_tid == 0) pending_flags[pending_index_14] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 81, scratch + offsets[14], scratch, scratch, pending_15, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_15 = pending_15;
  if (m3_tid == 0) pending_flags[pending_index_15] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== counter_pingpong#0 singleton#0
function_name: ptir_m3s_f4e949004f50e90e_r0
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44791 prefix=@runtime+@grouped tail_bytes=3249
kernel void ptir_m3s_f4e949004f50e90e_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== counter_pingpong#0 singleton#1
function_name: ptir_m3s_f4e949004f50e90e_r1
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44789 prefix=@runtime+@grouped tail_bytes=3247
kernel void ptir_m3s_f4e949004f50e90e_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 1, scratch, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== counter_pingpong#0 singleton#2
function_name: ptir_m3s_f4e949004f50e90e_r2
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=44814 prefix=@runtime+@grouped tail_bytes=3272
kernel void ptir_m3s_f4e949004f50e90e_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 2, scratch + offsets[0], scratch + offsets[1], scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== counter_pingpong#0 singleton#3
function_name: ptir_m3s_f4e949004f50e90e_r3
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=44946 prefix=@runtime+@grouped tail_bytes=3404
kernel void ptir_m3s_f4e949004f50e90e_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== counter_pingpong#0 fused#0
function_name: ptir_m3_f4e949004f50e90e_r0
region: library=0 library_op=0 schedule=1 nodes=5 inputs=0 outputs=0 sinks=2
ok: true
source: bytes=46225 prefix=@runtime+@grouped tail_bytes=4683
kernel void ptir_m3_f4e949004f50e90e_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 1, scratch, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 2, scratch + offsets[0], scratch + offsets[1], scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 4, scratch + offsets[2], scratch, scratch, pending_1, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_1 = pending_1;
  if (m3_tid == 0) pending_flags[pending_index_1] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== dfa_ingraph#0 singleton#0
function_name: ptir_m3s_4f4b856e5ef3e8b4_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=46794 prefix=@runtime+@grouped tail_bytes=5252
kernel void ptir_m3s_4f4b856e5ef3e8b4_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== dfa_ingraph#0 singleton#1
function_name: ptir_m3s_4f4b856e5ef3e8b4_r1
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45741 prefix=@runtime+@grouped tail_bytes=4199
kernel void ptir_m3s_4f4b856e5ef3e8b4_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 1, current_0, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== dfa_ingraph#0 singleton#3
function_name: ptir_m3s_4f4b856e5ef3e8b4_r3
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45741 prefix=@runtime+@grouped tail_bytes=4199
kernel void ptir_m3s_4f4b856e5ef3e8b4_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 3, current_2, scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== dfa_ingraph#0 singleton#4
function_name: ptir_m3s_4f4b856e5ef3e8b4_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45764 prefix=@runtime+@grouped tail_bytes=4222
kernel void ptir_m3s_4f4b856e5ef3e8b4_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 4, scratch + offsets[1], scratch + offsets[3], scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== dfa_ingraph#0 singleton#5
function_name: ptir_m3s_4f4b856e5ef3e8b4_r5
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45739 prefix=@runtime+@grouped tail_bytes=4197
kernel void ptir_m3s_4f4b856e5ef3e8b4_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 5, scratch, scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== dfa_ingraph#0 singleton#6
function_name: ptir_m3s_4f4b856e5ef3e8b4_r6
region: library=0 library_op=0 schedule=1 nodes=1 inputs=3 outputs=1 sinks=0
ok: true
source: bytes=45777 prefix=@runtime+@grouped tail_bytes=4235
kernel void ptir_m3s_4f4b856e5ef3e8b4_r6(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 6, scratch + offsets[4], scratch + offsets[0], scratch + offsets[5], scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== dfa_ingraph#0 singleton#7
function_name: ptir_m3s_4f4b856e5ef3e8b4_r7
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45751 prefix=@runtime+@grouped tail_bytes=4209
kernel void ptir_m3s_4f4b856e5ef3e8b4_r7(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== dfa_ingraph#0 singleton#8
function_name: ptir_m3s_4f4b856e5ef3e8b4_r8
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=45896 prefix=@runtime+@grouped tail_bytes=4354
kernel void ptir_m3s_4f4b856e5ef3e8b4_r8(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 8, scratch + offsets[7], scratch, scratch, pending_3, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_3 = pending_3;
  if (m3_tid == 0) pending_flags[pending_index_3] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== dfa_ingraph#0 singleton#10
function_name: ptir_m3s_4f4b856e5ef3e8b4_r10
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45766 prefix=@runtime+@grouped tail_bytes=4224
kernel void ptir_m3s_4f4b856e5ef3e8b4_r10(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(18u, status, descriptors, lane_params + 10, scratch + offsets[3], scratch + offsets[8], scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== dfa_ingraph#0 singleton#11
function_name: ptir_m3s_4f4b856e5ef3e8b4_r11
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45753 prefix=@runtime+@grouped tail_bytes=4211
kernel void ptir_m3s_4f4b856e5ef3e8b4_r11(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 11, scratch + offsets[7], scratch, scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== dfa_ingraph#0 singleton#12
function_name: ptir_m3s_4f4b856e5ef3e8b4_r12
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45768 prefix=@runtime+@grouped tail_bytes=4226
kernel void ptir_m3s_4f4b856e5ef3e8b4_r12(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 12, scratch + offsets[9], scratch + offsets[10], scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== dfa_ingraph#0 fused#0
function_name: ptir_m3_4f4b856e5ef3e8b4_r0
region: library=0 library_op=0 schedule=1 nodes=15 inputs=0 outputs=0 sinks=2
ok: true
source: bytes=51121 prefix=@runtime+@grouped tail_bytes=9579
kernel void ptir_m3_4f4b856e5ef3e8b4_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 1, current_0, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 2, current_1, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 3, current_2, scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 4, scratch + offsets[1], scratch + offsets[3], scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 5, scratch, scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 6, scratch + offsets[4], scratch + offsets[0], scratch + offsets[5], scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 8, scratch + offsets[7], scratch, scratch, pending_3, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_3 = pending_3;
  if (m3_tid == 0) pending_flags[pending_index_3] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 9, scratch, scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(18u, status, descriptors, lane_params + 10, scratch + offsets[3], scratch + offsets[8], scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 11, scratch + offsets[7], scratch, scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 12, scratch + offsets[9], scratch + offsets[10], scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 13, scratch + offsets[2], scratch + offsets[11], scratch, scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 14, scratch + offsets[12], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== extern_contrastive#0 singleton#0
function_name: ptir_m3s_d01f407b22863878_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45369 prefix=@runtime+@grouped tail_bytes=3827
kernel void ptir_m3s_d01f407b22863878_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== extern_contrastive#0 singleton#1
function_name: ptir_m3s_d01f407b22863878_r1
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=44471 prefix=@runtime+@grouped tail_bytes=2929
kernel void ptir_m3s_d01f407b22863878_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== extern_contrastive#0 fused#0
function_name: ptir_m3_d01f407b22863878_r0
region: library=0 library_op=0 schedule=1 nodes=2 inputs=0 outputs=0 sinks=1
ok: true
source: bytes=45799 prefix=@runtime+@grouped tail_bytes=4257
kernel void ptir_m3_d01f407b22863878_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== greedy_argmax#0 singleton#0
function_name: ptir_m3s_f2566cc0c1e0529a_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45844 prefix=@runtime+@grouped tail_bytes=4302
kernel void ptir_m3s_f2566cc0c1e0529a_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== greedy_argmax#0 singleton#1
function_name: ptir_m3s_f2566cc0c1e0529a_r1
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44801 prefix=@runtime+@grouped tail_bytes=3259
kernel void ptir_m3s_f2566cc0c1e0529a_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== greedy_argmax#0 singleton#2
function_name: ptir_m3s_f2566cc0c1e0529a_r2
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44801 prefix=@runtime+@grouped tail_bytes=3259
kernel void ptir_m3s_f2566cc0c1e0529a_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 2, scratch + offsets[1], scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== greedy_argmax#0 singleton#4
function_name: ptir_m3s_f2566cc0c1e0529a_r4
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=44946 prefix=@runtime+@grouped tail_bytes=3404
kernel void ptir_m3s_f2566cc0c1e0529a_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== greedy_argmax#0 fused#0
function_name: ptir_m3_f2566cc0c1e0529a_r0
region: library=0 library_op=0 schedule=1 nodes=6 inputs=0 outputs=0 sinks=2
ok: true
source: bytes=48088 prefix=@runtime+@grouped tail_bytes=6546
kernel void ptir_m3_f2566cc0c1e0529a_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  {
    const M1ValueDesc am_in = descriptors[0];
    const uint am_row_base = row_meta.offset + 0u;
    const uint am_vocab = layout->vocab;
    const uint am_width = am_in.last == 0u ? am_vocab : am_in.last;
    if (am_vocab == 0u || am_width == 0u || am_width > am_vocab || am_in.rows > row_meta.count) { m1_fault(status, 0xA0u); return; }
    device int* am_out = reinterpret_cast<device int*>(scratch + offsets[2]);
    for (uint am_r = 0u; am_r < am_in.rows; ++am_r) {
      const uint am_src_row = row_indices[am_row_base + am_r];
      const device bfloat* am_src = logits + ulong(am_src_row) * am_vocab;
      M1ArgmaxCandidate am_best = {-INFINITY, 0u, 0u, 0u};
      M1ArgmaxCandidate am_b1 = am_best, am_b2 = am_best, am_b3 = am_best;
      const uint am_w = m3_threads;
      uint am_c = m3_tid;
      for (; am_c + 3u * am_w < am_width; am_c += 4u * am_w) {
        const float v0 = float(am_src[am_c]);
        const float v1 = float(am_src[am_c + am_w]);
        const float v2 = float(am_src[am_c + 2u * am_w]);
        const float v3 = float(am_src[am_c + 3u * am_w]);
        am_best = m1_argmax_combine(am_best, M1ArgmaxCandidate{v0, am_c, isnan(v0) ? 0u : 1u, 0u});
        am_b1 = m1_argmax_combine(am_b1, M1ArgmaxCandidate{v1, am_c + am_w, isnan(v1) ? 0u : 1u, 0u});
        am_b2 = m1_argmax_combine(am_b2, M1ArgmaxCandidate{v2, am_c + 2u * am_w, isnan(v2) ? 0u : 1u, 0u});
        am_b3 = m1_argmax_combine(am_b3, M1ArgmaxCandidate{v3, am_c + 3u * am_w, isnan(v3) ? 0u : 1u, 0u});
      }
      for (; am_c < am_width; am_c += am_w) {
        const float am_v = float(am_src[am_c]);
        am_best = m1_argmax_combine(am_best, M1ArgmaxCandidate{am_v, am_c, isnan(am_v) ? 0u : 1u, 0u});
      }
      am_best = m1_argmax_combine(m1_argmax_combine(am_best, am_b1), m1_argmax_combine(am_b2, am_b3));
      m3_tgbuf[m3_tid] = am_best;
      threadgroup_barrier(mem_flags::mem_threadgroup);
      for (uint am_s = 1u; am_s < m3_threads; am_s <<= 1) {
        if ((m3_tid % (2u * am_s)) == 0u && m3_tid + am_s < m3_threads) m3_tgbuf[m3_tid] = m1_argmax_combine(m3_tgbuf[m3_tid], m3_tgbuf[m3_tid + am_s]);
        threadgroup_barrier(mem_flags::mem_threadgroup);
      }
      if (m3_tid == 0) am_out[am_r] = int(m3_tgbuf[0].index);
      threadgroup_barrier(mem_flags::mem_threadgroup);
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 5, scratch + offsets[3], scratch, scratch, pending_1, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_1 = pending_1;
  if (m3_tid == 0) pending_flags[pending_index_1] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== lora_prologue#0 singleton#0
function_name: ptir_m3s_26849b68dee54237_r0
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44791 prefix=@runtime+@grouped tail_bytes=3249
kernel void ptir_m3s_26849b68dee54237_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== lora_prologue#0 singleton#2
function_name: ptir_m3s_26849b68dee54237_r2
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44789 prefix=@runtime+@grouped tail_bytes=3247
kernel void ptir_m3s_26849b68dee54237_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 2, scratch, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== lora_prologue#0 singleton#3
function_name: ptir_m3s_26849b68dee54237_r3
region: library=1 library_op=5 schedule=3 nodes=1 inputs=3 outputs=0 sinks=0
ok: true
source: bytes=44815 prefix=@runtime+@grouped tail_bytes=3273
kernel void ptir_m3s_26849b68dee54237_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(162u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch + offsets[1], scratch + offsets[2], scratch, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== lora_prologue#0 fused#0
function_name: ptir_m3_26849b68dee54237_r0
region: library=0 library_op=0 schedule=1 nodes=3 inputs=0 outputs=3 sinks=0
ok: true
source: bytes=45340 prefix=@runtime+@grouped tail_bytes=3798
kernel void ptir_m3_26849b68dee54237_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 1, current_1, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 2, scratch, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== lora_prologue#0 fused#1
function_name: ptir_m3_26849b68dee54237_r1
region: library=1 library_op=5 schedule=3 nodes=1 inputs=3 outputs=0 sinks=0
ok: true
source: bytes=44814 prefix=@runtime+@grouped tail_bytes=3272
kernel void ptir_m3_26849b68dee54237_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(162u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch + offsets[1], scratch + offsets[2], scratch, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== matrix_mask_apply_packed#0 singleton#0
function_name: ptir_m3s_6612fe00a51d7edc_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45369 prefix=@runtime+@grouped tail_bytes=3827
kernel void ptir_m3s_6612fe00a51d7edc_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== matrix_mask_apply_packed#0 singleton#1
function_name: ptir_m3s_6612fe00a51d7edc_r1
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44314 prefix=@runtime+@grouped tail_bytes=2772
kernel void ptir_m3s_6612fe00a51d7edc_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 1, scratch, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== matrix_mask_apply_packed#0 singleton#2
function_name: ptir_m3s_6612fe00a51d7edc_r2
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44326 prefix=@runtime+@grouped tail_bytes=2784
kernel void ptir_m3s_6612fe00a51d7edc_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 2, scratch + offsets[1], scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== matrix_mask_apply_packed#0 singleton#3
function_name: ptir_m3s_6612fe00a51d7edc_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=44340 prefix=@runtime+@grouped tail_bytes=2798
kernel void ptir_m3s_6612fe00a51d7edc_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(101u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch + offsets[2], scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== matrix_mask_apply_packed#0 singleton#4
function_name: ptir_m3s_6612fe00a51d7edc_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44326 prefix=@runtime+@grouped tail_bytes=2784
kernel void ptir_m3s_6612fe00a51d7edc_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== matrix_mask_apply_packed#0 singleton#5
function_name: ptir_m3s_6612fe00a51d7edc_r5
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=44471 prefix=@runtime+@grouped tail_bytes=2929
kernel void ptir_m3s_6612fe00a51d7edc_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== matrix_mask_apply_packed#0 fused#0
function_name: ptir_m3_6612fe00a51d7edc_r0
region: library=0 library_op=0 schedule=1 nodes=6 inputs=0 outputs=0 sinks=1
ok: true
source: bytes=46659 prefix=@runtime+@grouped tail_bytes=5117
kernel void ptir_m3_6612fe00a51d7edc_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 1, scratch, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(101u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch + offsets[1], scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== matrix_select_mask#0 singleton#0
function_name: ptir_m3s_101abcfb79b08b89_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45844 prefix=@runtime+@grouped tail_bytes=4302
kernel void ptir_m3s_101abcfb79b08b89_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== matrix_select_mask#0 singleton#1
function_name: ptir_m3s_101abcfb79b08b89_r1
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44791 prefix=@runtime+@grouped tail_bytes=3249
kernel void ptir_m3s_101abcfb79b08b89_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 1, current_0, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== matrix_select_mask#0 singleton#2
function_name: ptir_m3s_101abcfb79b08b89_r2
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44789 prefix=@runtime+@grouped tail_bytes=3247
kernel void ptir_m3s_101abcfb79b08b89_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 2, scratch, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== matrix_select_mask#0 singleton#3
function_name: ptir_m3s_101abcfb79b08b89_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=3 outputs=1 sinks=0
ok: true
source: bytes=44827 prefix=@runtime+@grouped tail_bytes=3285
kernel void ptir_m3s_101abcfb79b08b89_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 3, scratch + offsets[1], scratch + offsets[0], scratch + offsets[2], scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== matrix_select_mask#0 singleton#4
function_name: ptir_m3s_101abcfb79b08b89_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44801 prefix=@runtime+@grouped tail_bytes=3259
kernel void ptir_m3s_101abcfb79b08b89_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== matrix_select_mask#0 singleton#5
function_name: ptir_m3s_101abcfb79b08b89_r5
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=44946 prefix=@runtime+@grouped tail_bytes=3404
kernel void ptir_m3s_101abcfb79b08b89_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, pending_1, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_1 = pending_1;
  if (m3_tid == 0) pending_flags[pending_index_1] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== matrix_select_mask#0 fused#0
function_name: ptir_m3_101abcfb79b08b89_r0
region: library=0 library_op=0 schedule=1 nodes=6 inputs=0 outputs=0 sinks=1
ok: true
source: bytes=47422 prefix=@runtime+@grouped tail_bytes=5880
kernel void ptir_m3_101abcfb79b08b89_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 1, current_0, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 2, scratch, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 3, scratch + offsets[1], scratch + offsets[0], scratch + offsets[2], scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, pending_1, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_1 = pending_1;
  if (m3_tid == 0) pending_flags[pending_index_1] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== mtp_verify_tail#0 singleton#0
function_name: ptir_m3s_963137d4ffed0445_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=46794 prefix=@runtime+@grouped tail_bytes=5252
kernel void ptir_m3s_963137d4ffed0445_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== mtp_verify_tail#0 singleton#2
function_name: ptir_m3s_963137d4ffed0445_r2
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45741 prefix=@runtime+@grouped tail_bytes=4199
kernel void ptir_m3s_963137d4ffed0445_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 2, current_0, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#3
function_name: ptir_m3s_963137d4ffed0445_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45739 prefix=@runtime+@grouped tail_bytes=4197
kernel void ptir_m3s_963137d4ffed0445_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 3, scratch, scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#4
function_name: ptir_m3s_963137d4ffed0445_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=3 outputs=1 sinks=0
ok: true
source: bytes=45777 prefix=@runtime+@grouped tail_bytes=4235
kernel void ptir_m3s_963137d4ffed0445_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 4, scratch + offsets[2], scratch + offsets[0], scratch + offsets[3], scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#5
function_name: ptir_m3s_963137d4ffed0445_r5
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45751 prefix=@runtime+@grouped tail_bytes=4209
kernel void ptir_m3s_963137d4ffed0445_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#7
function_name: ptir_m3s_963137d4ffed0445_r7
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45739 prefix=@runtime+@grouped tail_bytes=4197
kernel void ptir_m3s_963137d4ffed0445_r7(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 7, scratch, scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#8
function_name: ptir_m3s_963137d4ffed0445_r8
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45764 prefix=@runtime+@grouped tail_bytes=4222
kernel void ptir_m3s_963137d4ffed0445_r8(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 8, scratch + offsets[5], scratch + offsets[7], scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#9
function_name: ptir_m3s_963137d4ffed0445_r9
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45764 prefix=@runtime+@grouped tail_bytes=4222
kernel void ptir_m3s_963137d4ffed0445_r9(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(24u, status, descriptors, lane_params + 9, scratch + offsets[6], scratch + offsets[8], scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#13
function_name: ptir_m3s_963137d4ffed0445_r13
region: library=1 library_op=3 schedule=3 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45755 prefix=@runtime+@grouped tail_bytes=4213
kernel void ptir_m3s_963137d4ffed0445_r13(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(65u, status, descriptors, lane_params + 13, scratch + offsets[12], scratch, scratch, scratch + offsets[13], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#14
function_name: ptir_m3s_963137d4ffed0445_r14
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45755 prefix=@runtime+@grouped tail_bytes=4213
kernel void ptir_m3s_963137d4ffed0445_r14(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 14, scratch + offsets[13], scratch, scratch, scratch + offsets[14], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#15
function_name: ptir_m3s_963137d4ffed0445_r15
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45754 prefix=@runtime+@grouped tail_bytes=4212
kernel void ptir_m3s_963137d4ffed0445_r15(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 15, scratch + offsets[14], scratch, scratch, scratch + offsets[15], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#16
function_name: ptir_m3s_963137d4ffed0445_r16
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45755 prefix=@runtime+@grouped tail_bytes=4213
kernel void ptir_m3s_963137d4ffed0445_r16(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 16, scratch + offsets[15], scratch, scratch, scratch + offsets[16], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#18
function_name: ptir_m3s_963137d4ffed0445_r18
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45769 prefix=@runtime+@grouped tail_bytes=4227
kernel void ptir_m3s_963137d4ffed0445_r18(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(23u, status, descriptors, lane_params + 18, scratch + offsets[16], scratch + offsets[17], scratch, scratch + offsets[18], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 singleton#21
function_name: ptir_m3s_963137d4ffed0445_r21
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=45899 prefix=@runtime+@grouped tail_bytes=4357
kernel void ptir_m3s_963137d4ffed0445_r21(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 21, scratch + offsets[20], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== mtp_verify_tail#0 fused#0
function_name: ptir_m3_963137d4ffed0445_r0
region: library=0 library_op=0 schedule=1 nodes=13 inputs=0 outputs=3 sinks=0
ok: true
source: bytes=51303 prefix=@runtime+@grouped tail_bytes=9761
kernel void ptir_m3_963137d4ffed0445_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[1];
    const uint intrinsic_row_base = row_meta.mtp_offset;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[1]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 2, current_0, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 3, scratch, scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 4, scratch + offsets[2], scratch + offsets[0], scratch + offsets[3], scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 6, current_1, scratch, scratch, scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 7, scratch, scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 8, scratch + offsets[5], scratch + offsets[7], scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(24u, status, descriptors, lane_params + 9, scratch + offsets[6], scratch + offsets[8], scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 10, scratch, scratch, scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 11, scratch, scratch, scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 12, scratch + offsets[9], scratch + offsets[10], scratch + offsets[11], scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 fused#1
function_name: ptir_m3_963137d4ffed0445_r1
region: library=1 library_op=3 schedule=3 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45753 prefix=@runtime+@grouped tail_bytes=4211
kernel void ptir_m3_963137d4ffed0445_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(65u, status, descriptors, lane_params + 13, scratch + offsets[12], scratch, scratch, scratch + offsets[13], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== mtp_verify_tail#0 fused#2
function_name: ptir_m3_963137d4ffed0445_r2
region: library=0 library_op=0 schedule=1 nodes=11 inputs=3 outputs=0 sinks=3
ok: true
source: bytes=49088 prefix=@runtime+@grouped tail_bytes=7546
kernel void ptir_m3_963137d4ffed0445_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 14, scratch + offsets[13], scratch, scratch, scratch + offsets[14], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 15, scratch + offsets[14], scratch, scratch, scratch + offsets[15], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 16, scratch + offsets[15], scratch, scratch, scratch + offsets[16], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 17, scratch, scratch, scratch, scratch + offsets[17], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(23u, status, descriptors, lane_params + 18, scratch + offsets[16], scratch + offsets[17], scratch, scratch + offsets[18], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 19, scratch, scratch, scratch, scratch + offsets[19], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 20, scratch + offsets[18], scratch + offsets[5], scratch + offsets[19], scratch + offsets[20], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 21, scratch + offsets[20], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 22, scratch + offsets[1], scratch, scratch, scratch + offsets[21], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 23, scratch + offsets[21], scratch, scratch, pending_1, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_1 = pending_1;
  if (m3_tid == 0) pending_flags[pending_index_1] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 24, scratch + offsets[21], scratch, scratch, pending_3, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_3 = pending_3;
  if (m3_tid == 0) pending_flags[pending_index_3] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== nucleus_sample#0 singleton#0
function_name: ptir_m3s_82921040f7bc2612_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=46319 prefix=@runtime+@grouped tail_bytes=4777
kernel void ptir_m3s_82921040f7bc2612_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== nucleus_sample#0 singleton#1
function_name: ptir_m3s_82921040f7bc2612_r1
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45266 prefix=@runtime+@grouped tail_bytes=3724
kernel void ptir_m3s_82921040f7bc2612_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 1, current_0, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#2
function_name: ptir_m3s_82921040f7bc2612_r2
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45266 prefix=@runtime+@grouped tail_bytes=3724
kernel void ptir_m3s_82921040f7bc2612_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 2, current_1, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#3
function_name: ptir_m3s_82921040f7bc2612_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45276 prefix=@runtime+@grouped tail_bytes=3734
kernel void ptir_m3s_82921040f7bc2612_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#4
function_name: ptir_m3s_82921040f7bc2612_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45276 prefix=@runtime+@grouped tail_bytes=3734
kernel void ptir_m3s_82921040f7bc2612_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#5
function_name: ptir_m3s_82921040f7bc2612_r5
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45289 prefix=@runtime+@grouped tail_bytes=3747
kernel void ptir_m3s_82921040f7bc2612_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 5, scratch + offsets[0], scratch + offsets[4], scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#6
function_name: ptir_m3s_82921040f7bc2612_r6
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45275 prefix=@runtime+@grouped tail_bytes=3733
kernel void ptir_m3s_82921040f7bc2612_r6(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(1u, status, descriptors, lane_params + 6, scratch + offsets[5], scratch, scratch, scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#7
function_name: ptir_m3s_82921040f7bc2612_r7
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45276 prefix=@runtime+@grouped tail_bytes=3734
kernel void ptir_m3s_82921040f7bc2612_r7(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#9
function_name: ptir_m3s_82921040f7bc2612_r9
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45289 prefix=@runtime+@grouped tail_bytes=3747
kernel void ptir_m3s_82921040f7bc2612_r9(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(19u, status, descriptors, lane_params + 9, scratch + offsets[6], scratch + offsets[8], scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#10
function_name: ptir_m3s_82921040f7bc2612_r10
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45292 prefix=@runtime+@grouped tail_bytes=3750
kernel void ptir_m3s_82921040f7bc2612_r10(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(88u, status, descriptors, lane_params + 10, scratch + offsets[9], scratch + offsets[2], scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#11
function_name: ptir_m3s_82921040f7bc2612_r11
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45267 prefix=@runtime+@grouped tail_bytes=3725
kernel void ptir_m3s_82921040f7bc2612_r11(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 11, scratch, scratch, scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#12
function_name: ptir_m3s_82921040f7bc2612_r12
region: library=0 library_op=0 schedule=1 nodes=1 inputs=3 outputs=1 sinks=0
ok: true
source: bytes=45307 prefix=@runtime+@grouped tail_bytes=3765
kernel void ptir_m3s_82921040f7bc2612_r12(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 12, scratch + offsets[10], scratch + offsets[0], scratch + offsets[11], scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#13
function_name: ptir_m3s_82921040f7bc2612_r13
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45280 prefix=@runtime+@grouped tail_bytes=3738
kernel void ptir_m3s_82921040f7bc2612_r13(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(113u, status, descriptors, lane_params + 13, scratch + offsets[1], scratch, scratch, scratch + offsets[13], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#14
function_name: ptir_m3s_82921040f7bc2612_r14
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45294 prefix=@runtime+@grouped tail_bytes=3752
kernel void ptir_m3s_82921040f7bc2612_r14(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 14, scratch + offsets[12], scratch + offsets[13], scratch, scratch + offsets[14], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#15
function_name: ptir_m3s_82921040f7bc2612_r15
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45280 prefix=@runtime+@grouped tail_bytes=3738
kernel void ptir_m3s_82921040f7bc2612_r15(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 15, scratch + offsets[14], scratch, scratch, scratch + offsets[15], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 singleton#16
function_name: ptir_m3s_82921040f7bc2612_r16
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=45424 prefix=@runtime+@grouped tail_bytes=3882
kernel void ptir_m3s_82921040f7bc2612_r16(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 16, scratch + offsets[15], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== nucleus_sample#0 fused#0
function_name: ptir_m3_82921040f7bc2612_r0
region: library=0 library_op=0 schedule=1 nodes=3 inputs=0 outputs=3 sinks=0
ok: true
source: bytes=46870 prefix=@runtime+@grouped tail_bytes=5328
kernel void ptir_m3_82921040f7bc2612_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 1, current_0, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 2, current_1, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 fused#1
function_name: ptir_m3_82921040f7bc2612_r1
region: library=1 library_op=0 schedule=3 nodes=13 inputs=3 outputs=1 sinks=0
ok: true
source: bytes=48790 prefix=@runtime+@grouped tail_bytes=7248
kernel void ptir_m3_82921040f7bc2612_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 5, scratch + offsets[0], scratch + offsets[4], scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(1u, status, descriptors, lane_params + 6, scratch + offsets[5], scratch, scratch, scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 8, scratch + offsets[7], scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(19u, status, descriptors, lane_params + 9, scratch + offsets[6], scratch + offsets[8], scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(88u, status, descriptors, lane_params + 10, scratch + offsets[9], scratch + offsets[2], scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 11, scratch, scratch, scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 12, scratch + offsets[10], scratch + offsets[0], scratch + offsets[11], scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(113u, status, descriptors, lane_params + 13, scratch + offsets[1], scratch, scratch, scratch + offsets[13], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 14, scratch + offsets[12], scratch + offsets[13], scratch, scratch + offsets[14], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 15, scratch + offsets[14], scratch, scratch, scratch + offsets[15], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== nucleus_sample#0 fused#2
function_name: ptir_m3_82921040f7bc2612_r2
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=45422 prefix=@runtime+@grouped tail_bytes=3880
kernel void ptir_m3_82921040f7bc2612_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 16, scratch + offsets[15], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== pentathlon_iter#0 singleton#0
function_name: ptir_m3s_14f667a41eb30669_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: false
error: Metal binds only the logits buffer for intrinsics; intrinsic id 3 has no binding
=== end
=== pentathlon_iter#0 singleton#1
function_name: ptir_m3s_14f667a41eb30669_r1
region: library=1 library_op=5 schedule=3 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44327 prefix=@runtime+@grouped tail_bytes=2785
kernel void ptir_m3s_14f667a41eb30669_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(161u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#0 singleton#2
function_name: ptir_m3s_14f667a41eb30669_r2
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44316 prefix=@runtime+@grouped tail_bytes=2774
kernel void ptir_m3s_14f667a41eb30669_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 2, current_0, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#0 singleton#3
function_name: ptir_m3s_14f667a41eb30669_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44326 prefix=@runtime+@grouped tail_bytes=2784
kernel void ptir_m3s_14f667a41eb30669_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#0 singleton#4
function_name: ptir_m3s_14f667a41eb30669_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=44339 prefix=@runtime+@grouped tail_bytes=2797
kernel void ptir_m3s_14f667a41eb30669_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(88u, status, descriptors, lane_params + 4, scratch + offsets[1], scratch + offsets[3], scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#0 singleton#5
function_name: ptir_m3s_14f667a41eb30669_r5
region: library=1 library_op=5 schedule=3 nodes=1 inputs=1 outputs=0 sinks=0
ok: true
source: bytes=44314 prefix=@runtime+@grouped tail_bytes=2772
kernel void ptir_m3s_14f667a41eb30669_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(162u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, scratch, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#0 fused#0
function_name: ptir_m3_14f667a41eb30669_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: false
error: Metal binds only the logits buffer for intrinsics; intrinsic id 3 has no binding
=== end
=== pentathlon_iter#0 fused#1
function_name: ptir_m3_14f667a41eb30669_r1
region: library=1 library_op=5 schedule=3 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44326 prefix=@runtime+@grouped tail_bytes=2784
kernel void ptir_m3_14f667a41eb30669_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(161u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#0 fused#2
function_name: ptir_m3_14f667a41eb30669_r2
region: library=0 library_op=0 schedule=1 nodes=3 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44614 prefix=@runtime+@grouped tail_bytes=3072
kernel void ptir_m3_14f667a41eb30669_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 2, current_0, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(88u, status, descriptors, lane_params + 4, scratch + offsets[1], scratch + offsets[2], scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#0 fused#3
function_name: ptir_m3_14f667a41eb30669_r3
region: library=1 library_op=5 schedule=3 nodes=1 inputs=1 outputs=0 sinks=0
ok: true
source: bytes=44313 prefix=@runtime+@grouped tail_bytes=2771
kernel void ptir_m3_14f667a41eb30669_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(162u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, scratch, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#0
function_name: ptir_m3s_dff3bc6795a9dcdf_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=47269 prefix=@runtime+@grouped tail_bytes=5727
kernel void ptir_m3s_dff3bc6795a9dcdf_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== pentathlon_iter#1 singleton#1
function_name: ptir_m3s_dff3bc6795a9dcdf_r1
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=46216 prefix=@runtime+@grouped tail_bytes=4674
kernel void ptir_m3s_dff3bc6795a9dcdf_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 1, current_0, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#3
function_name: ptir_m3s_dff3bc6795a9dcdf_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=46226 prefix=@runtime+@grouped tail_bytes=4684
kernel void ptir_m3s_dff3bc6795a9dcdf_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#4
function_name: ptir_m3s_dff3bc6795a9dcdf_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=46226 prefix=@runtime+@grouped tail_bytes=4684
kernel void ptir_m3s_dff3bc6795a9dcdf_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#5
function_name: ptir_m3s_dff3bc6795a9dcdf_r5
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=46239 prefix=@runtime+@grouped tail_bytes=4697
kernel void ptir_m3s_dff3bc6795a9dcdf_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 5, scratch + offsets[0], scratch + offsets[4], scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#6
function_name: ptir_m3s_dff3bc6795a9dcdf_r6
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=46225 prefix=@runtime+@grouped tail_bytes=4683
kernel void ptir_m3s_dff3bc6795a9dcdf_r6(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(1u, status, descriptors, lane_params + 6, scratch + offsets[5], scratch, scratch, scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#7
function_name: ptir_m3s_dff3bc6795a9dcdf_r7
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=46226 prefix=@runtime+@grouped tail_bytes=4684
kernel void ptir_m3s_dff3bc6795a9dcdf_r7(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#8
function_name: ptir_m3s_dff3bc6795a9dcdf_r8
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=46225 prefix=@runtime+@grouped tail_bytes=4683
kernel void ptir_m3s_dff3bc6795a9dcdf_r8(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(2u, status, descriptors, lane_params + 8, scratch + offsets[7], scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#19
function_name: ptir_m3s_dff3bc6795a9dcdf_r19
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=46217 prefix=@runtime+@grouped tail_bytes=4675
kernel void ptir_m3s_dff3bc6795a9dcdf_r19(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 19, scratch, scratch, scratch, scratch + offsets[19], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#20
function_name: ptir_m3s_dff3bc6795a9dcdf_r20
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=46244 prefix=@runtime+@grouped tail_bytes=4702
kernel void ptir_m3s_dff3bc6795a9dcdf_r20(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(18u, status, descriptors, lane_params + 20, scratch + offsets[18], scratch + offsets[19], scratch, scratch + offsets[20], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#23
function_name: ptir_m3s_dff3bc6795a9dcdf_r23
region: library=0 library_op=0 schedule=1 nodes=1 inputs=3 outputs=1 sinks=0
ok: true
source: bytes=46257 prefix=@runtime+@grouped tail_bytes=4715
kernel void ptir_m3s_dff3bc6795a9dcdf_r23(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 23, scratch + offsets[2], scratch + offsets[10], scratch + offsets[22], scratch + offsets[23], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#25
function_name: ptir_m3s_dff3bc6795a9dcdf_r25
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=46230 prefix=@runtime+@grouped tail_bytes=4688
kernel void ptir_m3s_dff3bc6795a9dcdf_r25(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 25, scratch + offsets[24], scratch, scratch, scratch + offsets[25], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#28
function_name: ptir_m3s_dff3bc6795a9dcdf_r28
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=46244 prefix=@runtime+@grouped tail_bytes=4702
kernel void ptir_m3s_dff3bc6795a9dcdf_r28(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 28, scratch + offsets[26], scratch + offsets[27], scratch, scratch + offsets[28], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#29
function_name: ptir_m3s_dff3bc6795a9dcdf_r29
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=46244 prefix=@runtime+@grouped tail_bytes=4702
kernel void ptir_m3s_dff3bc6795a9dcdf_r29(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(23u, status, descriptors, lane_params + 29, scratch + offsets[23], scratch + offsets[28], scratch, scratch + offsets[29], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#31
function_name: ptir_m3s_dff3bc6795a9dcdf_r31
region: library=1 library_op=1 schedule=3 nodes=1 inputs=1 outputs=2 sinks=0
ok: true
source: bytes=46244 prefix=@runtime+@grouped tail_bytes=4702
kernel void ptir_m3s_dff3bc6795a9dcdf_r31(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(81u, status, descriptors, lane_params + 31, scratch + offsets[30], scratch, scratch, scratch + offsets[31], scratch + offsets[32], temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 singleton#34
function_name: ptir_m3s_dff3bc6795a9dcdf_r34
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=46374 prefix=@runtime+@grouped tail_bytes=4832
kernel void ptir_m3s_dff3bc6795a9dcdf_r34(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 34, scratch + offsets[33], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== pentathlon_iter#1 fused#0
function_name: ptir_m3_dff3bc6795a9dcdf_r0
region: library=0 library_op=0 schedule=1 nodes=31 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=55721 prefix=@runtime+@grouped tail_bytes=14179
kernel void ptir_m3_dff3bc6795a9dcdf_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 1, current_0, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 2, current_1, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 5, scratch + offsets[0], scratch + offsets[4], scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(1u, status, descriptors, lane_params + 6, scratch + offsets[5], scratch, scratch, scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(2u, status, descriptors, lane_params + 8, scratch + offsets[7], scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 9, scratch + offsets[8], scratch, scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 10, scratch + offsets[5], scratch + offsets[9], scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 11, scratch + offsets[1], scratch, scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 12, scratch + offsets[11], scratch, scratch, scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 13, scratch + offsets[1], scratch + offsets[12], scratch, scratch + offsets[13], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(1u, status, descriptors, lane_params + 14, scratch + offsets[13], scratch, scratch, scratch + offsets[14], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 15, scratch + offsets[14], scratch, scratch, scratch + offsets[15], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(2u, status, descriptors, lane_params + 16, scratch + offsets[15], scratch, scratch, scratch + offsets[16], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 17, scratch + offsets[16], scratch, scratch, scratch + offsets[17], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 18, scratch + offsets[13], scratch + offsets[17], scratch, scratch + offsets[18], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 19, scratch, scratch, scratch, scratch + offsets[19], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(18u, status, descriptors, lane_params + 20, scratch + offsets[18], scratch + offsets[19], scratch, scratch + offsets[20], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 21, scratch + offsets[10], scratch + offsets[20], scratch, scratch + offsets[21], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 22, scratch, scratch, scratch, scratch + offsets[22], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 23, scratch + offsets[2], scratch + offsets[10], scratch + offsets[22], scratch + offsets[23], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 24, scratch + offsets[23], scratch, scratch, scratch + offsets[24], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 26, scratch + offsets[24], scratch, scratch, scratch + offsets[26], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 27, scratch, scratch, scratch, scratch + offsets[27], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 28, scratch + offsets[26], scratch + offsets[27], scratch, scratch + offsets[28], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(23u, status, descriptors, lane_params + 29, scratch + offsets[23], scratch + offsets[28], scratch, scratch + offsets[29], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 30, scratch + offsets[29], scratch + offsets[21], scratch + offsets[22], scratch + offsets[30], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 fused#1
function_name: ptir_m3_dff3bc6795a9dcdf_r1
region: library=1 library_op=1 schedule=3 nodes=1 inputs=1 outputs=2 sinks=0
ok: true
source: bytes=46242 prefix=@runtime+@grouped tail_bytes=4700
kernel void ptir_m3_dff3bc6795a9dcdf_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(81u, status, descriptors, lane_params + 31, scratch + offsets[30], scratch, scratch, scratch + offsets[31], scratch + offsets[32], temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pentathlon_iter#1 fused#2
function_name: ptir_m3_dff3bc6795a9dcdf_r2
region: library=0 library_op=0 schedule=1 nodes=6 inputs=2 outputs=0 sinks=3
ok: false
error: Metal binds only the logits buffer for intrinsics; intrinsic id 4 has no binding
=== end
=== pivot_predicates_multistage#1 singleton#0
function_name: ptir_m3s_09a2719886970a85_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=46794 prefix=@runtime+@grouped tail_bytes=5252
kernel void ptir_m3s_09a2719886970a85_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== pivot_predicates_multistage#1 singleton#1
function_name: ptir_m3s_09a2719886970a85_r1
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45751 prefix=@runtime+@grouped tail_bytes=4209
kernel void ptir_m3s_09a2719886970a85_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pivot_predicates_multistage#1 singleton#2
function_name: ptir_m3s_09a2719886970a85_r2
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45751 prefix=@runtime+@grouped tail_bytes=4209
kernel void ptir_m3s_09a2719886970a85_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 2, scratch + offsets[1], scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pivot_predicates_multistage#1 singleton#3
function_name: ptir_m3s_09a2719886970a85_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45764 prefix=@runtime+@grouped tail_bytes=4222
kernel void ptir_m3s_09a2719886970a85_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch + offsets[2], scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pivot_predicates_multistage#1 singleton#4
function_name: ptir_m3s_09a2719886970a85_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45750 prefix=@runtime+@grouped tail_bytes=4208
kernel void ptir_m3s_09a2719886970a85_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(1u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pivot_predicates_multistage#1 singleton#5
function_name: ptir_m3s_09a2719886970a85_r5
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=45751 prefix=@runtime+@grouped tail_bytes=4209
kernel void ptir_m3s_09a2719886970a85_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pivot_predicates_multistage#1 singleton#7
function_name: ptir_m3s_09a2719886970a85_r7
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45764 prefix=@runtime+@grouped tail_bytes=4222
kernel void ptir_m3s_09a2719886970a85_r7(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(19u, status, descriptors, lane_params + 7, scratch + offsets[4], scratch + offsets[6], scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pivot_predicates_multistage#1 singleton#8
function_name: ptir_m3s_09a2719886970a85_r8
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45741 prefix=@runtime+@grouped tail_bytes=4199
kernel void ptir_m3s_09a2719886970a85_r8(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 8, current_0, scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pivot_predicates_multistage#1 singleton#10
function_name: ptir_m3s_09a2719886970a85_r10
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45767 prefix=@runtime+@grouped tail_bytes=4225
kernel void ptir_m3s_09a2719886970a85_r10(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(88u, status, descriptors, lane_params + 10, scratch + offsets[7], scratch + offsets[8], scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== pivot_predicates_multistage#1 singleton#12
function_name: ptir_m3s_09a2719886970a85_r12
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=45899 prefix=@runtime+@grouped tail_bytes=4357
kernel void ptir_m3s_09a2719886970a85_r12(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 12, scratch + offsets[10], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== pivot_predicates_multistage#1 fused#0
function_name: ptir_m3_09a2719886970a85_r0
region: library=0 library_op=0 schedule=1 nodes=14 inputs=0 outputs=0 sinks=2
ok: true
source: bytes=50840 prefix=@runtime+@grouped tail_bytes=9298
kernel void ptir_m3_09a2719886970a85_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 2, scratch + offsets[1], scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(17u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch + offsets[2], scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(1u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(48u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(56u, status, descriptors, lane_params + 6, scratch + offsets[5], scratch, scratch, scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(19u, status, descriptors, lane_params + 7, scratch + offsets[4], scratch + offsets[6], scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 8, current_0, scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 9, current_1, scratch, scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(88u, status, descriptors, lane_params + 10, scratch + offsets[7], scratch + offsets[8], scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(88u, status, descriptors, lane_params + 11, scratch + offsets[7], scratch + offsets[9], scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 12, scratch + offsets[10], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 13, scratch + offsets[11], scratch, scratch, pending_3, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_3 = pending_3;
  if (m3_tid == 0) pending_flags[pending_index_3] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== section3_masked_gumbel#0 singleton#0
function_name: ptir_m3s_4cf3882bf2fff913_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=47269 prefix=@runtime+@grouped tail_bytes=5727
kernel void ptir_m3s_4cf3882bf2fff913_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== section3_masked_gumbel#0 singleton#1
function_name: ptir_m3s_4cf3882bf2fff913_r1
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=46226 prefix=@runtime+@grouped tail_bytes=4684
kernel void ptir_m3s_4cf3882bf2fff913_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== section3_masked_gumbel#0 singleton#2
function_name: ptir_m3s_4cf3882bf2fff913_r2
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=46216 prefix=@runtime+@grouped tail_bytes=4674
kernel void ptir_m3s_4cf3882bf2fff913_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 2, current_0, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== section3_masked_gumbel#0 singleton#4
function_name: ptir_m3s_4cf3882bf2fff913_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=46227 prefix=@runtime+@grouped tail_bytes=4685
kernel void ptir_m3s_4cf3882bf2fff913_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(113u, status, descriptors, lane_params + 4, scratch + offsets[2], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== section3_masked_gumbel#0 singleton#5
function_name: ptir_m3s_4cf3882bf2fff913_r5
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=46214 prefix=@runtime+@grouped tail_bytes=4672
kernel void ptir_m3s_4cf3882bf2fff913_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 5, scratch, scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== section3_masked_gumbel#0 singleton#6
function_name: ptir_m3s_4cf3882bf2fff913_r6
region: library=0 library_op=0 schedule=1 nodes=1 inputs=3 outputs=1 sinks=0
ok: true
source: bytes=46252 prefix=@runtime+@grouped tail_bytes=4710
kernel void ptir_m3s_4cf3882bf2fff913_r6(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 6, scratch + offsets[3], scratch + offsets[1], scratch + offsets[5], scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== section3_masked_gumbel#0 singleton#7
function_name: ptir_m3s_4cf3882bf2fff913_r7
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=46239 prefix=@runtime+@grouped tail_bytes=4697
kernel void ptir_m3s_4cf3882bf2fff913_r7(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch + offsets[4], scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== section3_masked_gumbel#0 singleton#8
function_name: ptir_m3s_4cf3882bf2fff913_r8
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=46226 prefix=@runtime+@grouped tail_bytes=4684
kernel void ptir_m3s_4cf3882bf2fff913_r8(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 8, scratch + offsets[7], scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== section3_masked_gumbel#0 singleton#9
function_name: ptir_m3s_4cf3882bf2fff913_r9
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=46214 prefix=@runtime+@grouped tail_bytes=4672
kernel void ptir_m3s_4cf3882bf2fff913_r9(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 9, scratch, scratch, scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== section3_masked_gumbel#0 singleton#11
function_name: ptir_m3s_4cf3882bf2fff913_r11
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=46374 prefix=@runtime+@grouped tail_bytes=4832
kernel void ptir_m3s_4cf3882bf2fff913_r11(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 11, scratch + offsets[10], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== section3_masked_gumbel#0 fused#0
function_name: ptir_m3_4cf3882bf2fff913_r0
region: library=0 library_op=0 schedule=1 nodes=19 inputs=0 outputs=0 sinks=4
ok: true
source: bytes=52730 prefix=@runtime+@grouped tail_bytes=11188
kernel void ptir_m3_4cf3882bf2fff913_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  {
    const uint gather_begin = m3_tid;
    const uint gather_step = m3_threads;
    const M1ValueDesc intrinsic_desc = descriptors[0];
    const uint intrinsic_row_base = 0u;
    const uint gather_stride = layout->vocab;
    const uint gather_width = intrinsic_desc.last == 0u ? gather_stride : intrinsic_desc.last;
    if (gather_stride == 0u || gather_width == 0u || gather_stride < gather_width || intrinsic_desc.len % gather_width != 0u || intrinsic_row_base > row_meta.count || intrinsic_desc.len / gather_width > row_meta.count - intrinsic_row_base) { m1_fault(status, 0xA0u); return; }
    device float* intrinsic_out = reinterpret_cast<device float*>(scratch + offsets[0]);
    const uint gather_rows = intrinsic_desc.len / gather_width;
    const uint gather_row_base = row_meta.offset + intrinsic_row_base;
    for (uint gr = 0u; gr < gather_rows; ++gr) {
      const uint source_row = row_indices[gather_row_base + gr];
      const device bfloat* gather_src = logits + ulong(source_row) * gather_stride;
      device float* gather_dst = intrinsic_out + ulong(gr) * gather_width;
      for (uint column = gather_begin; column < gather_width; column += gather_step) {
        gather_dst[column] = float(gather_src[column]);
      }
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 2, current_0, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 3, current_1, scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(113u, status, descriptors, lane_params + 4, scratch + offsets[2], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 5, scratch, scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(32u, status, descriptors, lane_params + 6, scratch + offsets[3], scratch + offsets[0], scratch + offsets[5], scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 7, scratch + offsets[6], scratch + offsets[4], scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 8, scratch + offsets[7], scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 9, scratch, scratch, scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 10, scratch + offsets[2], scratch + offsets[9], scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 11, scratch + offsets[10], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 12, scratch + offsets[8], scratch, scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 13, scratch + offsets[11], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 14, current_3, scratch, scratch, scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 15, scratch, scratch, scratch, scratch + offsets[13], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 16, scratch + offsets[12], scratch + offsets[13], scratch, scratch + offsets[14], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 17, scratch + offsets[14], scratch, scratch, pending_3, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_3 = pending_3;
  if (m3_tid == 0) pending_flags[pending_index_3] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 18, scratch + offsets[11], scratch, scratch, pending_4, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_4 = pending_4;
  if (m3_tid == 0) pending_flags[pending_index_4] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== structured_masks#0 singleton#0
function_name: ptir_m3s_1752e0dfb0bf5827_r0
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=47166 prefix=@runtime+@grouped tail_bytes=5624
kernel void ptir_m3s_1752e0dfb0bf5827_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#3
function_name: ptir_m3s_1752e0dfb0bf5827_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=47177 prefix=@runtime+@grouped tail_bytes=5635
kernel void ptir_m3s_1752e0dfb0bf5827_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(102u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#4
function_name: ptir_m3s_1752e0dfb0bf5827_r4
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=47321 prefix=@runtime+@grouped tail_bytes=5779
kernel void ptir_m3s_1752e0dfb0bf5827_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, pending_3, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_3 = pending_3;
  if (m3_tid == 0) pending_flags[pending_index_3] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== structured_masks#0 singleton#5
function_name: ptir_m3s_1752e0dfb0bf5827_r5
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=47177 prefix=@runtime+@grouped tail_bytes=5635
kernel void ptir_m3s_1752e0dfb0bf5827_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(103u, status, descriptors, lane_params + 5, scratch + offsets[0], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#7
function_name: ptir_m3s_1752e0dfb0bf5827_r7
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=47177 prefix=@runtime+@grouped tail_bytes=5635
kernel void ptir_m3s_1752e0dfb0bf5827_r7(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(104u, status, descriptors, lane_params + 7, scratch + offsets[0], scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#9
function_name: ptir_m3s_1752e0dfb0bf5827_r9
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=47176 prefix=@runtime+@grouped tail_bytes=5634
kernel void ptir_m3s_1752e0dfb0bf5827_r9(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(57u, status, descriptors, lane_params + 9, scratch + offsets[1], scratch, scratch, scratch + offsets[6], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#10
function_name: ptir_m3s_1752e0dfb0bf5827_r10
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=47166 prefix=@runtime+@grouped tail_bytes=5624
kernel void ptir_m3s_1752e0dfb0bf5827_r10(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 10, scratch, scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#11
function_name: ptir_m3s_1752e0dfb0bf5827_r11
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=47166 prefix=@runtime+@grouped tail_bytes=5624
kernel void ptir_m3s_1752e0dfb0bf5827_r11(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 11, scratch, scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#12
function_name: ptir_m3s_1752e0dfb0bf5827_r12
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=47191 prefix=@runtime+@grouped tail_bytes=5649
kernel void ptir_m3s_1752e0dfb0bf5827_r12(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(19u, status, descriptors, lane_params + 12, scratch + offsets[7], scratch + offsets[8], scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#14
function_name: ptir_m3s_1752e0dfb0bf5827_r14
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=47193 prefix=@runtime+@grouped tail_bytes=5651
kernel void ptir_m3s_1752e0dfb0bf5827_r14(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(31u, status, descriptors, lane_params + 14, scratch + offsets[7], scratch + offsets[10], scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#15
function_name: ptir_m3s_1752e0dfb0bf5827_r15
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=47193 prefix=@runtime+@grouped tail_bytes=5651
kernel void ptir_m3s_1752e0dfb0bf5827_r15(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(18u, status, descriptors, lane_params + 15, scratch + offsets[9], scratch + offsets[10], scratch, scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#16
function_name: ptir_m3s_1752e0dfb0bf5827_r16
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=47194 prefix=@runtime+@grouped tail_bytes=5652
kernel void ptir_m3s_1752e0dfb0bf5827_r16(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 16, scratch + offsets[11], scratch + offsets[12], scratch, scratch + offsets[13], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#17
function_name: ptir_m3s_1752e0dfb0bf5827_r17
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=47193 prefix=@runtime+@grouped tail_bytes=5651
kernel void ptir_m3s_1752e0dfb0bf5827_r17(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 17, scratch + offsets[6], scratch + offsets[13], scratch, scratch + offsets[14], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#24
function_name: ptir_m3s_1752e0dfb0bf5827_r24
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=47194 prefix=@runtime+@grouped tail_bytes=5652
kernel void ptir_m3s_1752e0dfb0bf5827_r24(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(24u, status, descriptors, lane_params + 24, scratch + offsets[19], scratch + offsets[20], scratch, scratch + offsets[21], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#25
function_name: ptir_m3s_1752e0dfb0bf5827_r25
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=47179 prefix=@runtime+@grouped tail_bytes=5637
kernel void ptir_m3s_1752e0dfb0bf5827_r25(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 25, scratch + offsets[21], scratch, scratch, scratch + offsets[22], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 singleton#26
function_name: ptir_m3s_1752e0dfb0bf5827_r26
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=47180 prefix=@runtime+@grouped tail_bytes=5638
kernel void ptir_m3s_1752e0dfb0bf5827_r26(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 26, scratch + offsets[22], scratch, scratch, scratch + offsets[23], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== structured_masks#0 fused#0
function_name: ptir_m3_1752e0dfb0bf5827_r0
region: library=0 library_op=0 schedule=1 nodes=29 inputs=0 outputs=0 sinks=4
ok: true
source: bytes=54990 prefix=@runtime+@grouped tail_bytes=13448
kernel void ptir_m3_1752e0dfb0bf5827_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  const uint dense_3 = channel_bindings[dispatch_lane * layout->reserved0 + 3];
  const M3LaneChannelSlot channel_3 = slots[lane.channel_slot_offset + dense_3];
  const uint pending_index_3 = lane.channel_slot_offset + dense_3;
  const device uchar* current_3 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_3] != 0 ? channel_3.pending_cell : channel_3.committed_cell);
  device uchar* pending_3 = reinterpret_cast<device uchar*>(channel_3.pending_cell);
  const uint dense_4 = channel_bindings[dispatch_lane * layout->reserved0 + 4];
  const M3LaneChannelSlot channel_4 = slots[lane.channel_slot_offset + dense_4];
  const uint pending_index_4 = lane.channel_slot_offset + dense_4;
  const device uchar* current_4 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_4] != 0 ? channel_4.pending_cell : channel_4.committed_cell);
  device uchar* pending_4 = reinterpret_cast<device uchar*>(channel_4.pending_cell);
  const uint dense_5 = channel_bindings[dispatch_lane * layout->reserved0 + 5];
  const M3LaneChannelSlot channel_5 = slots[lane.channel_slot_offset + dense_5];
  const uint pending_index_5 = lane.channel_slot_offset + dense_5;
  const device uchar* current_5 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_5] != 0 ? channel_5.pending_cell : channel_5.committed_cell);
  device uchar* pending_5 = reinterpret_cast<device uchar*>(channel_5.pending_cell);
  const uint dense_6 = channel_bindings[dispatch_lane * layout->reserved0 + 6];
  const M3LaneChannelSlot channel_6 = slots[lane.channel_slot_offset + dense_6];
  const uint pending_index_6 = lane.channel_slot_offset + dense_6;
  const device uchar* current_6 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_6] != 0 ? channel_6.pending_cell : channel_6.committed_cell);
  device uchar* pending_6 = reinterpret_cast<device uchar*>(channel_6.pending_cell);
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 1, current_1, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(144u, status, descriptors, lane_params + 2, current_2, scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(102u, status, descriptors, lane_params + 3, scratch + offsets[0], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 4, scratch + offsets[3], scratch, scratch, pending_3, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_3 = pending_3;
  if (m3_tid == 0) pending_flags[pending_index_3] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(103u, status, descriptors, lane_params + 5, scratch + offsets[0], scratch, scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 6, scratch + offsets[4], scratch, scratch, pending_4, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_4 = pending_4;
  if (m3_tid == 0) pending_flags[pending_index_4] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(104u, status, descriptors, lane_params + 7, scratch + offsets[0], scratch, scratch, scratch + offsets[5], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 8, scratch + offsets[5], scratch, scratch, pending_5, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_5 = pending_5;
  if (m3_tid == 0) pending_flags[pending_index_5] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(100u, status, descriptors, lane_params + 10, scratch, scratch, scratch, scratch + offsets[7], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 11, scratch, scratch, scratch, scratch + offsets[8], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(19u, status, descriptors, lane_params + 12, scratch + offsets[7], scratch + offsets[8], scratch, scratch + offsets[9], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 13, scratch, scratch, scratch, scratch + offsets[10], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(31u, status, descriptors, lane_params + 14, scratch + offsets[7], scratch + offsets[10], scratch, scratch + offsets[11], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(18u, status, descriptors, lane_params + 15, scratch + offsets[9], scratch + offsets[10], scratch, scratch + offsets[12], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 16, scratch + offsets[11], scratch + offsets[12], scratch, scratch + offsets[13], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 17, scratch + offsets[1], scratch + offsets[13], scratch, scratch + offsets[14], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(19u, status, descriptors, lane_params + 18, scratch + offsets[7], scratch + offsets[10], scratch, scratch + offsets[15], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(129u, status, descriptors, lane_params + 19, scratch, scratch, scratch, scratch + offsets[16], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(31u, status, descriptors, lane_params + 20, scratch + offsets[15], scratch + offsets[16], scratch, scratch + offsets[17], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(96u, status, descriptors, lane_params + 21, scratch + offsets[2], scratch + offsets[17], scratch, scratch + offsets[18], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(24u, status, descriptors, lane_params + 24, scratch + offsets[14], scratch + offsets[18], scratch, scratch + offsets[21], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 25, scratch + offsets[21], scratch, scratch, scratch + offsets[22], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(49u, status, descriptors, lane_params + 26, scratch + offsets[22], scratch, scratch, scratch + offsets[23], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 27, scratch + offsets[23], scratch, scratch, scratch + offsets[24], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 28, scratch + offsets[24], scratch, scratch, pending_6, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_6 = pending_6;
  if (m3_tid == 0) pending_flags[pending_index_6] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== synthetic_sort_desc#0 singleton#0
function_name: ptir_m3s_dc132d209d85d958_r0
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45266 prefix=@runtime+@grouped tail_bytes=3724
kernel void ptir_m3s_dc132d209d85d958_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_sort_desc#0 singleton#1
function_name: ptir_m3s_dc132d209d85d958_r1
region: library=1 library_op=2 schedule=3 nodes=1 inputs=1 outputs=2 sinks=0
ok: true
source: bytes=45289 prefix=@runtime+@grouped tail_bytes=3747
kernel void ptir_m3s_dc132d209d85d958_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(80u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch + offsets[1], scratch + offsets[2], temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_sort_desc#0 singleton#2
function_name: ptir_m3s_dc132d209d85d958_r2
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=45421 prefix=@runtime+@grouped tail_bytes=3879
kernel void ptir_m3s_dc132d209d85d958_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 2, scratch + offsets[1], scratch, scratch, pending_1, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_1 = pending_1;
  if (m3_tid == 0) pending_flags[pending_index_1] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== synthetic_sort_desc#0 fused#0
function_name: ptir_m3_dc132d209d85d958_r0
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45265 prefix=@runtime+@grouped tail_bytes=3723
kernel void ptir_m3_dc132d209d85d958_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_sort_desc#0 fused#1
function_name: ptir_m3_dc132d209d85d958_r1
region: library=1 library_op=2 schedule=3 nodes=1 inputs=1 outputs=2 sinks=0
ok: true
source: bytes=45288 prefix=@runtime+@grouped tail_bytes=3746
kernel void ptir_m3_dc132d209d85d958_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(80u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch + offsets[1], scratch + offsets[2], temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_sort_desc#0 fused#2
function_name: ptir_m3_dc132d209d85d958_r2
region: library=0 library_op=0 schedule=0 nodes=2 inputs=2 outputs=0 sinks=2
ok: true
source: bytes=45851 prefix=@runtime+@grouped tail_bytes=4309
kernel void ptir_m3_dc132d209d85d958_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 2, scratch + offsets[1], scratch, scratch, pending_1, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_1 = pending_1;
  if (m3_tid == 0) pending_flags[pending_index_1] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== synthetic_matmul#0 singleton#0
function_name: ptir_m3s_92d86f95b487e729_r0
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45266 prefix=@runtime+@grouped tail_bytes=3724
kernel void ptir_m3s_92d86f95b487e729_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_matmul#0 singleton#2
function_name: ptir_m3s_92d86f95b487e729_r2
region: library=1 library_op=4 schedule=3 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45289 prefix=@runtime+@grouped tail_bytes=3747
kernel void ptir_m3s_92d86f95b487e729_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(85u, status, descriptors, lane_params + 2, scratch + offsets[0], scratch + offsets[1], scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_matmul#0 singleton#3
function_name: ptir_m3s_92d86f95b487e729_r3
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=45421 prefix=@runtime+@grouped tail_bytes=3879
kernel void ptir_m3s_92d86f95b487e729_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== synthetic_matmul#0 fused#0
function_name: ptir_m3_92d86f95b487e729_r0
region: library=0 library_op=0 schedule=0 nodes=2 inputs=0 outputs=2 sinks=0
ok: true
source: bytes=45541 prefix=@runtime+@grouped tail_bytes=3999
kernel void ptir_m3_92d86f95b487e729_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 1, current_1, scratch, scratch, scratch + offsets[1], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_matmul#0 fused#1
function_name: ptir_m3_92d86f95b487e729_r1
region: library=1 library_op=4 schedule=3 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=45288 prefix=@runtime+@grouped tail_bytes=3746
kernel void ptir_m3_92d86f95b487e729_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(85u, status, descriptors, lane_params + 2, scratch + offsets[0], scratch + offsets[1], scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_matmul#0 fused#2
function_name: ptir_m3_92d86f95b487e729_r2
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=45420 prefix=@runtime+@grouped tail_bytes=3878
kernel void ptir_m3_92d86f95b487e729_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  const uint dense_1 = channel_bindings[dispatch_lane * layout->reserved0 + 1];
  const M3LaneChannelSlot channel_1 = slots[lane.channel_slot_offset + dense_1];
  const uint pending_index_1 = lane.channel_slot_offset + dense_1;
  const device uchar* current_1 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_1] != 0 ? channel_1.pending_cell : channel_1.committed_cell);
  device uchar* pending_1 = reinterpret_cast<device uchar*>(channel_1.pending_cell);
  const uint dense_2 = channel_bindings[dispatch_lane * layout->reserved0 + 2];
  const M3LaneChannelSlot channel_2 = slots[lane.channel_slot_offset + dense_2];
  const uint pending_index_2 = lane.channel_slot_offset + dense_2;
  const device uchar* current_2 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_2] != 0 ? channel_2.pending_cell : channel_2.committed_cell);
  device uchar* pending_2 = reinterpret_cast<device uchar*>(channel_2.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, pending_2, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_2 = pending_2;
  if (m3_tid == 0) pending_flags[pending_index_2] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== synthetic_mtp_drafts#0 singleton#0
function_name: ptir_m3s_9eda8aa42400c4c2_r0
region: library=0 library_op=0 schedule=1 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=45141 prefix=@runtime+@grouped tail_bytes=3599
kernel void ptir_m3s_9eda8aa42400c4c2_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  {
    const uint draft_begin = m3_tid;
    const uint draft_step = m3_threads;
    const M1ValueDesc draft_desc = descriptors[0];
    if (layout->vocab == 0u || row_meta.mtp_offset > row_meta.count || draft_desc.len > row_meta.count - row_meta.mtp_offset) { m1_fault(status, 0xA0u); return; }
    device int* draft_out = reinterpret_cast<device int*>(scratch + offsets[0]);
    for (uint draft = draft_begin; draft < draft_desc.len; draft += draft_step) {
      const uint source_row = row_indices[row_meta.offset + row_meta.mtp_offset + draft];
      float best_value = -INFINITY;
      uint best_index = 0u;
      bool have = false;
      for (uint column = 0; column < layout->vocab; ++column) {
        const float value = float(logits[ulong(source_row) * layout->vocab + column]);
        if (!isnan(value) && (!have || value > best_value || (value == best_value && column < best_index))) { best_value = value; best_index = column; have = true; }
      }
      draft_out[draft] = int(have ? best_index : 0u);
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== synthetic_mtp_drafts#0 singleton#2
function_name: ptir_m3s_9eda8aa42400c4c2_r2
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44326 prefix=@runtime+@grouped tail_bytes=2784
kernel void ptir_m3s_9eda8aa42400c4c2_r2(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(51u, status, descriptors, lane_params + 2, scratch + offsets[1], scratch, scratch, scratch + offsets[2], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_mtp_drafts#0 singleton#3
function_name: ptir_m3s_9eda8aa42400c4c2_r3
region: library=0 library_op=0 schedule=1 nodes=1 inputs=1 outputs=1 sinks=0
ok: true
source: bytes=44325 prefix=@runtime+@grouped tail_bytes=2783
kernel void ptir_m3s_9eda8aa42400c4c2_r3(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_mtp_drafts#0 singleton#4
function_name: ptir_m3s_9eda8aa42400c4c2_r4
region: library=0 library_op=0 schedule=1 nodes=1 inputs=2 outputs=1 sinks=0
ok: true
source: bytes=44339 prefix=@runtime+@grouped tail_bytes=2797
kernel void ptir_m3s_9eda8aa42400c4c2_r4(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 4, scratch + offsets[0], scratch + offsets[3], scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_mtp_drafts#0 singleton#5
function_name: ptir_m3s_9eda8aa42400c4c2_r5
region: library=0 library_op=0 schedule=0 nodes=1 inputs=1 outputs=0 sinks=1
ok: true
source: bytes=44471 prefix=@runtime+@grouped tail_bytes=2929
kernel void ptir_m3s_9eda8aa42400c4c2_r5(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== synthetic_mtp_drafts#0 fused#0
function_name: ptir_m3_9eda8aa42400c4c2_r0
region: library=0 library_op=0 schedule=1 nodes=6 inputs=0 outputs=0 sinks=1
ok: true
source: bytes=48598 prefix=@runtime+@grouped tail_bytes=7056
kernel void ptir_m3_9eda8aa42400c4c2_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  {
    const uint draft_begin = m3_tid;
    const uint draft_step = m3_threads;
    const M1ValueDesc draft_desc = descriptors[0];
    if (layout->vocab == 0u || row_meta.mtp_offset > row_meta.count || draft_desc.len > row_meta.count - row_meta.mtp_offset) { m1_fault(status, 0xA0u); return; }
    device int* draft_out = reinterpret_cast<device int*>(scratch + offsets[0]);
    for (uint draft = draft_begin; draft < draft_desc.len; draft += draft_step) {
      const uint source_row = row_indices[row_meta.offset + row_meta.mtp_offset + draft];
      float best_value = -INFINITY;
      uint best_index = 0u;
      bool have = false;
      for (uint column = 0; column < layout->vocab; ++column) {
        const float value = float(logits[ulong(source_row) * layout->vocab + column]);
        if (!isnan(value) && (!have || value > best_value || (value == best_value && column < best_index))) { best_value = value; best_index = column; have = true; }
      }
      draft_out[draft] = int(have ? best_index : 0u);
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  {
    const M1ValueDesc am_in = descriptors[1];
    const uint am_row_base = row_meta.offset + row_meta.mtp_offset;
    const uint am_vocab = layout->vocab;
    const uint am_width = am_in.last == 0u ? am_vocab : am_in.last;
    if (am_vocab == 0u || am_width == 0u || am_width > am_vocab || am_in.rows > row_meta.count) { m1_fault(status, 0xA0u); return; }
    device int* am_out = reinterpret_cast<device int*>(scratch + offsets[2]);
    for (uint am_r = 0u; am_r < am_in.rows; ++am_r) {
      const uint am_src_row = row_indices[am_row_base + am_r];
      const device bfloat* am_src = logits + ulong(am_src_row) * am_vocab;
      M1ArgmaxCandidate am_best = {-INFINITY, 0u, 0u, 0u};
      M1ArgmaxCandidate am_b1 = am_best, am_b2 = am_best, am_b3 = am_best;
      const uint am_w = m3_threads;
      uint am_c = m3_tid;
      for (; am_c + 3u * am_w < am_width; am_c += 4u * am_w) {
        const float v0 = float(am_src[am_c]);
        const float v1 = float(am_src[am_c + am_w]);
        const float v2 = float(am_src[am_c + 2u * am_w]);
        const float v3 = float(am_src[am_c + 3u * am_w]);
        am_best = m1_argmax_combine(am_best, M1ArgmaxCandidate{v0, am_c, isnan(v0) ? 0u : 1u, 0u});
        am_b1 = m1_argmax_combine(am_b1, M1ArgmaxCandidate{v1, am_c + am_w, isnan(v1) ? 0u : 1u, 0u});
        am_b2 = m1_argmax_combine(am_b2, M1ArgmaxCandidate{v2, am_c + 2u * am_w, isnan(v2) ? 0u : 1u, 0u});
        am_b3 = m1_argmax_combine(am_b3, M1ArgmaxCandidate{v3, am_c + 3u * am_w, isnan(v3) ? 0u : 1u, 0u});
      }
      for (; am_c < am_width; am_c += am_w) {
        const float am_v = float(am_src[am_c]);
        am_best = m1_argmax_combine(am_best, M1ArgmaxCandidate{am_v, am_c, isnan(am_v) ? 0u : 1u, 0u});
      }
      am_best = m1_argmax_combine(m1_argmax_combine(am_best, am_b1), m1_argmax_combine(am_b2, am_b3));
      m3_tgbuf[m3_tid] = am_best;
      threadgroup_barrier(mem_flags::mem_threadgroup);
      for (uint am_s = 1u; am_s < m3_threads; am_s <<= 1) {
        if ((m3_tid % (2u * am_s)) == 0u && m3_tid + am_s < m3_threads) m3_tgbuf[m3_tid] = m1_argmax_combine(m3_tgbuf[m3_tid], m3_tgbuf[m3_tid + am_s]);
        threadgroup_barrier(mem_flags::mem_threadgroup);
      }
      if (m3_tid == 0) am_out[am_r] = int(m3_tgbuf[0].index);
      threadgroup_barrier(mem_flags::mem_threadgroup);
    }
  }
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(7u, status, descriptors, lane_params + 3, scratch + offsets[2], scratch, scratch, scratch + offsets[3], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(16u, status, descriptors, lane_params + 4, scratch + offsets[0], scratch + offsets[3], scratch, scratch + offsets[4], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  ptir_m1_execute_mt(146u, status, descriptors, lane_params + 5, scratch + offsets[4], scratch, scratch, pending_0, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
  current_0 = pending_0;
  if (m3_tid == 0) pending_flags[pending_index_0] = 1;
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
}
=== end
=== synthetic_sink_call#0 singleton#0
function_name: ptir_m3s_a297c60f7a5de47e_r0
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44316 prefix=@runtime+@grouped tail_bytes=2774
kernel void ptir_m3s_a297c60f7a5de47e_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_sink_call#0 singleton#1
function_name: ptir_m3s_a297c60f7a5de47e_r1
region: library=1 library_op=5 schedule=3 nodes=1 inputs=1 outputs=0 sinks=0
ok: true
source: bytes=44314 prefix=@runtime+@grouped tail_bytes=2772
kernel void ptir_m3s_a297c60f7a5de47e_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(162u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_sink_call#0 fused#0
function_name: ptir_m3_a297c60f7a5de47e_r0
region: library=0 library_op=0 schedule=0 nodes=1 inputs=0 outputs=1 sinks=0
ok: true
source: bytes=44315 prefix=@runtime+@grouped tail_bytes=2773
kernel void ptir_m3_a297c60f7a5de47e_r0(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(145u, status, descriptors, lane_params + 0, current_0, scratch, scratch, scratch + offsets[0], scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
=== synthetic_sink_call#0 fused#1
function_name: ptir_m3_a297c60f7a5de47e_r1
region: library=1 library_op=5 schedule=3 nodes=1 inputs=1 outputs=0 sinks=0
ok: true
source: bytes=44313 prefix=@runtime+@grouped tail_bytes=2771
kernel void ptir_m3_a297c60f7a5de47e_r1(
    const device uchar* lane_bytes [[buffer(0)]],
    const device M1ValueDesc* all_descriptors [[buffer(1)]],
    const device M1OpParams* params [[buffer(2)]],
    const device uint* offsets [[buffer(3)]],
    device uchar* all_scratch [[buffer(4)]],
    const device M3GroupLayout* layout [[buffer(5)]],
    const device uint* channel_bindings [[buffer(6)]],
    device uchar* pending_flags [[buffer(7)]],
    const device uint* lane_indices [[buffer(8)]],
    const device M3RowMeta* all_row_meta [[buffer(9)]],
    const device uint* row_indices [[buffer(10)]],
    uint dispatch_lane [[threadgroup_position_in_grid]],
    uint m3_tid [[thread_position_in_threadgroup]],
    uint m3_threads [[threads_per_threadgroup]]) {
  threadgroup M1ArgmaxCandidate m3_tgbuf[512];
  if (dispatch_lane >= layout->lane_count) return;
  const uint lane_index = lane_indices[dispatch_lane];
  const device M3LaneHeader* header = reinterpret_cast<const device M3LaneHeader*>(lane_bytes);
  const device M3LaneRecord* lanes = reinterpret_cast<const device M3LaneRecord*>(lane_bytes + sizeof(M3LaneHeader));
  const device M3LaneChannelSlot* slots = reinterpret_cast<const device M3LaneChannelSlot*>(lane_bytes + sizeof(M3LaneHeader) + header->lane_count * sizeof(M3LaneRecord));
  const M3LaneRecord lane = lanes[lane_index];
  const M3RowMeta row_meta = all_row_meta[lane_index];
  device M1Status* status = reinterpret_cast<device M1Status*>(lane.commit_slot);
  if (status->state != 1) return;
  if (m3_threads > 512u) { if (m3_tid == 0) m1_fault(status, 0xB3u); return; }
  const device M1ValueDesc* descriptors = all_descriptors + dispatch_lane * layout->value_count;
  const device M1OpParams* lane_params = params + dispatch_lane * layout->reserved2;
  device uchar* scratch = all_scratch + dispatch_lane * layout->scratch_stride;
  device uchar* temporary = scratch + layout->temporary_offset;
  const device bfloat* logits = reinterpret_cast<const device bfloat*>(lane.logits_base);
  const uint dense_0 = channel_bindings[dispatch_lane * layout->reserved0 + 0];
  const M3LaneChannelSlot channel_0 = slots[lane.channel_slot_offset + dense_0];
  const uint pending_index_0 = lane.channel_slot_offset + dense_0;
  const device uchar* current_0 = reinterpret_cast<const device uchar*>(pending_flags[pending_index_0] != 0 ? channel_0.pending_cell : channel_0.committed_cell);
  device uchar* pending_0 = reinterpret_cast<device uchar*>(channel_0.pending_cell);
  ptir_m1_execute_mt(162u, status, descriptors, lane_params + 1, scratch + offsets[0], scratch, scratch, scratch, scratch, temporary, m3_tid, m3_threads, m3_tgbuf);
  threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup);
  if (status->state != 1) return;
}
=== end
