{% extends "base.html" %}

{% block title %}Evalground - Memorizz{% endblock %}
{% block head %}<link rel="stylesheet" href="/static/css/analytics.css?v=1"><link rel="stylesheet" href="/static/css/evalground-runs.css?v=2"><link rel="stylesheet" href="/static/css/pages/evalground.css?v=2">{% endblock %}

{% block content %}
{% from '_ui.html' import page_header, evalground_tabs %}
{% call page_header('Evalground', 'Choose models and retrieval strategies with evidence from your workloads.', 'Evaluate', nav=evalground_tabs('evaluations'), actions_class='evaluation-actions') %}<a class="btn btn-primary" href="/evalground/compare">New comparison</a>
    <a class="btn" href="#agent-evaluation" data-open-section="agent-evaluation">Evaluate an agent</a>
    <a class="btn" href="#terminal-bench" data-open-section="terminal-bench">Test a harness</a>{% endcall %}
{#- Filled from the run library by evalground-runs.js; placeholders keep the layout steady while it loads. -#}
<div id="run-library-stats" class="fleet-tape eval-tape" role="list" aria-label="Evaluation summary" aria-busy="true">
    {% for label in ['Evaluations', 'In progress', 'Completed', 'Needs attention', 'Known API cost', 'Latest run'] %}<div role="listitem"><span>{{ label }}</span><strong>—</strong></div>{% endfor %}
</div>
<div class="evaluation-starts" aria-label="Start an evaluation">
    <a href="/evalground/compare?type=reader"><strong>Compare answer models <span aria-hidden="true">↗</span></strong><p>Same evidence. Compare answer quality, latency and API cost.</p></a>
    <a href="/evalground/compare?type=reranker"><strong>Compare rerankers <span aria-hidden="true">↗</span></strong><p>Same candidates. Measure which method finds the right evidence.</p></a>
    <a href="#agent-evaluation" data-open-section="agent-evaluation"><strong>Evaluate an agent <span aria-hidden="true">↗</span></strong><p>Run a memory benchmark against an agent or retrieval pipeline.</p></a>
    <a href="#terminal-bench" data-open-section="terminal-bench"><strong>Test a harness <span aria-hidden="true">↗</span></strong><p>Run Codex, Claude Code or a MemAgent on Terminal-Bench 4.0 tasks in Docker.</p></a>
</div>

<section id="run-library" class="detail-section evalground-section eval-library">
    <div class="run-library-heading"><h2>Recent evaluations</h2><p class="form-hint">Search, compare and revisit your evaluation results.</p></div>
    <div class="fleet-bar eval-bar">
        <label class="fleet-search">
            <kbd aria-hidden="true">/</kbd>
            <input id="run-search" type="search" placeholder="Name, model, agent or run ID" aria-label="Search evaluations" autocomplete="off" data-monitor-filter-for="run-library-table">
        </label>
        <label class="fleet-order">Run type<select id="run-kind"><option value="all">All run types</option><option value="reranker">Reranking experiments</option><option value="reader">Answer-model experiments</option><option value="pipeline">Pipeline experiments</option><option value="agent">Agent evaluations</option><option value="harness">Harness benchmarks</option></select></label>
        <label class="fleet-order">Status<select id="run-status"><option value="all">All statuses</option><option value="active">In progress</option><option value="completed">Completed</option><option value="attention">Needs attention</option></select></label>
        <label class="fleet-order">Sort by<select id="run-sort"><option value="newest">Newest first</option><option value="oldest">Oldest first</option><option value="cost">Highest API cost</option><option value="column" hidden disabled>Column order</option></select></label>
        <button type="button" id="refresh-run-library" class="btn btn-sm">Refresh runs</button>
        <p class="fleet-keys" aria-label="Keyboard shortcuts"><span><kbd>j</kbd> <kbd>k</kbd> move</span><span><kbd>↵</kbd> open result</span></p>
    </div>
    <div class="fleet-table-wrap eval-runs-wrap">
        <table id="run-library-table" class="fleet-table eval-runs-table">
            <caption class="visually-hidden">Saved model comparisons and agent evaluations. Select a row and press Enter to open its result.</caption>
            <thead>
                <tr>
                    <th scope="col" aria-sort="none"><button type="button" data-run-sort="name" data-sort-type="text">Experiment / agent</button></th>
                    <th scope="col" class="eval-col-models">Models</th>
                    <th scope="col" class="eval-col-type" aria-sort="none"><button type="button" data-run-sort="kind" data-sort-type="text">Type</button></th>
                    <th scope="col" aria-sort="none"><button type="button" data-run-sort="status" data-sort-type="text">Status</button></th>
                    <th scope="col" class="num eval-col-progress" aria-sort="none"><button type="button" data-run-sort="progress">Progress</button></th>
                    <th scope="col" class="num eval-col-quality" aria-sort="none"><button type="button" data-run-sort="quality">Quality</button></th>
                    <th scope="col" class="num" aria-sort="none"><button type="button" data-run-sort="cost">API cost</button></th>
                    <th scope="col" class="num eval-col-started" aria-sort="descending"><button type="button" data-run-sort="created">Started</button></th>
                    <th scope="col" class="eval-col-result">Result</th>
                </tr>
            </thead>
            <tbody id="run-library-rows"></tbody>
        </table>
    </div>
    <div class="run-library-pagination"><p id="run-library-message" role="status">Loading saved runs…</p><button id="run-prev" type="button" class="btn btn-sm">Previous</button><span id="run-page"></span><button id="run-next" type="button" class="btn btn-sm">Next</button></div>
    <details class="history-scope"><summary>About these measurements</summary><p id="run-library-scope" class="form-hint"></p></details>
</section>

{% if warnings %}
<div class="warning-message">
    <strong>Heads up</strong>
    <ul>
        {% for warning in warnings %}
        <li>{{ warning }}</li>
        {% endfor %}
    </ul>
</div>
{% endif %}

{% if error %}
<div class="error-message">
    <span>!</span> {{ error }}
</div>
{% endif %}

<details id="suggested-experiments" class="detail-section evalground-section evaluation-resource"><summary>Suggested experiments</summary>
    <p class="form-hint">Accepted Trace Insights become versioned drafts here. A draft is evidence for an evaluation plan; it does not mutate the production agent.</p>
    {% if trace_experiments %}
    <div class="traces-table-wrap">
        <table class="traces-agents-table">
            <thead>
                <tr>
                    <th>Experiment</th>
                    <th>Version</th>
                    <th>Status</th>
                    <th>Agent</th>
                    <th>Component</th>
                    <th>Target</th>
                    <th>Created</th>
                </tr>
            </thead>
            <tbody>
                {% for experiment in trace_experiments %}
                <tr>
                    <td><code>{{ experiment.experiment_id|truncate(24) }}</code></td>
                    <td>v{{ experiment.experiment_version }}</td>
                    <td><span class="badge badge-warning">{{ experiment.status|default('draft')|title }}</span></td>
                    <td><code>{{ experiment.agent_id|default('—')|truncate(16) }}</code></td>
                    <td>{{ experiment.component or '—' }}</td>
                    <td>{{ experiment.target or '—' }}</td>
                    <td>{{ experiment.created_at or '—' }}</td>
                </tr>
                {% endfor %}
            </tbody>
        </table>
    </div>
    {% else %}
    <p class="empty-hint">No accepted trace recommendations yet.</p>
    {% endif %}
</details>

<details id="dataset-library" class="detail-section evalground-section evaluation-resource"><summary>Benchmark datasets</summary>
    <p class="form-hint">Official datasets remain external. Configure each environment variable or enter a path for the run.</p>
    <div class="dataset-status">
        {% for item in benchmark_catalog %}
        <div class="dataset-row">
            <div>
                <div class="dataset-name"><a href="{{ item.paper_url }}" target="_blank" rel="noopener">{{ item.name }}</a></div>
                <div class="dataset-file">{{ item.variants|join(', ') }} · protocol {{ item.protocol_version }} · <code>{{ item.dataset_env }}</code></div>
                {% if item.source_sync_supported %}
                <div class="dataset-file"><code>memorizz eval dataset sync {{ item.benchmark_id }}</code></div>
                {% endif %}
            </div>
            <div class="dataset-meta">
                {% if item.dataset_ready %}
                <span class="badge badge-success">Ready</span>
                {% else %}
                <span class="badge badge-warning">Path needed</span>
                {% endif %}
                <span class="badge">Diagnostic</span>
            </div>
        </div>
        {% endfor %}
    </div>

    {% if download_message %}
    <div class="success-message">{{ download_message }}</div>
    {% endif %}

    {% if download_error %}
    <div class="error-message">
        <span>!</span> {{ download_error }}
    </div>
    {% endif %}

    <details class="dataset-actions">
        <summary>Legacy LongMemEval dataset helper</summary>
        <div class="dataset-status">
            {% for item in dataset_status %}
            <div class="dataset-row">
                <div class="dataset-name">LongMemEval {{ item.variant|upper }}</div>
                <div class="dataset-meta"><span class="dataset-file">{{ item.filename }}</span> <span class="badge {% if item.exists %}badge-success{% else %}badge-warning{% endif %}">{% if item.exists %}Ready{% else %}Missing{% endif %}</span></div>
            </div>
            {% endfor %}
        </div>
        {% if missing_variants %}
        <form method="post" action="/evalground/datasets" class="dataset-actions">
            <button type="submit" class="btn">Download legacy datasets</button>
        </form>
        {% endif %}
    </details>
</details>

<details id="agent-evaluation" class="detail-section evalground-section evaluation-resource"{% if selected_run_id %} open{% endif %}><summary>Configure an agent evaluation</summary>
    <form method="post" action="/evalground/runs" class="evalground-form">
        <div class="form-grid">
            <div class="form-group">
                <label for="evaluation_mode">Execution Mode</label>
                <select name="evaluation_mode" id="evaluation_mode">
                    <option value="retrieval" {% if (evaluation_mode or 'retrieval') == 'retrieval' %}selected{% endif %}>Memory retrieval diagnostic</option>
                    <option value="memagent" {% if evaluation_mode == 'memagent' %}selected{% endif %}>Full MemAgent execution</option>
                </select>
                <span class="form-hint">Diagnostic mode isolates retrieval and reader quality. Full mode runs the selected agent through <code>MemAgent.run()</code> on isolated benchmark memory.</span>
            </div>
            <div class="form-group">
                <label for="agent_id">Agent</label>
                <select name="agent_id" id="agent_id">
                    <option value="">No agent · diagnostic defaults</option>
                    {% for agent in agents %}
                    {% set agent_name = agent.persona.name if agent.persona and agent.persona.name else 'Agent' %}
                    <option value="{{ agent.agent_id }}" {% if agent.agent_id == selected_agent_id %}selected{% endif %}>
                        {{ agent_name }} ({{ agent.agent_id[:8] }}...)
                    </option>
                    {% endfor %}
                </select>
                <span class="form-hint">Required for Full MemAgent execution; optional attribution in diagnostic mode.</span>
            </div>
            <div class="form-group">
                <label for="benchmark">Benchmark</label>
                <select name="benchmark" id="benchmark">
                    {% for item in benchmark_catalog %}
                    <option value="{{ item.benchmark_id }}" {% if benchmark == item.benchmark_id %}selected{% endif %}>{{ item.name }}</option>
                    {% endfor %}
                    <option value="longmemeval" {% if benchmark == 'longmemeval' %}selected{% endif %}>LongMemEval (legacy paid path)</option>
                </select>
            </div>
            <div class="form-group">
                <label for="dataset_variant">Dataset Variant</label>
                <select name="dataset_variant" id="dataset_variant">
                    <option value="{{ dataset_variant }}" selected>{{ dataset_variant }}</option>
                </select>
            </div>
            <div class="form-group">
                <label for="profile">Evaluation Profile</label>
                <select id="profile" name="profile">
                    <option value="smoke" {% if (profile or 'smoke') == 'smoke' %}selected{% endif %}>Smoke · bounded integration check</option>
                    <option value="regression" {% if profile == 'regression' %}selected{% endif %}>Regression · fixed stratified subset</option>
                    <option value="paper" {% if profile == 'paper' %}selected{% endif %}>Paper · full official split</option>
                </select>
            </div>
            <div class="form-group">
                <label for="data_path">Official Dataset Path</label>
                <input id="data_path" name="data_path" type="text" value="{{ data_path or '' }}" placeholder="/path/to/official/data">
            </div>
            <div class="form-group">
                <label for="num_samples">Samples</label>
                <input id="num_samples" name="num_samples" type="number" min="1" value="{{ num_samples }}">
            </div>
            <div class="form-group">
                <label for="memory_provider">Memory Provider</label>
                <select id="memory_provider" name="memory_provider">
                    <option value="filesystem" {% if (memory_provider or 'filesystem') == 'filesystem' %}selected{% endif %}>Filesystem</option>
                    <option value="oracle" {% if memory_provider == 'oracle' %}selected{% endif %}>Oracle AI Database</option>
                </select>
            </div>
            <div class="form-group">
                <label for="top_k">Final Evidence (top-k)</label>
                <input id="top_k" name="top_k" type="number" min="1" value="{{ top_k or 8 }}">
            </div>
            <div class="form-group">
                <label for="candidate_pool_size">First-stage Candidates</label>
                <input id="candidate_pool_size" name="candidate_pool_size" type="number" min="8" value="{{ candidate_pool_size or 256 }}">
            </div>
            <div class="form-group">
                <label for="lexical_weight">Lexical Fusion Weight</label>
                <input id="lexical_weight" name="lexical_weight" type="number" min="0" max="1" step="0.05" value="{{ lexical_weight if lexical_weight is defined else 0.35 }}">
            </div>
            <div class="form-group">
                <label for="rerank_weight">Local Rerank Weight</label>
                <input id="rerank_weight" name="rerank_weight" type="number" min="0" step="0.05" value="{{ rerank_weight if rerank_weight is defined else 0.15 }}">
            </div>
            <div class="form-group">
                <label for="query_expansion">Concept Query Expansion</label>
                <select id="query_expansion" name="query_expansion">
                    <option value="true" {% if query_expansion %}selected{% endif %}>Enabled</option>
                    <option value="false" {% if not query_expansion %}selected{% endif %}>Disabled</option>
                </select>
            </div>
            <div class="form-group">
                <label for="reader_repair">Structured-output Repair</label>
                <select id="reader_repair" name="reader_repair">
                    <option value="true" {% if reader_repair %}selected{% endif %}>One bounded repair</option>
                    <option value="false" {% if not reader_repair %}selected{% endif %}>Disabled</option>
                </select>
            </div>
            <div class="form-group">
                <label for="model_provider">Reader Provider</label>
                <select id="model_provider" name="model_provider">
                    <option value="ollama" {% if (model_provider or 'ollama') == 'ollama' %}selected{% endif %}>Ollama (local)</option>
                    <option value="openai" {% if model_provider == 'openai' %}selected{% endif %}>OpenAI (metered)</option>
                    <option value="anthropic" {% if model_provider == 'anthropic' %}selected{% endif %}>Anthropic (metered)</option>
                </select>
            </div>
            <div class="form-group">
                <label for="model">Reader Model</label>
                <input id="model" name="model" type="text" value="{{ model or 'qwen2.5:3b' }}">
            </div>
            <div class="form-group">
                <label for="judge_model">Judge Model (optional)</label>
                <input id="judge_model" name="judge_model" type="text" value="{{ judge_model or '' }}" placeholder="Uses reader model">
            </div>
            <div class="form-group">
                <label for="embedding_model">Ollama Embeddings</label>
                <input id="embedding_model" name="embedding_model" type="text" value="{{ embedding_model or 'nomic-embed-text' }}">
            </div>
            <div class="form-group">
                <label for="ollama_host">Ollama Host</label>
                <input id="ollama_host" name="ollama_host" type="text" value="{{ ollama_host or 'http://localhost:11434' }}">
            </div>
            <div class="form-group form-check-group">
                <label for="oracle_reader">Gold-evidence Oracle Reader</label>
                <select id="oracle_reader" name="oracle_reader">
                    <option value="true" {% if oracle_reader %}selected{% endif %}>Enabled</option>
                    <option value="false" {% if not oracle_reader %}selected{% endif %}>Disabled</option>
                </select>
                <span class="form-hint">Separates retrieval failures from answer-synthesis failures.</span>
            </div>
        </div>
        <div class="eval-run-actions">
            <button type="submit" id="eval-run-start-btn" class="btn btn-primary" {% if not can_run %}disabled{% endif %}>Run Evaluation</button>
            <button type="button" id="eval-run-stop-btn" class="btn btn-danger" hidden>Stop Run</button>
        </div>
        <p class="form-hint">Every run is fail-closed as <strong>Diagnostic</strong> unless its versioned manifest proves the official runner, scorer, full split, source revision, models, prompts, and runtime metadata all match. Full MemAgent mode disables side-effect tools and uses isolated benchmark memory while preserving the selected agent's memory configuration. Ollama readers cost $0 externally; OpenAI usage is metered. Paper profiles ignore the sample field.</p>
        <p class="form-hint">Results saved to <code>{{ eval_results_dir }}</code>.</p>
        <p id="eval-run-status" class="form-hint eval-run-status"></p>
    </form>
</details>

<details id="terminal-bench" class="detail-section evalground-section evaluation-resource tb-section"{% if tb_run %} open{% endif %}><summary>Test a harness on Terminal-Bench 4.0</summary>
    <p class="form-hint tb-intro">Harbor runs each task in its own Docker container and grades it with the task’s own tests. Codex and Claude Code are installed inside the container; a MemAgent works it from here. With MemoRizz memory on, a harness starts with what earlier runs learned, but never with lessons from the task it is solving.</p>
    <ul class="tb-checks" id="tb-checks" aria-live="polite" aria-label="What this machine has">
        <li data-check="harbor"><span class="tb-dot" aria-hidden="true"></span><span>Harbor</span><small>checking…</small></li>
        <li data-check="docker"><span class="tb-dot" aria-hidden="true"></span><span>Docker</span><small>checking…</small></li>
        <li data-check="key"><span class="tb-dot" aria-hidden="true"></span><span>Model key</span><small>checking…</small></li>
        <li data-check="memory"><span class="tb-dot" aria-hidden="true"></span><span>Memory</span><small>checking…</small></li>
    </ul>
    <form id="tb-form" class="evalground-form tb-form" novalidate>
        <div class="form-grid">
            <div class="form-group">
                <label for="tb-harness">Harness</label>
                <select id="tb-harness" name="harness">
                    <option value="codex">Codex · in the container</option>
                    <option value="claude-code">Claude Code · in the container</option>
                    <option value="memagent">MemAgent · drives the container</option>
                    <option value="oracle">Reference solutions · free setup check</option>
                </select>
            </div>
            <div class="form-group" data-tb-for="codex claude-code memagent">
                <label for="tb-model">Model</label>
                <input id="tb-model" name="model" list="tb-models" autocomplete="off" spellcheck="false">
                <datalist id="tb-models"></datalist>
                <span class="form-hint" id="tb-model-hint"></span>
            </div>
            <div class="form-group" data-tb-for="codex">
                <label for="tb-codex-auth">Codex sign-in</label>
                <select id="tb-codex-auth" name="codex_auth">
                    <option value="api_key">API key · per token</option>
                    <option value="chatgpt">ChatGPT sign-in · your plan</option>
                </select>
                <span class="form-hint is-warn" id="tb-codex-warning" hidden>Copies <code>~/.codex/auth.json</code> into each task container, where the task’s own code can read it.</span>
            </div>
            <div class="form-group">
                <label for="tb-timeout">Time per task</label>
                <select id="tb-timeout" name="agent_timeout_multiplier">
                    <option value="0.0625">30 minutes</option>
                    <option value="0.1" selected>48 minutes</option>
                    <option value="0.25">2 hours</option>
                    <option value="1">8 hours · the official limit</option>
                </select>
            </div>
            <div class="form-group">
                <label for="tb-attempts">Attempts per task</label>
                <input id="tb-attempts" name="attempts" type="number" min="1" max="5" value="1">
            </div>
            <div class="form-group">
                <label for="tb-concurrent">Parallel trials</label>
                <input id="tb-concurrent" name="n_concurrent" type="number" min="1" max="4" value="1">
                <span class="form-hint" id="tb-concurrent-hint"></span>
            </div>
            <div class="form-group" data-tb-for="memagent">
                <label for="tb-cost">Cost cap per task (USD)</label>
                <input id="tb-cost" name="max_cost_per_task" type="number" min="0.1" step="0.1" value="4">
            </div>
            <div class="form-group" data-tb-for="codex claude-code memagent">
                <label for="tb-memory">MemoRizz memory</label>
                <select id="tb-memory" name="memory">
                    <option value="off">Off · the plain harness</option>
                    <option value="on">On · recall earlier lessons</option>
                </select>
            </div>
            <div class="form-group" data-tb-for="codex claude-code memagent" data-tb-memory>
                <label for="tb-memory-id">Memory ID</label>
                <input id="tb-memory-id" name="memory_id" value="terminal-bench">
                <label class="tb-check"><input type="checkbox" name="learn" value="on" checked> Save a lesson from each finished task</label>
            </div>
        </div>
        <fieldset class="tb-tasks">
            <legend>Tasks <span class="optional" id="tb-task-count"></span></legend>
            <div class="tb-task-tools">
                <button type="button" class="btn btn-sm" data-tb-pick="smoke">Smoke set · {{ terminal_bench.smoke|length }} tasks</button>
                <button type="button" class="btn btn-sm" data-tb-pick="none">Clear</button>
                <input type="search" id="tb-task-filter" placeholder="Filter tasks" aria-label="Filter tasks" autocomplete="off">
            </div>
            <div class="tb-groups" id="tb-groups">
                {% for group in terminal_bench.groups %}
                <section class="tb-group" data-tb-group>
                    <h4>{{ group.category }} <span>{{ group.tasks|length }}</span></h4>
                    <div class="tb-task-grid">
                        {% for task in group.tasks %}
                        <label class="tb-task" data-name="{{ task.name }} {{ task.subcategory or '' }}">
                            <input type="checkbox" name="tasks" value="{{ task.name }}"{% if tb_run and task.name in (tb_run.tasks or []) %} checked{% elif not tb_run and task.name in terminal_bench.smoke %} checked{% endif %}>
                            <span class="tb-task-name">{{ task.name }}</span>
                            <small>{{ task.subcategory or group.category }}{% if task.expert_h %} · ~{{ task.expert_h|round(1)|string|replace('.0', '') }} h expert{% endif %} · {{ task.cpus }} CPU · {{ (task.mem_mb / 1024)|round(0)|int }} GB</small>
                        </label>
                        {% endfor %}
                    </div>
                </section>
                {% endfor %}
            </div>
            <p class="form-hint">{{ terminal_bench.task_count }} of the dataset’s tasks run without a GPU; {{ terminal_bench.gpu_tasks|join(', ') }} are left out.</p>
        </fieldset>
        <p class="tb-estimate" id="tb-estimate" aria-live="polite"></p>
        <div class="eval-run-actions">
            <button type="submit" id="tb-start" class="btn btn-primary">Run Terminal-Bench</button>
            <button type="button" id="tb-stop" class="btn btn-danger" hidden>Stop run</button>
        </div>
        <p id="tb-status" class="form-hint eval-run-status" aria-live="polite"></p>
    </form>
</details>

<div id="live-run-output-section" class="detail-section evalground-section" hidden>
    <h2>Live Output</h2>
    <pre id="live-run-output" class="eval-run-output"></pre>
</div>

<details id="agent-history" class="detail-section evalground-section evaluation-resource"{% if selected_run_id %} open{% endif %}><summary>Agent evaluation history</summary>
    {% if eval_charts %}{% from '_analytics_charts.html' import bars %}
    <div class="usage-dashboard"><div class="usage-grid">{{ bars(eval_charts.cost, 'Recent evaluation charges', 'USD · benchmark-reported estimates') }}</div><p class="usage-notice">Charges are benchmark estimates, not provider invoices. Runs may differ in samples, model and configuration; compare like for like. Local hardware cost is not measured.</p></div>{% endif %}
    {% if runs_history %}
    <div class="traces-table-wrap">
        <table class="traces-agents-table evalground-runs-table">
            <thead>
                <tr>
                    <th>Run ID</th>
                    <th>Status</th>
                    <th>Benchmark</th>
                    <th>Mode</th>
                    <th>Agent</th>
                    <th>Dataset</th>
                    <th>Samples</th>
                    <th>Accuracy</th>
                    <th>API cost estimate / time</th>
                    <th>Created</th>
                    <th>Finished</th>
                    <th>Actions</th>
                </tr>
            </thead>
            <tbody>
                {% for run in runs_history %}
                {% set run_status = run.status or 'unknown' %}
                {% set badge_class = 'badge-warning' %}
                {% if run_status == 'completed' %}
                    {% set badge_class = 'badge-success' %}
                {% elif run_status in ['failed', 'canceled'] %}
                    {% set badge_class = 'badge-danger' %}
                {% endif %}
                <tr {% if selected_run_id and selected_run_id == run.run_id %}class="is-selected"{% endif %}>
                    <td><code>{{ run.run_id[:8] }}...</code></td>
                    <td><span class="badge {{ badge_class }}">{{ run_status }}</span></td>
                    <td>{{ run.benchmark or 'longmemeval' }}</td>
                    <td>{{ 'Full MemAgent' if run.evaluation_mode == 'memagent' else 'Retrieval' }}</td>
                    <td>{{ run.agent_name }}</td>
                    <td>{{ run.dataset_variant or 'oracle' }}</td>
                    <td>{{ run.num_samples }}</td>
                    <td>
                        {% if run.overall_accuracy is not none %}
                            {{ run.overall_accuracy }}%
                        {% else %}
                            -
                        {% endif %}
                    </td>
                    <td>{{ '$%.6f'|format(run.cost_usd) if run.cost_usd is not none else 'Unknown' }} / {{ '%.2fs'|format(run.processing_seconds) if run.processing_seconds is not none else 'Unknown' }}</td>
                    <td>{{ run.created_at or '-' }}</td>
                    <td>{{ run.finished_at or '-' }}</td>
                    <td>
                        <a class="btn btn-sm" href="/evalground?run_id={{ run.run_id }}">View</a>
                        {% if run_status in ['queued', 'running', 'canceling'] %}
                        <button type="button" class="btn btn-sm btn-danger" data-eval-stop-run="{{ run.run_id }}">Stop</button>
                        {% endif %}
                    </td>
                </tr>
                {% endfor %}
            </tbody>
        </table>
    </div>
    {% else %}
    <p class="empty-hint">No agent evaluations in this server session. Saved model comparisons are listed in Recent evaluations above.</p>
    {% endif %}
</details>

<p class="evaluation-help"><a href="/evalground/compare#experiment-library">Examples &amp; developer guides</a> · Explore sample datasets and runnable memory experiments.</p>

{% if selected_agent %}
<div class="agent-summary">
    <div class="summary-item">
        <span class="summary-label">Agent</span>
        <span class="summary-value">
            {% if selected_agent.persona and selected_agent.persona.name %}
                {{ selected_agent.persona.name }}
            {% else %}
                Agent
            {% endif %}
        </span>
    </div>
    <div class="summary-item">
        <span class="summary-label">Agent ID</span>
        <code class="summary-value">{{ selected_agent.agent_id }}</code>
    </div>
    <div class="summary-item">
        <span class="summary-label">Mode</span>
        <span class="badge">{{ selected_agent.application_mode or 'assistant' }}</span>
    </div>
    {% if selected_agent.memory_ids %}
    <div class="summary-item">
        <span class="summary-label">Memory IDs</span>
        <span class="summary-value">{{ selected_agent.memory_ids|length }}</span>
    </div>
    {% endif %}
</div>
{% endif %}

{% if eval_results and eval_results.metadata and eval_results.metadata.benchmark == 'terminal-bench' %}
{% set tb = eval_results.metadata %}
<div class="detail-section evalground-section tb-results">
    <h2>Terminal-Bench 4.0 · <span class="mono">{{ tb.harness }}</span>{% if tb.model %} · {{ tb.model }}{% endif %}</h2>
    <div class="eval-summary-grid">
        <div class="eval-summary-card">
            <div class="eval-summary-label">Tasks passed</div>
            <div class="eval-summary-value">{{ (eval_results.overall_accuracy * 100)|round(1) }}%</div>
            <small class="form-hint">{{ tb.passed }} of {{ tb.num_trials }} trial{{ '' if tb.num_trials == 1 else 's' }}</small>
        </div>
        <div class="eval-summary-card">
            <div class="eval-summary-label">API cost{% if eval_results.cost_is_estimate %} · estimate{% endif %}</div>
            <div class="eval-summary-value">{% if eval_results.external_api_cost_usd is not none %}${{ '%.2f'|format(eval_results.external_api_cost_usd) }}{% else %}Unknown{% endif %}</div>
        </div>
        <div class="eval-summary-card">
            <div class="eval-summary-label">Memory</div>
            <div class="eval-summary-value tb-summary-text">{% if tb.memory_id %}<span class="mono">{{ tb.memory_id }}</span>{% else %}Off{% endif %}</div>
            {% if tb.lessons_saved is defined %}<small class="form-hint">{{ tb.lessons_saved }} lesson{{ '' if tb.lessons_saved == 1 else 's' }} saved</small>{% endif %}
        </div>
        <div class="eval-summary-card">
            <div class="eval-summary-label">Run time</div>
            <div class="eval-summary-value">{% if tb.total_processing_time %}{{ (tb.total_processing_time / 60)|round(1) }} min{% else %}—{% endif %}</div>
        </div>
    </div>
    {% if eval_charts %}{% from '_analytics_charts.html' import bars %}
    <div class="usage-dashboard"><div class="usage-grid">{{ bars(eval_charts.categories, 'Pass rate by category', 'share of trials whose tests passed') }}</div></div>{% endif %}
    <div class="tb-cases-wrap">
        <table class="fleet-table tb-cases">
            <caption class="visually-hidden">Each trial: its task, result, time, tokens and cost.</caption>
            <thead><tr><th scope="col">Task</th><th scope="col">Result</th><th scope="col" class="num">Time</th><th scope="col" class="num">Tokens in / out</th><th scope="col" class="num">Cost</th><th scope="col">Memory</th></tr></thead>
            <tbody>
                {% for case in eval_results.cases %}
                <tr>
                    <td><span class="mono">{{ case.task }}</span><small>{{ case.category }}{% if case.subcategory %} · {{ case.subcategory }}{% endif %}</small></td>
                    <td>{% if case.passed %}<span class="badge badge-success">Passed</span>{% elif case.reward is not none %}<span class="badge badge-warning">Failed</span>{% else %}<span class="badge badge-danger">Error</span>{% endif %}{% if case.error %}<small class="tb-error" title="{{ case.error }}">{{ case.error|truncate(90) }}</small>{% endif %}</td>
                    <td class="num">{% if case.duration_s is not none %}{{ (case.duration_s / 60)|round(1) }} min{% else %}—{% endif %}</td>
                    <td class="num">{% if case.input_tokens is not none %}{{ '{:,}'.format(case.input_tokens) }} / {{ '{:,}'.format(case.output_tokens or 0) }}{% else %}—{% endif %}</td>
                    <td class="num">{% if case.cost_usd is not none %}${{ '%.2f'|format(case.cost_usd) }}{% else %}—{% endif %}</td>
                    <td>{% if tb.memory_id %}{{ case.memory_sources }} record{{ '' if case.memory_sources == 1 else 's' }}{% else %}—{% endif %}</td>
                </tr>
                {% endfor %}
            </tbody>
        </table>
    </div>
    <p class="form-hint">Dataset <code>{{ tb.dataset }}</code> · Harbor job folder <code>{{ tb.job_dir }}</code>{% if tb.harbor_exit_code %} · Harbor exited with code {{ tb.harbor_exit_code }}{% endif %}. Runs with a shortened time limit or memory from earlier runs are not leaderboard-comparable.</p>
</div>
{% elif eval_results %}
<div class="detail-section evalground-section">
    <h2>Results</h2>
    {% if eval_charts %}{% from '_analytics_charts.html' import bars %}
    <div class="usage-dashboard"><div class="usage-grid">{{ bars(eval_charts.categories, 'Quality by category', 'reported accuracy') }}{{ bars(eval_charts.latency, 'Evaluation latency', 'mean seconds; stages may overlap') }}</div></div>{% endif %}
    <div class="eval-summary-grid">
        <div class="eval-summary-card">
            <div class="eval-summary-label">Overall accuracy</div>
            <div class="eval-summary-value">{{ (eval_results.overall_accuracy * 100)|round(2) }}%</div>
        </div>
        <div class="eval-summary-card">
            <div class="eval-summary-label">Overall score</div>
            <div class="eval-summary-value">{{ eval_results.overall_score|round(3) }}</div>
        </div>
        <div class="eval-summary-card">
            <div class="eval-summary-label">Samples</div>
            <div class="eval-summary-value">{{ eval_results.metadata.num_samples }}</div>
        </div>
        <div class="eval-summary-card">
            <div class="eval-summary-label">Processing time</div>
            <div class="eval-summary-value">{{ eval_results.metadata.total_processing_time|round(2) }}s</div>
        </div>
        {% if eval_results.retrieval %}
        <div class="eval-summary-card">
            <div class="eval-summary-label">Retrieval recall@k</div>
            <div class="eval-summary-value">{% if eval_results.retrieval.recall_at_k is not none %}{{ eval_results.retrieval.recall_at_k|round(3) }}{% else %}—{% endif %}</div>
        </div>
        <div class="eval-summary-card">
            <div class="eval-summary-label">Gold-evidence ceiling</div>
            <div class="eval-summary-value">{% if eval_results.answer_quality.gold_evidence_oracle_score is not none %}{{ eval_results.answer_quality.gold_evidence_oracle_score|round(3) }}{% else %}—{% endif %}</div>
        </div>
        <div class="eval-summary-card">
            <div class="eval-summary-label">Grounded answers</div>
            <div class="eval-summary-value">{{ (eval_results.answer_quality.grounded_rate * 100)|round(1) }}%</div>
        </div>
        <div class="eval-summary-card">
            <div class="eval-summary-label">Corpus cache</div>
            <div class="eval-summary-value">{{ eval_results.efficiency.embedding_cache_hits }} hit / {{ eval_results.efficiency.embedding_cache_misses }} miss</div>
        </div>
        {% endif %}
    </div>

    <table class="info-table eval-metadata-table">
        {% if eval_results.benchmark_name %}
        <tr>
            <td class="info-label">Benchmark</td>
            <td class="info-value">{{ eval_results.benchmark_name }}</td>
        </tr>
        {% endif %}
        <tr>
            <td class="info-label">Dataset</td>
            <td class="info-value"><span class="badge">{{ eval_results.metadata.dataset_variant }}</span></td>
        </tr>
        <tr>
            <td class="info-label">Execution</td>
            <td class="info-value">
                {% if eval_results.metadata.evaluation_mode == 'memagent' %}
                Full MemAgent · {{ eval_results.metadata.application_mode|default('assistant') }}
                {% else %}
                Memory retrieval diagnostic
                {% endif %}
            </td>
        </tr>
        <tr>
            <td class="info-label">Timestamp</td>
            <td class="info-value">{{ eval_results.metadata.timestamp }}</td>
        </tr>
        {% if eval_results.schema_version %}
        <tr>
            <td class="info-label">Retrieval basis</td>
            <td class="info-value">{{ eval_results.retrieval.basis or 'diagnostic_fusion' }} · top {{ eval_results.retrieval.top_k }} of {{ eval_results.retrieval.candidate_pool_size }}</td>
        </tr>
        <tr>
            <td class="info-label">Models</td>
            <td class="info-value">{{ eval_results.metadata.model_provider }}:{{ eval_results.metadata.reader_model }} reader · {{ eval_results.metadata.judge_model }} judge · {{ eval_results.metadata.embedding_model }} embeddings</td>
        </tr>
        <tr>
            <td class="info-label">External API cost estimate</td>
            <td class="info-value"><span class="badge">{% if eval_results.metadata.external_api_cost_usd is not none %}${{ eval_results.metadata.external_api_cost_usd }}{% else %}Unknown{% endif %}</span></td>
        </tr>
        <tr>
            <td class="info-label">Comparison status</td>
            <td class="info-value"><span class="badge {% if eval_results.paper_comparable %}badge-success{% else %}badge-warning{% endif %}">{{ eval_results.comparison_label or 'Diagnostic' }}</span></td>
        </tr>
        {% endif %}
        {% if eval_output_path %}
        <tr>
            <td class="info-label">Output</td>
            <td class="info-value"><code>{{ eval_output_path }}</code></td>
        </tr>
        {% endif %}
    </table>

    {% include "_eval_usage.html" %}
    <h3 class="eval-ability-heading">Ability breakdown</h3>
    <div class="eval-category-grid">
        {% for cat_key, metrics in eval_results.category_results.items() %}
        <div class="eval-category-card">
            <div class="eval-category-name">{{ cat_key|replace('-', ' ')|replace('_', ' ')|title }}</div>
            <div class="eval-category-metric">
                <span class="metric-label">Accuracy</span>
                <span class="metric-value">{% if eval_results.schema_version %}{{ (metrics.accuracy * 100)|round(1) }}%{% else %}{{ metrics.accuracy|round(2) }}{% endif %}</span>
            </div>
            <div class="eval-category-metric">
                <span class="metric-label">Avg Score</span>
                <span class="metric-value">{{ metrics.average_score|round(3) }}</span>
            </div>
            <div class="eval-category-metric">
                <span class="metric-label">Samples</span>
                <span class="metric-value">{{ metrics.num_samples }}</span>
            </div>
            {% if metrics.retrieval_recall_at_k is defined and metrics.retrieval_recall_at_k is not none %}
            <div class="eval-category-metric">
                <span class="metric-label">Recall@k</span>
                <span class="metric-value">{{ metrics.retrieval_recall_at_k|round(3) }}</span>
            </div>
            {% endif %}
            {% if metrics.faithfulness is defined and metrics.faithfulness is not none %}
            <div class="eval-category-metric">
                <span class="metric-label">Faithfulness</span>
                <span class="metric-value">{{ metrics.faithfulness|round(3) }}</span>
            </div>
            {% endif %}
            {% if metrics.oracle_reader_score is defined and metrics.oracle_reader_score is not none %}
            <div class="eval-category-metric">
                <span class="metric-label">Oracle reader</span>
                <span class="metric-value">{{ metrics.oracle_reader_score|round(3) }}</span>
            </div>
            {% endif %}
            {% if metrics.grounded_rate is defined %}
            <div class="eval-category-metric">
                <span class="metric-label">Grounded</span>
                <span class="metric-value">{{ (metrics.grounded_rate * 100)|round(1) }}%</span>
            </div>
            {% endif %}
        </div>
        {% endfor %}
    </div>

    {% if eval_results.schema_version %}
    <div class="eval-abstention-callout">
        <span class="eval-abstention-icon">&#9432;</span>
        <div>
            <strong>{{ eval_results.comparison_label or 'Diagnostic' }}</strong> &mdash;
            {% if eval_results.paper_comparable %}
            Every required paper-protocol field matched the versioned manifest.
            {% else %}
            This run is useful for engineering diagnosis, not as an official paper or leaderboard score.
            {% if eval_results.non_comparability_reasons %}<ul>{% for reason in eval_results.non_comparability_reasons %}<li>{{ reason }}</li>{% endfor %}</ul>{% endif %}
            {% endif %}
        </div>
    </div>
    {% endif %}
</div>
{% elif selected_run_id %}
<div class="empty-state">
    <div class="empty-icon">EVAL</div>
    <h3>No results yet</h3>
    <p>Run a benchmark to see results.</p>
</div>
{% endif %}

{% if run_output %}
<div class="detail-section evalground-section">
    <h2>Run Output</h2>
    <pre class="eval-run-output">{{ run_output }}</pre>
</div>
{% endif %}
{% endblock %}

{% block scripts %}
<script type="application/json" id="tb-data">{{ {'smoke': terminal_bench.smoke, 'default_models': terminal_bench.default_models, 'run': tb_run}|tojson }}</script>
<script src="/static/js/evalground-runs.js?v=4" defer></script>
<script src="/static/js/terminal-bench.js?v=1" defer></script>
<script>
document.addEventListener("DOMContentLoaded", () => {
    const evalMonitor = window.memorizzEvalMonitor || null;
    const selectedRunId = {{ (selected_run_id or '')|tojson }};
    const selectedRunStatus = {{ (selected_run_status or '')|tojson }};
    const activeRunStatuses = new Set(["queued", "running", "canceling"]);
    const benchmarkCatalog = {{ benchmark_catalog|tojson }};
    const variantMap = {{ suite_variant_map|tojson }};
    const initialVariant = {{ (dataset_variant or '')|tojson }};
    const benchmarkSelect = document.getElementById("benchmark");
    const variantSelect = document.getElementById("dataset_variant");
    const dataPathInput = document.getElementById("data_path");
    const evaluationModeSelect = document.getElementById("evaluation_mode");
    const agentSelect = document.getElementById("agent_id");

    function syncEvaluationMode() {
        if (!evaluationModeSelect || !agentSelect) return;
        agentSelect.required = evaluationModeSelect.value === "memagent";
    }

    function syncBenchmark(initial = false) {
        if (!benchmarkSelect || !variantSelect) return;
        const benchmarkId = benchmarkSelect.value;
        const variants = variantMap[benchmarkId] || [];
        const wanted = initial && variants.includes(initialVariant) ? initialVariant : variants[0];
        variantSelect.replaceChildren(...variants.map((variant) => {
            const option = document.createElement("option");
            option.value = variant;
            option.textContent = variant;
            option.selected = variant === wanted;
            return option;
        }));
        const localBenchmark = benchmarkCatalog.find((item) => item.benchmark_id === benchmarkId);
        if (dataPathInput) {
            dataPathInput.disabled = !localBenchmark;
            dataPathInput.placeholder = localBenchmark
                ? `${localBenchmark.dataset_env} or /path/to/official/data`
                : "Not used by legacy LongMemEval";
            if (!initial) {
                dataPathInput.value = localBenchmark?.dataset_path || "";
            }
        }
    }

    if (benchmarkSelect) {
        benchmarkSelect.addEventListener("change", () => syncBenchmark(false));
        syncBenchmark(true);
    }
    if (evaluationModeSelect) {
        evaluationModeSelect.addEventListener("change", syncEvaluationMode);
        syncEvaluationMode();
    }

    if (evalMonitor && selectedRunId && activeRunStatuses.has(selectedRunStatus)) {
        evalMonitor.trackRun(selectedRunId, {
            status: selectedRunStatus,
            keepLogs: true
        });
    }

    const form = document.querySelector(".evalground-form");
    const historyStopButtons = document.querySelectorAll("[data-eval-stop-run]");
    historyStopButtons.forEach((button) => {
        button.addEventListener("click", async () => {
            const runId = button.getAttribute("data-eval-stop-run");
            if (!runId) return;
            button.disabled = true;
            try {
                const response = await fetch(`/evalground/runs/${encodeURIComponent(runId)}/stop`, {
                    method: "POST",
                    headers: { "Accept": "application/json" }
                });
                const payload = await response.json().catch(() => ({}));
                if (!response.ok) {
                    throw new Error(payload.error || payload.detail || "Unable to stop evaluation.");
                }
                if (evalMonitor) {
                    evalMonitor.trackRun(runId, {
                        status: "canceling",
                        keepLogs: true
                    });
                }
                window.location.href = `/evalground?run_id=${encodeURIComponent(runId)}`;
            } catch (error) {
                button.disabled = false;
                window.alert(error?.message || String(error));
            }
        });
    });

    if (!form) return;
    const startButton = document.getElementById("eval-run-start-btn");
    const stopButton = document.getElementById("eval-run-stop-btn");
    const status = document.getElementById("eval-run-status");
    const liveSection = document.getElementById("live-run-output-section");
    const liveOutput = document.getElementById("live-run-output");
    let activeRunId = null;

    function setIdleButtons() {
        activeRunId = null;
        if (startButton) {
            startButton.disabled = false;
            startButton.textContent = "Run Evaluation";
        }
        if (stopButton) {
            stopButton.hidden = true;
            stopButton.disabled = false;
        }
    }

    function setRunningButtons(runId) {
        activeRunId = runId;
        if (startButton) {
            startButton.disabled = true;
            startButton.textContent = "Running...";
        }
        if (stopButton) {
            stopButton.hidden = false;
            stopButton.disabled = false;
        }
    }

    function setStartingState() {
        activeRunId = null;
        if (startButton) {
            startButton.disabled = true;
            startButton.textContent = "Running...";
        }
        if (stopButton) {
            stopButton.hidden = true;
            stopButton.disabled = false;
        }
    }

    async function pollRun(runId) {
        let cursor = 0;
        setRunningButtons(runId);
        while (true) {
            const response = await fetch(`/evalground/runs/${encodeURIComponent(runId)}?after=${cursor}`, {
                headers: { "Accept": "application/json" }
            });

            if (!response.ok) {
                throw new Error("Lost connection to the evaluation run.");
            }

            const payload = await response.json();
            const lines = Array.isArray(payload.logs) ? payload.logs : [];

            if (evalMonitor) {
                evalMonitor.syncRunPayload(runId, payload);
            }

            if (lines.length && liveOutput) {
                const chunk = lines.join("\n");
                liveOutput.textContent += (liveOutput.textContent ? "\n" : "") + chunk;
                liveOutput.scrollTop = liveOutput.scrollHeight;
            }

            cursor = typeof payload.next_index === "number"
                ? payload.next_index
                : cursor + lines.length;

            if (payload.status === "queued" || payload.status === "running" || payload.status === "canceling") {
                if (status) {
                    status.textContent = payload.status === "canceling"
                        ? "Stopping evaluation..."
                        : "Evaluation running. Live logs are updating below.";
                }
                await new Promise((resolve) => setTimeout(resolve, 1000));
                continue;
            }

            if (payload.status === "completed") {
                if (status) {
                    status.textContent = "Evaluation finished. Loading results...";
                }
                window.location.href = `/evalground?run_id=${encodeURIComponent(runId)}`;
                return;
            }

            if (payload.status === "canceled") {
                if (status) {
                    status.textContent = "Evaluation canceled.";
                }
                setIdleButtons();
                return;
            }

            const failure = payload.error || "Evaluation failed.";
            if (status) {
                status.textContent = failure;
            }
            setIdleButtons();
            return;
        }
    }

    if (stopButton) {
        stopButton.addEventListener("click", async () => {
            if (!activeRunId) return;
            stopButton.disabled = true;
            if (status) {
                status.textContent = "Stopping evaluation...";
            }
            try {
                const response = await fetch(`/evalground/runs/${encodeURIComponent(activeRunId)}/stop`, {
                    method: "POST",
                    headers: { "Accept": "application/json" }
                });
                const payload = await response.json().catch(() => ({}));
                if (!response.ok) {
                    throw new Error(payload.error || payload.detail || "Unable to stop evaluation.");
                }
                if (evalMonitor) {
                    evalMonitor.trackRun(activeRunId, {
                        status: "canceling",
                        keepLogs: true
                    });
                }
            } catch (error) {
                if (status) {
                    status.textContent = error?.message || String(error);
                }
                stopButton.disabled = false;
            }
        });
    }

    form.addEventListener("submit", async (event) => {
        event.preventDefault();

        setStartingState();
        if (liveSection) {
            liveSection.hidden = false;
        }
        if (liveOutput) {
            liveOutput.textContent = "";
        }
        if (status) {
            status.textContent = "Starting evaluation...";
        }

        try {
            const formData = new FormData(form);
            const response = await fetch("/evalground/runs", {
                method: "POST",
                body: formData,
                headers: { "Accept": "application/json" }
            });

            const payload = await response.json().catch(() => ({}));
            if (!response.ok) {
                throw new Error(payload.error || payload.detail || "Unable to start evaluation.");
            }

            if (!payload.run_id) {
                throw new Error("Missing evaluation run ID.");
            }

            setRunningButtons(payload.run_id);
            if (evalMonitor) {
                evalMonitor.trackRun(payload.run_id, {
                    status: payload.status || "queued",
                    keepLogs: false
                });
            }
            await pollRun(payload.run_id);
        } catch (error) {
            if (status) {
                status.textContent = error?.message || String(error);
            }
            setIdleButtons();
        }
    });
});
</script>
{% endblock %}