ctfy.sdk.resources.eval

client.eval — the public model-evaluation leaderboard.

 1"""``client.eval`` — the public model-evaluation leaderboard."""
 2
 3from __future__ import annotations
 4
 5from ctfy.sdk._helpers import _raise_for_status
 6from ctfy.sdk.base import BaseHttpClient
 7from ctfy.server.models import EvalLeaderboard
 8
 9
10class EvalResource:
11    """Public, multi-dimensional model leaderboard (no auth required)."""
12
13    def __init__(self, http: BaseHttpClient) -> None:
14        self._http = http
15
16    def leaderboard(self, *, competition_id: str = "", tier: str = "official") -> EvalLeaderboard:
17        """Ranked model standings + per-dimension (difficulty / bucket / tag)
18        breakdowns. ``competition_id`` scopes to one competition; empty = global.
19
20        ``tier`` selects the credibility band: ``official`` (default,
21        platform-run), ``self_reported`` (vendor-run, unverified), or ``all``
22        for both. Sandbox runs are never exposed here."""
23        resp = self._http.request(
24            "GET", "/eval/leaderboard", params={"competition_id": competition_id, "tier": tier}
25        )
26        _raise_for_status(resp)
27        return EvalLeaderboard.model_validate(resp.json())
class EvalResource: 
11class EvalResource:
12    """Public, multi-dimensional model leaderboard (no auth required)."""
13
14    def __init__(self, http: BaseHttpClient) -> None:
15        self._http = http
16
17    def leaderboard(self, *, competition_id: str = "", tier: str = "official") -> EvalLeaderboard:
18        """Ranked model standings + per-dimension (difficulty / bucket / tag)
19        breakdowns. ``competition_id`` scopes to one competition; empty = global.
20
21        ``tier`` selects the credibility band: ``official`` (default,
22        platform-run), ``self_reported`` (vendor-run, unverified), or ``all``
23        for both. Sandbox runs are never exposed here."""
24        resp = self._http.request(
25            "GET", "/eval/leaderboard", params={"competition_id": competition_id, "tier": tier}
26        )
27        _raise_for_status(resp)
28        return EvalLeaderboard.model_validate(resp.json())

Public, multi-dimensional model leaderboard (no auth required).

EvalResource(http: ctfy.sdk.base.BaseHttpClient) 
14    def __init__(self, http: BaseHttpClient) -> None:
15        self._http = http
def leaderboard( self, *, competition_id: str = '', tier: str = 'official') -> ctfy.server.models.EvalLeaderboard: 
17    def leaderboard(self, *, competition_id: str = "", tier: str = "official") -> EvalLeaderboard:
18        """Ranked model standings + per-dimension (difficulty / bucket / tag)
19        breakdowns. ``competition_id`` scopes to one competition; empty = global.
20
21        ``tier`` selects the credibility band: ``official`` (default,
22        platform-run), ``self_reported`` (vendor-run, unverified), or ``all``
23        for both. Sandbox runs are never exposed here."""
24        resp = self._http.request(
25            "GET", "/eval/leaderboard", params={"competition_id": competition_id, "tier": tier}
26        )
27        _raise_for_status(resp)
28        return EvalLeaderboard.model_validate(resp.json())

Ranked model standings + per-dimension (difficulty / bucket / tag) breakdowns. competition_id scopes to one competition; empty = global.

tier selects the credibility band: official (default, platform-run), self_reported (vendor-run, unverified), or all for both. Sandbox runs are never exposed here.