ctfy.sdk.admin_resources.eval
client.admin.eval_models + client.admin.eval_runs — model evaluation (admin).
1"""``client.admin.eval_models`` + ``client.admin.eval_runs`` — model evaluation (admin).""" 2 3from __future__ import annotations 4 5import builtins 6from typing import Any 7 8from ctfy.core.state.models import ( 9 EvalCampaignState, 10 EvalModelState, 11 EvalRunState, 12 HarnessState, 13 VendorState, 14 VendorTokenState, 15) 16from ctfy.sdk._helpers import _raise_for_status 17from ctfy.sdk.base import BaseHttpClient 18from ctfy.server.models import AdminTaskInfo 19 20 21class AdminEvalModelsResource: 22 """The registry of models under evaluation: register / list / delete. 23 24 No API key is ever stored — a registered model is addressing only 25 (flavor + base_url + provider model id); credentials are supplied to the 26 run at dispatch time via env. 27 """ 28 29 def __init__(self, http: BaseHttpClient) -> None: 30 self._http = http 31 32 def list( 33 self, *, active_only: bool = False, offset: int = 0, limit: int = 100 34 ) -> builtins.list[EvalModelState]: 35 resp = self._http.request( 36 "GET", 37 "/admin/eval-models", 38 params={"active_only": active_only, "offset": offset, "limit": limit}, 39 ) 40 _raise_for_status(resp) 41 return [EvalModelState.model_validate(row) for row in resp.json()] 42 43 def register( 44 self, 45 *, 46 id: str, 47 flavor: str, 48 display_name: str = "", 49 provider_model: str = "", 50 base_url: str = "", 51 vendor: str = "", 52 notes: str = "", 53 input_price_per_mtok: float = 0.0, 54 output_price_per_mtok: float = 0.0, 55 ) -> EvalModelState: 56 """Register (or overwrite) a model. ``flavor`` is ``anthropic`` / ``openai``. 57 58 ``*_price_per_mtok`` is USD per 1M tokens (0 = unpriced) — drives the 59 leaderboard's cost columns. 60 """ 61 resp = self._http.request( 62 "POST", 63 "/admin/eval-models", 64 json={ 65 "id": id, 66 "flavor": flavor, 67 "display_name": display_name, 68 "provider_model": provider_model, 69 "base_url": base_url, 70 "vendor": vendor, 71 "notes": notes, 72 "input_price_per_mtok": input_price_per_mtok, 73 "output_price_per_mtok": output_price_per_mtok, 74 }, 75 ) 76 _raise_for_status(resp) 77 return EvalModelState.model_validate(resp.json()) 78 79 def delete(self, eval_model_id: str) -> bool: 80 resp = self._http.request("DELETE", f"/admin/eval-models/{eval_model_id}") 81 _raise_for_status(resp) 82 return bool(resp.json().get("deleted")) 83 84 def review(self, eval_model_id: str, *, status: str, note: str = "") -> EvalModelState: 85 """Approve / reject a model. ``status`` is ``approved`` / ``rejected`` / 86 ``pending``; only ``approved`` models may back an official campaign.""" 87 resp = self._http.request( 88 "POST", 89 f"/admin/eval-models/{eval_model_id}/review", 90 json={"status": status, "note": note}, 91 ) 92 _raise_for_status(resp) 93 return EvalModelState.model_validate(resp.json()) 94 95 96class AdminEvalVendorsResource: 97 """The registry of model vendors / orgs: register / list / delete. 98 99 A vendor's ``id`` is a slug matching the ``vendor`` label on each of its 100 registered models, so the leaderboard's vendor scorecard resolves display 101 metadata from one curated row. 102 """ 103 104 def __init__(self, http: BaseHttpClient) -> None: 105 self._http = http 106 107 def list( 108 self, *, active_only: bool = False, offset: int = 0, limit: int = 100 109 ) -> builtins.list[VendorState]: 110 resp = self._http.request( 111 "GET", 112 "/admin/eval-vendors", 113 params={"active_only": active_only, "offset": offset, "limit": limit}, 114 ) 115 _raise_for_status(resp) 116 return [VendorState.model_validate(row) for row in resp.json()] 117 118 def register( 119 self, 120 *, 121 id: str, 122 display_name: str = "", 123 homepage: str = "", 124 contact_email: str = "", 125 active: bool = True, 126 notes: str = "", 127 max_concurrent_campaigns: int = 1, 128 daily_run_quota: int = 200, 129 monthly_cost_budget_usd: float = 0.0, 130 ) -> VendorState: 131 """Register (or overwrite) a vendor by slug ``id``. 132 133 The quota trio caps the vendor's self-service campaign usage 134 (docs/model-eval-vendor-tenancy.md §6.2); ``0`` on any axis means "no 135 limit" there. A re-register overwrites the whole row, so pass the 136 current quota values to keep them.""" 137 resp = self._http.request( 138 "POST", 139 "/admin/eval-vendors", 140 json={ 141 "id": id, 142 "display_name": display_name, 143 "homepage": homepage, 144 "contact_email": contact_email, 145 "active": active, 146 "notes": notes, 147 "max_concurrent_campaigns": max_concurrent_campaigns, 148 "daily_run_quota": daily_run_quota, 149 "monthly_cost_budget_usd": monthly_cost_budget_usd, 150 }, 151 ) 152 _raise_for_status(resp) 153 return VendorState.model_validate(resp.json()) 154 155 def delete(self, vendor_id: str) -> bool: 156 resp = self._http.request("DELETE", f"/admin/eval-vendors/{vendor_id}") 157 _raise_for_status(resp) 158 return bool(resp.json().get("deleted")) 159 160 # -- vendor access tokens (tenancy) ------------------------------------- 161 162 def mint_token(self, vendor_id: str, *, label: str = "", scope: str = "read") -> dict[str, Any]: 163 """Mint a ``pv_`` vendor bearer. The plaintext ``token`` in the reply 164 is shown **once** — store it now; only its hash is persisted. ``scope`` 165 is ``read`` (default) or ``submit`` (self-service writes).""" 166 resp = self._http.request( 167 "POST", 168 f"/admin/eval-vendors/{vendor_id}/tokens", 169 json={"label": label, "scope": scope}, 170 ) 171 _raise_for_status(resp) 172 data: dict[str, Any] = resp.json() 173 return data 174 175 def list_tokens(self, vendor_id: str) -> builtins.list[VendorTokenState]: 176 """List a vendor's tokens (hashes blanked server-side).""" 177 resp = self._http.request("GET", f"/admin/eval-vendors/{vendor_id}/tokens") 178 _raise_for_status(resp) 179 return [VendorTokenState.model_validate(row) for row in resp.json()] 180 181 def revoke_token(self, vendor_id: str, token_id: str) -> bool: 182 resp = self._http.request("DELETE", f"/admin/eval-vendors/{vendor_id}/tokens/{token_id}") 183 _raise_for_status(resp) 184 return bool(resp.json().get("deleted")) 185 186 187class AdminEvalHarnessesResource: 188 """The registry of eval harnesses (agentic scaffolds): register / list / delete. 189 190 A harness's ``id`` is a slug (``ctfy`` / ``claude-code`` / …); ``image`` is 191 the container a node runs for it. Lets harness be a comparison variable 192 alongside the model. 193 """ 194 195 def __init__(self, http: BaseHttpClient) -> None: 196 self._http = http 197 198 def list( 199 self, *, active_only: bool = False, offset: int = 0, limit: int = 100 200 ) -> builtins.list[HarnessState]: 201 resp = self._http.request( 202 "GET", 203 "/admin/eval-harnesses", 204 params={"active_only": active_only, "offset": offset, "limit": limit}, 205 ) 206 _raise_for_status(resp) 207 return [HarnessState.model_validate(row) for row in resp.json()] 208 209 def register( 210 self, 211 *, 212 id: str, 213 image: str = "", 214 display_name: str = "", 215 version: str = "", 216 notes: str = "", 217 active: bool = True, 218 ) -> HarnessState: 219 """Register (or overwrite) a harness by slug ``id``.""" 220 resp = self._http.request( 221 "POST", 222 "/admin/eval-harnesses", 223 json={ 224 "id": id, 225 "image": image, 226 "display_name": display_name, 227 "version": version, 228 "notes": notes, 229 "active": active, 230 }, 231 ) 232 _raise_for_status(resp) 233 return HarnessState.model_validate(resp.json()) 234 235 def delete(self, harness_id: str) -> bool: 236 resp = self._http.request("DELETE", f"/admin/eval-harnesses/{harness_id}") 237 _raise_for_status(resp) 238 return bool(resp.json().get("deleted")) 239 240 241class AdminEvalRunsResource: 242 """Submit + read model × challenge evaluation runs.""" 243 244 def __init__(self, http: BaseHttpClient) -> None: 245 self._http = http 246 247 def list( 248 self, 249 *, 250 model_id: str = "", 251 competition_id: str = "", 252 status: str = "", 253 offset: int = 0, 254 limit: int = 50, 255 ) -> builtins.list[EvalRunState]: 256 resp = self._http.request( 257 "GET", 258 "/admin/eval-runs", 259 params={ 260 "model_id": model_id, 261 "competition_id": competition_id, 262 "status": status, 263 "offset": offset, 264 "limit": limit, 265 }, 266 ) 267 _raise_for_status(resp) 268 return [EvalRunState.model_validate(row) for row in resp.json()] 269 270 def get(self, eval_run_id: str) -> EvalRunState: 271 resp = self._http.request("GET", f"/admin/eval-runs/{eval_run_id}") 272 _raise_for_status(resp) 273 return EvalRunState.model_validate(resp.json()) 274 275 def submit( 276 self, 277 *, 278 eval_model_id: str, 279 challenge_id: str, 280 competition_id: str = "", 281 harness_id: str = "", 282 max_turns: int | None = None, 283 samples: int = 1, 284 ) -> AdminTaskInfo: 285 """Queue a ``model_eval_run`` task for one model × challenge. ``harness_id`` 286 selects the agentic scaffold (``""`` = the built-in ctfy harness); 287 ``samples`` > 1 queues k runs (pass@k). Returns the first pending task; 288 poll :meth:`list` for the resulting run records.""" 289 body: dict[str, Any] = { 290 "eval_model_id": eval_model_id, 291 "challenge_id": challenge_id, 292 "competition_id": competition_id, 293 "harness_id": harness_id, 294 "samples": samples, 295 } 296 if max_turns is not None: 297 body["max_turns"] = max_turns 298 resp = self._http.request("POST", "/admin/eval-runs", json=body) 299 _raise_for_status(resp) 300 return AdminTaskInfo.model_validate(resp.json()) 301 302 303class AdminEvalCampaignsResource: 304 """Submit + read operator batch evaluations (a model over a standard set).""" 305 306 def __init__(self, http: BaseHttpClient) -> None: 307 self._http = http 308 309 def list( 310 self, 311 *, 312 model_id: str = "", 313 vendor_id: str = "", 314 status: str = "", 315 tier: str = "", 316 offset: int = 0, 317 limit: int = 50, 318 ) -> builtins.list[EvalCampaignState]: 319 resp = self._http.request( 320 "GET", 321 "/admin/eval-campaigns", 322 params={ 323 "model_id": model_id, 324 "vendor_id": vendor_id, 325 "status": status, 326 "tier": tier, 327 "offset": offset, 328 "limit": limit, 329 }, 330 ) 331 _raise_for_status(resp) 332 return [EvalCampaignState.model_validate(row) for row in resp.json()] 333 334 def get(self, eval_campaign_id: str) -> EvalCampaignState: 335 resp = self._http.request("GET", f"/admin/eval-campaigns/{eval_campaign_id}") 336 _raise_for_status(resp) 337 return EvalCampaignState.model_validate(resp.json()) 338 339 def submit( 340 self, 341 *, 342 model_id: str, 343 standard_set_id: str, 344 name: str = "", 345 harness_id: str = "", 346 num_samples: int = 1, 347 max_cost_usd: float = 0.0, 348 ) -> AdminTaskInfo: 349 """Queue an ``eval_campaign_run`` fan-out over a standard set. The model 350 must be ``approved``; the standard set (a competition id) must carry 351 challenges. Returns the driving task; poll :meth:`get` for the rollup.""" 352 resp = self._http.request( 353 "POST", 354 "/admin/eval-campaigns", 355 json={ 356 "model_id": model_id, 357 "standard_set_id": standard_set_id, 358 "name": name, 359 "harness_id": harness_id, 360 "num_samples": num_samples, 361 "max_cost_usd": max_cost_usd, 362 }, 363 ) 364 _raise_for_status(resp) 365 return AdminTaskInfo.model_validate(resp.json()) 366 367 def cancel(self, eval_campaign_id: str) -> EvalCampaignState: 368 """Cooperatively cancel a campaign via its driving task.""" 369 resp = self._http.request("POST", f"/admin/eval-campaigns/{eval_campaign_id}/cancel") 370 _raise_for_status(resp) 371 return EvalCampaignState.model_validate(resp.json())
class
AdminEvalModelsResource:
22class AdminEvalModelsResource: 23 """The registry of models under evaluation: register / list / delete. 24 25 No API key is ever stored — a registered model is addressing only 26 (flavor + base_url + provider model id); credentials are supplied to the 27 run at dispatch time via env. 28 """ 29 30 def __init__(self, http: BaseHttpClient) -> None: 31 self._http = http 32 33 def list( 34 self, *, active_only: bool = False, offset: int = 0, limit: int = 100 35 ) -> builtins.list[EvalModelState]: 36 resp = self._http.request( 37 "GET", 38 "/admin/eval-models", 39 params={"active_only": active_only, "offset": offset, "limit": limit}, 40 ) 41 _raise_for_status(resp) 42 return [EvalModelState.model_validate(row) for row in resp.json()] 43 44 def register( 45 self, 46 *, 47 id: str, 48 flavor: str, 49 display_name: str = "", 50 provider_model: str = "", 51 base_url: str = "", 52 vendor: str = "", 53 notes: str = "", 54 input_price_per_mtok: float = 0.0, 55 output_price_per_mtok: float = 0.0, 56 ) -> EvalModelState: 57 """Register (or overwrite) a model. ``flavor`` is ``anthropic`` / ``openai``. 58 59 ``*_price_per_mtok`` is USD per 1M tokens (0 = unpriced) — drives the 60 leaderboard's cost columns. 61 """ 62 resp = self._http.request( 63 "POST", 64 "/admin/eval-models", 65 json={ 66 "id": id, 67 "flavor": flavor, 68 "display_name": display_name, 69 "provider_model": provider_model, 70 "base_url": base_url, 71 "vendor": vendor, 72 "notes": notes, 73 "input_price_per_mtok": input_price_per_mtok, 74 "output_price_per_mtok": output_price_per_mtok, 75 }, 76 ) 77 _raise_for_status(resp) 78 return EvalModelState.model_validate(resp.json()) 79 80 def delete(self, eval_model_id: str) -> bool: 81 resp = self._http.request("DELETE", f"/admin/eval-models/{eval_model_id}") 82 _raise_for_status(resp) 83 return bool(resp.json().get("deleted")) 84 85 def review(self, eval_model_id: str, *, status: str, note: str = "") -> EvalModelState: 86 """Approve / reject a model. ``status`` is ``approved`` / ``rejected`` / 87 ``pending``; only ``approved`` models may back an official campaign.""" 88 resp = self._http.request( 89 "POST", 90 f"/admin/eval-models/{eval_model_id}/review", 91 json={"status": status, "note": note}, 92 ) 93 _raise_for_status(resp) 94 return EvalModelState.model_validate(resp.json())
The registry of models under evaluation: register / list / delete.
No API key is ever stored — a registered model is addressing only (flavor + base_url + provider model id); credentials are supplied to the run at dispatch time via env.
def
list( self, *, active_only: bool = False, offset: int = 0, limit: int = 100) -> list[ctfy.core.state.models.EvalModelState]:
33 def list( 34 self, *, active_only: bool = False, offset: int = 0, limit: int = 100 35 ) -> builtins.list[EvalModelState]: 36 resp = self._http.request( 37 "GET", 38 "/admin/eval-models", 39 params={"active_only": active_only, "offset": offset, "limit": limit}, 40 ) 41 _raise_for_status(resp) 42 return [EvalModelState.model_validate(row) for row in resp.json()]
def
register( self, *, id: str, flavor: str, display_name: str = '', provider_model: str = '', base_url: str = '', vendor: str = '', notes: str = '', input_price_per_mtok: float = 0.0, output_price_per_mtok: float = 0.0) -> ctfy.core.state.models.EvalModelState:
44 def register( 45 self, 46 *, 47 id: str, 48 flavor: str, 49 display_name: str = "", 50 provider_model: str = "", 51 base_url: str = "", 52 vendor: str = "", 53 notes: str = "", 54 input_price_per_mtok: float = 0.0, 55 output_price_per_mtok: float = 0.0, 56 ) -> EvalModelState: 57 """Register (or overwrite) a model. ``flavor`` is ``anthropic`` / ``openai``. 58 59 ``*_price_per_mtok`` is USD per 1M tokens (0 = unpriced) — drives the 60 leaderboard's cost columns. 61 """ 62 resp = self._http.request( 63 "POST", 64 "/admin/eval-models", 65 json={ 66 "id": id, 67 "flavor": flavor, 68 "display_name": display_name, 69 "provider_model": provider_model, 70 "base_url": base_url, 71 "vendor": vendor, 72 "notes": notes, 73 "input_price_per_mtok": input_price_per_mtok, 74 "output_price_per_mtok": output_price_per_mtok, 75 }, 76 ) 77 _raise_for_status(resp) 78 return EvalModelState.model_validate(resp.json())
Register (or overwrite) a model. flavor is anthropic / openai.
*_price_per_mtok is USD per 1M tokens (0 = unpriced) — drives the
leaderboard's cost columns.
def
review( self, eval_model_id: str, *, status: str, note: str = '') -> ctfy.core.state.models.EvalModelState:
85 def review(self, eval_model_id: str, *, status: str, note: str = "") -> EvalModelState: 86 """Approve / reject a model. ``status`` is ``approved`` / ``rejected`` / 87 ``pending``; only ``approved`` models may back an official campaign.""" 88 resp = self._http.request( 89 "POST", 90 f"/admin/eval-models/{eval_model_id}/review", 91 json={"status": status, "note": note}, 92 ) 93 _raise_for_status(resp) 94 return EvalModelState.model_validate(resp.json())
Approve / reject a model. status is approved / rejected /
pending; only approved models may back an official campaign.
class
AdminEvalVendorsResource:
97class AdminEvalVendorsResource: 98 """The registry of model vendors / orgs: register / list / delete. 99 100 A vendor's ``id`` is a slug matching the ``vendor`` label on each of its 101 registered models, so the leaderboard's vendor scorecard resolves display 102 metadata from one curated row. 103 """ 104 105 def __init__(self, http: BaseHttpClient) -> None: 106 self._http = http 107 108 def list( 109 self, *, active_only: bool = False, offset: int = 0, limit: int = 100 110 ) -> builtins.list[VendorState]: 111 resp = self._http.request( 112 "GET", 113 "/admin/eval-vendors", 114 params={"active_only": active_only, "offset": offset, "limit": limit}, 115 ) 116 _raise_for_status(resp) 117 return [VendorState.model_validate(row) for row in resp.json()] 118 119 def register( 120 self, 121 *, 122 id: str, 123 display_name: str = "", 124 homepage: str = "", 125 contact_email: str = "", 126 active: bool = True, 127 notes: str = "", 128 max_concurrent_campaigns: int = 1, 129 daily_run_quota: int = 200, 130 monthly_cost_budget_usd: float = 0.0, 131 ) -> VendorState: 132 """Register (or overwrite) a vendor by slug ``id``. 133 134 The quota trio caps the vendor's self-service campaign usage 135 (docs/model-eval-vendor-tenancy.md §6.2); ``0`` on any axis means "no 136 limit" there. A re-register overwrites the whole row, so pass the 137 current quota values to keep them.""" 138 resp = self._http.request( 139 "POST", 140 "/admin/eval-vendors", 141 json={ 142 "id": id, 143 "display_name": display_name, 144 "homepage": homepage, 145 "contact_email": contact_email, 146 "active": active, 147 "notes": notes, 148 "max_concurrent_campaigns": max_concurrent_campaigns, 149 "daily_run_quota": daily_run_quota, 150 "monthly_cost_budget_usd": monthly_cost_budget_usd, 151 }, 152 ) 153 _raise_for_status(resp) 154 return VendorState.model_validate(resp.json()) 155 156 def delete(self, vendor_id: str) -> bool: 157 resp = self._http.request("DELETE", f"/admin/eval-vendors/{vendor_id}") 158 _raise_for_status(resp) 159 return bool(resp.json().get("deleted")) 160 161 # -- vendor access tokens (tenancy) ------------------------------------- 162 163 def mint_token(self, vendor_id: str, *, label: str = "", scope: str = "read") -> dict[str, Any]: 164 """Mint a ``pv_`` vendor bearer. The plaintext ``token`` in the reply 165 is shown **once** — store it now; only its hash is persisted. ``scope`` 166 is ``read`` (default) or ``submit`` (self-service writes).""" 167 resp = self._http.request( 168 "POST", 169 f"/admin/eval-vendors/{vendor_id}/tokens", 170 json={"label": label, "scope": scope}, 171 ) 172 _raise_for_status(resp) 173 data: dict[str, Any] = resp.json() 174 return data 175 176 def list_tokens(self, vendor_id: str) -> builtins.list[VendorTokenState]: 177 """List a vendor's tokens (hashes blanked server-side).""" 178 resp = self._http.request("GET", f"/admin/eval-vendors/{vendor_id}/tokens") 179 _raise_for_status(resp) 180 return [VendorTokenState.model_validate(row) for row in resp.json()] 181 182 def revoke_token(self, vendor_id: str, token_id: str) -> bool: 183 resp = self._http.request("DELETE", f"/admin/eval-vendors/{vendor_id}/tokens/{token_id}") 184 _raise_for_status(resp) 185 return bool(resp.json().get("deleted"))
The registry of model vendors / orgs: register / list / delete.
A vendor's id is a slug matching the vendor label on each of its
registered models, so the leaderboard's vendor scorecard resolves display
metadata from one curated row.
def
list( self, *, active_only: bool = False, offset: int = 0, limit: int = 100) -> list[ctfy.core.state.models.VendorState]:
108 def list( 109 self, *, active_only: bool = False, offset: int = 0, limit: int = 100 110 ) -> builtins.list[VendorState]: 111 resp = self._http.request( 112 "GET", 113 "/admin/eval-vendors", 114 params={"active_only": active_only, "offset": offset, "limit": limit}, 115 ) 116 _raise_for_status(resp) 117 return [VendorState.model_validate(row) for row in resp.json()]
def
register( self, *, id: str, display_name: str = '', homepage: str = '', contact_email: str = '', active: bool = True, notes: str = '', max_concurrent_campaigns: int = 1, daily_run_quota: int = 200, monthly_cost_budget_usd: float = 0.0) -> ctfy.core.state.models.VendorState:
119 def register( 120 self, 121 *, 122 id: str, 123 display_name: str = "", 124 homepage: str = "", 125 contact_email: str = "", 126 active: bool = True, 127 notes: str = "", 128 max_concurrent_campaigns: int = 1, 129 daily_run_quota: int = 200, 130 monthly_cost_budget_usd: float = 0.0, 131 ) -> VendorState: 132 """Register (or overwrite) a vendor by slug ``id``. 133 134 The quota trio caps the vendor's self-service campaign usage 135 (docs/model-eval-vendor-tenancy.md §6.2); ``0`` on any axis means "no 136 limit" there. A re-register overwrites the whole row, so pass the 137 current quota values to keep them.""" 138 resp = self._http.request( 139 "POST", 140 "/admin/eval-vendors", 141 json={ 142 "id": id, 143 "display_name": display_name, 144 "homepage": homepage, 145 "contact_email": contact_email, 146 "active": active, 147 "notes": notes, 148 "max_concurrent_campaigns": max_concurrent_campaigns, 149 "daily_run_quota": daily_run_quota, 150 "monthly_cost_budget_usd": monthly_cost_budget_usd, 151 }, 152 ) 153 _raise_for_status(resp) 154 return VendorState.model_validate(resp.json())
Register (or overwrite) a vendor by slug id.
The quota trio caps the vendor's self-service campaign usage
(docs/model-eval-vendor-tenancy.md §6.2); 0 on any axis means "no
limit" there. A re-register overwrites the whole row, so pass the
current quota values to keep them.
def
mint_token( self, vendor_id: str, *, label: str = '', scope: str = 'read') -> dict[str, typing.Any]:
163 def mint_token(self, vendor_id: str, *, label: str = "", scope: str = "read") -> dict[str, Any]: 164 """Mint a ``pv_`` vendor bearer. The plaintext ``token`` in the reply 165 is shown **once** — store it now; only its hash is persisted. ``scope`` 166 is ``read`` (default) or ``submit`` (self-service writes).""" 167 resp = self._http.request( 168 "POST", 169 f"/admin/eval-vendors/{vendor_id}/tokens", 170 json={"label": label, "scope": scope}, 171 ) 172 _raise_for_status(resp) 173 data: dict[str, Any] = resp.json() 174 return data
Mint a pv_ vendor bearer. The plaintext token in the reply
is shown once — store it now; only its hash is persisted. scope
is read (default) or submit (self-service writes).
def
list_tokens(self, vendor_id: str) -> list[ctfy.core.state.models.VendorTokenState]:
176 def list_tokens(self, vendor_id: str) -> builtins.list[VendorTokenState]: 177 """List a vendor's tokens (hashes blanked server-side).""" 178 resp = self._http.request("GET", f"/admin/eval-vendors/{vendor_id}/tokens") 179 _raise_for_status(resp) 180 return [VendorTokenState.model_validate(row) for row in resp.json()]
List a vendor's tokens (hashes blanked server-side).
class
AdminEvalHarnessesResource:
188class AdminEvalHarnessesResource: 189 """The registry of eval harnesses (agentic scaffolds): register / list / delete. 190 191 A harness's ``id`` is a slug (``ctfy`` / ``claude-code`` / …); ``image`` is 192 the container a node runs for it. Lets harness be a comparison variable 193 alongside the model. 194 """ 195 196 def __init__(self, http: BaseHttpClient) -> None: 197 self._http = http 198 199 def list( 200 self, *, active_only: bool = False, offset: int = 0, limit: int = 100 201 ) -> builtins.list[HarnessState]: 202 resp = self._http.request( 203 "GET", 204 "/admin/eval-harnesses", 205 params={"active_only": active_only, "offset": offset, "limit": limit}, 206 ) 207 _raise_for_status(resp) 208 return [HarnessState.model_validate(row) for row in resp.json()] 209 210 def register( 211 self, 212 *, 213 id: str, 214 image: str = "", 215 display_name: str = "", 216 version: str = "", 217 notes: str = "", 218 active: bool = True, 219 ) -> HarnessState: 220 """Register (or overwrite) a harness by slug ``id``.""" 221 resp = self._http.request( 222 "POST", 223 "/admin/eval-harnesses", 224 json={ 225 "id": id, 226 "image": image, 227 "display_name": display_name, 228 "version": version, 229 "notes": notes, 230 "active": active, 231 }, 232 ) 233 _raise_for_status(resp) 234 return HarnessState.model_validate(resp.json()) 235 236 def delete(self, harness_id: str) -> bool: 237 resp = self._http.request("DELETE", f"/admin/eval-harnesses/{harness_id}") 238 _raise_for_status(resp) 239 return bool(resp.json().get("deleted"))
The registry of eval harnesses (agentic scaffolds): register / list / delete.
A harness's id is a slug (ctfy / claude-code / …); image is
the container a node runs for it. Lets harness be a comparison variable
alongside the model.
def
list( self, *, active_only: bool = False, offset: int = 0, limit: int = 100) -> list[ctfy.core.state.models.HarnessState]:
199 def list( 200 self, *, active_only: bool = False, offset: int = 0, limit: int = 100 201 ) -> builtins.list[HarnessState]: 202 resp = self._http.request( 203 "GET", 204 "/admin/eval-harnesses", 205 params={"active_only": active_only, "offset": offset, "limit": limit}, 206 ) 207 _raise_for_status(resp) 208 return [HarnessState.model_validate(row) for row in resp.json()]
def
register( self, *, id: str, image: str = '', display_name: str = '', version: str = '', notes: str = '', active: bool = True) -> ctfy.core.state.models.HarnessState:
210 def register( 211 self, 212 *, 213 id: str, 214 image: str = "", 215 display_name: str = "", 216 version: str = "", 217 notes: str = "", 218 active: bool = True, 219 ) -> HarnessState: 220 """Register (or overwrite) a harness by slug ``id``.""" 221 resp = self._http.request( 222 "POST", 223 "/admin/eval-harnesses", 224 json={ 225 "id": id, 226 "image": image, 227 "display_name": display_name, 228 "version": version, 229 "notes": notes, 230 "active": active, 231 }, 232 ) 233 _raise_for_status(resp) 234 return HarnessState.model_validate(resp.json())
Register (or overwrite) a harness by slug id.
class
AdminEvalRunsResource:
242class AdminEvalRunsResource: 243 """Submit + read model × challenge evaluation runs.""" 244 245 def __init__(self, http: BaseHttpClient) -> None: 246 self._http = http 247 248 def list( 249 self, 250 *, 251 model_id: str = "", 252 competition_id: str = "", 253 status: str = "", 254 offset: int = 0, 255 limit: int = 50, 256 ) -> builtins.list[EvalRunState]: 257 resp = self._http.request( 258 "GET", 259 "/admin/eval-runs", 260 params={ 261 "model_id": model_id, 262 "competition_id": competition_id, 263 "status": status, 264 "offset": offset, 265 "limit": limit, 266 }, 267 ) 268 _raise_for_status(resp) 269 return [EvalRunState.model_validate(row) for row in resp.json()] 270 271 def get(self, eval_run_id: str) -> EvalRunState: 272 resp = self._http.request("GET", f"/admin/eval-runs/{eval_run_id}") 273 _raise_for_status(resp) 274 return EvalRunState.model_validate(resp.json()) 275 276 def submit( 277 self, 278 *, 279 eval_model_id: str, 280 challenge_id: str, 281 competition_id: str = "", 282 harness_id: str = "", 283 max_turns: int | None = None, 284 samples: int = 1, 285 ) -> AdminTaskInfo: 286 """Queue a ``model_eval_run`` task for one model × challenge. ``harness_id`` 287 selects the agentic scaffold (``""`` = the built-in ctfy harness); 288 ``samples`` > 1 queues k runs (pass@k). Returns the first pending task; 289 poll :meth:`list` for the resulting run records.""" 290 body: dict[str, Any] = { 291 "eval_model_id": eval_model_id, 292 "challenge_id": challenge_id, 293 "competition_id": competition_id, 294 "harness_id": harness_id, 295 "samples": samples, 296 } 297 if max_turns is not None: 298 body["max_turns"] = max_turns 299 resp = self._http.request("POST", "/admin/eval-runs", json=body) 300 _raise_for_status(resp) 301 return AdminTaskInfo.model_validate(resp.json())
Submit + read model × challenge evaluation runs.
def
list( self, *, model_id: str = '', competition_id: str = '', status: str = '', offset: int = 0, limit: int = 50) -> list[ctfy.core.state.models.EvalRunState]:
248 def list( 249 self, 250 *, 251 model_id: str = "", 252 competition_id: str = "", 253 status: str = "", 254 offset: int = 0, 255 limit: int = 50, 256 ) -> builtins.list[EvalRunState]: 257 resp = self._http.request( 258 "GET", 259 "/admin/eval-runs", 260 params={ 261 "model_id": model_id, 262 "competition_id": competition_id, 263 "status": status, 264 "offset": offset, 265 "limit": limit, 266 }, 267 ) 268 _raise_for_status(resp) 269 return [EvalRunState.model_validate(row) for row in resp.json()]
def
submit( self, *, eval_model_id: str, challenge_id: str, competition_id: str = '', harness_id: str = '', max_turns: int | None = None, samples: int = 1) -> ctfy.server.models.AdminTaskInfo:
276 def submit( 277 self, 278 *, 279 eval_model_id: str, 280 challenge_id: str, 281 competition_id: str = "", 282 harness_id: str = "", 283 max_turns: int | None = None, 284 samples: int = 1, 285 ) -> AdminTaskInfo: 286 """Queue a ``model_eval_run`` task for one model × challenge. ``harness_id`` 287 selects the agentic scaffold (``""`` = the built-in ctfy harness); 288 ``samples`` > 1 queues k runs (pass@k). Returns the first pending task; 289 poll :meth:`list` for the resulting run records.""" 290 body: dict[str, Any] = { 291 "eval_model_id": eval_model_id, 292 "challenge_id": challenge_id, 293 "competition_id": competition_id, 294 "harness_id": harness_id, 295 "samples": samples, 296 } 297 if max_turns is not None: 298 body["max_turns"] = max_turns 299 resp = self._http.request("POST", "/admin/eval-runs", json=body) 300 _raise_for_status(resp) 301 return AdminTaskInfo.model_validate(resp.json())
Queue a model_eval_run task for one model × challenge. harness_id
selects the agentic scaffold ("" = the built-in ctfy harness);
samples > 1 queues k runs (pass@k). Returns the first pending task;
poll list() for the resulting run records.
class
AdminEvalCampaignsResource:
304class AdminEvalCampaignsResource: 305 """Submit + read operator batch evaluations (a model over a standard set).""" 306 307 def __init__(self, http: BaseHttpClient) -> None: 308 self._http = http 309 310 def list( 311 self, 312 *, 313 model_id: str = "", 314 vendor_id: str = "", 315 status: str = "", 316 tier: str = "", 317 offset: int = 0, 318 limit: int = 50, 319 ) -> builtins.list[EvalCampaignState]: 320 resp = self._http.request( 321 "GET", 322 "/admin/eval-campaigns", 323 params={ 324 "model_id": model_id, 325 "vendor_id": vendor_id, 326 "status": status, 327 "tier": tier, 328 "offset": offset, 329 "limit": limit, 330 }, 331 ) 332 _raise_for_status(resp) 333 return [EvalCampaignState.model_validate(row) for row in resp.json()] 334 335 def get(self, eval_campaign_id: str) -> EvalCampaignState: 336 resp = self._http.request("GET", f"/admin/eval-campaigns/{eval_campaign_id}") 337 _raise_for_status(resp) 338 return EvalCampaignState.model_validate(resp.json()) 339 340 def submit( 341 self, 342 *, 343 model_id: str, 344 standard_set_id: str, 345 name: str = "", 346 harness_id: str = "", 347 num_samples: int = 1, 348 max_cost_usd: float = 0.0, 349 ) -> AdminTaskInfo: 350 """Queue an ``eval_campaign_run`` fan-out over a standard set. The model 351 must be ``approved``; the standard set (a competition id) must carry 352 challenges. Returns the driving task; poll :meth:`get` for the rollup.""" 353 resp = self._http.request( 354 "POST", 355 "/admin/eval-campaigns", 356 json={ 357 "model_id": model_id, 358 "standard_set_id": standard_set_id, 359 "name": name, 360 "harness_id": harness_id, 361 "num_samples": num_samples, 362 "max_cost_usd": max_cost_usd, 363 }, 364 ) 365 _raise_for_status(resp) 366 return AdminTaskInfo.model_validate(resp.json()) 367 368 def cancel(self, eval_campaign_id: str) -> EvalCampaignState: 369 """Cooperatively cancel a campaign via its driving task.""" 370 resp = self._http.request("POST", f"/admin/eval-campaigns/{eval_campaign_id}/cancel") 371 _raise_for_status(resp) 372 return EvalCampaignState.model_validate(resp.json())
Submit + read operator batch evaluations (a model over a standard set).
def
list( self, *, model_id: str = '', vendor_id: str = '', status: str = '', tier: str = '', offset: int = 0, limit: int = 50) -> list[ctfy.core.state.models.EvalCampaignState]:
310 def list( 311 self, 312 *, 313 model_id: str = "", 314 vendor_id: str = "", 315 status: str = "", 316 tier: str = "", 317 offset: int = 0, 318 limit: int = 50, 319 ) -> builtins.list[EvalCampaignState]: 320 resp = self._http.request( 321 "GET", 322 "/admin/eval-campaigns", 323 params={ 324 "model_id": model_id, 325 "vendor_id": vendor_id, 326 "status": status, 327 "tier": tier, 328 "offset": offset, 329 "limit": limit, 330 }, 331 ) 332 _raise_for_status(resp) 333 return [EvalCampaignState.model_validate(row) for row in resp.json()]
def
submit( self, *, model_id: str, standard_set_id: str, name: str = '', harness_id: str = '', num_samples: int = 1, max_cost_usd: float = 0.0) -> ctfy.server.models.AdminTaskInfo:
340 def submit( 341 self, 342 *, 343 model_id: str, 344 standard_set_id: str, 345 name: str = "", 346 harness_id: str = "", 347 num_samples: int = 1, 348 max_cost_usd: float = 0.0, 349 ) -> AdminTaskInfo: 350 """Queue an ``eval_campaign_run`` fan-out over a standard set. The model 351 must be ``approved``; the standard set (a competition id) must carry 352 challenges. Returns the driving task; poll :meth:`get` for the rollup.""" 353 resp = self._http.request( 354 "POST", 355 "/admin/eval-campaigns", 356 json={ 357 "model_id": model_id, 358 "standard_set_id": standard_set_id, 359 "name": name, 360 "harness_id": harness_id, 361 "num_samples": num_samples, 362 "max_cost_usd": max_cost_usd, 363 }, 364 ) 365 _raise_for_status(resp) 366 return AdminTaskInfo.model_validate(resp.json())
Queue an eval_campaign_run fan-out over a standard set. The model
must be approved; the standard set (a competition id) must carry
challenges. Returns the driving task; poll get() for the rollup.
def
cancel(self, eval_campaign_id: str) -> ctfy.core.state.models.EvalCampaignState:
368 def cancel(self, eval_campaign_id: str) -> EvalCampaignState: 369 """Cooperatively cancel a campaign via its driving task.""" 370 resp = self._http.request("POST", f"/admin/eval-campaigns/{eval_campaign_id}/cancel") 371 _raise_for_status(resp) 372 return EvalCampaignState.model_validate(resp.json())
Cooperatively cancel a campaign via its driving task.