tooltangle report

{{ report.model }}, {{ report.created_at }}

accuracy	{{ "%.2f"\|format(report.accuracy) }} (95% interval {{ "%.2f"\|format(report.interval[0]) }} to {{ "%.2f"\|format(report.interval[1]) }})
messages	{{ report.evaluated }}{% if report.failed %}, {{ report.failed }} failed{% endif %}
tools	{{ report.tools\|length }}
definitions	about {{ "{:,}".format(report.definition_tokens) }} tokens per request

Findings

{{ finding.severity|upper }} {{ finding.title }} {% if finding.detail %}{{ finding.detail }}{% endif %} {% for example in finding.examples %}
e.g. {{ example }}
{% endfor %}
Nothing to report.

{% if matrix.labels %}

Confusion matrix

Rows: the tool a message was written for. Columns: the tool the model called.

{% for label in matrix.labels %}{% endfor %} {% for expected in matrix.labels %} {% for picked in matrix.labels %} {% set count = matrix.cells.get(expected ~ "|" ~ picked, 0) %} {% endfor %} {% endfor %}

	{{ label }}
{{ expected }}	{{ count or "" }}

{% endif %}

Tools

{% for key, tool in tools %} {% endfor %}

tool	messages	accuracy	precision	description
`{{ key }}`	{{ tool.queries }}	{{ "%.0f%%"\|format(tool.accuracy * 100) if tool.accuracy is not none else "-" }}	{{ "%.0f%%"\|format(tool.precision * 100) if tool.precision is not none else "-" }}	{{ tool.description\|truncate(160) }}

Messages

only mistakes

{% for row in report.queries %} {% endfor %}

message	written for	model called	split
{{ row.text }}	`{{ row.tool or "no tool" }}` {% if row.accepted\|length > 1 %} also fine: {{ row.accepted\|reject("equalto", row.tool)\|join(", ") }} {% endif %}	`{{ row.picked\|pick }}`	{{ row.split }}