{% extends layout %}{% import 'ui/kit.html' as kit %}
{% import 'ui/telemetry/_t.html' as t with context %}
{% block title %}Ingest protocol · amebo{% endblock %}
{%- macro field_table(caption, sub, fields) %}
<h3 class="doc-h3">{{ caption }} <span class="panel-sub">{{ sub }}</span></h3>
<div class="table-wrap">
  <table class="data">
    <thead><tr><th>field</th><th>type</th><th>required</th><th>notes</th></tr></thead>
    <tbody>{% for name, type, required, blurb in fields %}
      <tr><td class="mono fw">{{ name }}</td><td class="dim">{{ type }}</td><td class="dim">{{ required }}</td><td class="doc-note">{{ blurb }}</td></tr>
    {%- endfor %}</tbody>
  </table>
</div>
{%- endmacro %}
{%- macro code_table(codes) %}
<div class="table-wrap">
  <table class="data"><tbody>{% for status, tone, blurb in codes %}
    <tr><td class="nowrap"><span class="pill {{ tone }}">{{ status }}</span></td><td class="doc-note">{{ blurb }}</td></tr>
  {%- endfor %}</tbody></table>
</div>
{%- endmacro %}
{% block content %}
{{ t.css() }}
{%- set otlp_codes = [
  ('200', 'ok', 'Accepted. An empty Export*ServiceResponse in the encoding you sent: zero bytes of protobuf, or {} as JSON.'),
  ('400', 'warn', 'Undecodable protobuf or JSON, or Content-Encoding said gzip and the bytes were not.'),
  ('401', 'bad', 'Missing, unknown, revoked or deactivated api key (unless AMEBO_INGEST_OPEN=1).'),
  ('413', 'warn', 'The body decompressed past 32 MiB.'),
  ('415', 'warn', 'Wrong content type. Send application/x-protobuf or application/json.'),
  ('429', 'warn', 'This worker\'s ingest queue is full. Retry with backoff (Retry-After is set); nothing was written.')] %}
{%- set error_codes = [
  ('202', 'ok', 'Accepted. The body is {"accepted": n, "event_ids": [...]}, one id per report you sent.'),
  ('400', 'warn', 'The body is not valid JSON, or Content-Encoding said gzip and the bytes were not gzip.'),
  ('401', 'bad', 'Missing, unknown, revoked or deactivated api key.'),
  ('413', 'warn', 'The body decompressed past 1 MiB.'),
  ('422', 'warn', 'The envelope parsed but a report was unacceptable (or the batch carried more than 50). The reason is verbatim in the error field.'),
  ('429', 'warn', 'The error queue is full. Retry with backoff.')] %}
{%- set event_fields = [
  ('protocol', 'integer', 'no', 'Always 1. Absent means 1.'),
  ('event_id', 'string', 'no', '32 lowercase hex characters. Generated server side when absent. A repeat of an id already stored is dropped idempotently.'),
  ('timestamp', 'string or number', 'no', 'RFC3339, or unix seconds. Clamped to now when it is missing or implausible.'),
  ('level', 'string', 'no', 'One of fatal, error, warning, info, debug. Defaults to error.'),
  ('message', 'string', 'conditional', 'Required only when the report carries no exceptions.'),
  ('logger', 'string', 'no', 'The logger or subsystem that produced the report.'),
  ('platform', 'string', 'no', 'Free text, for example node, python, go.'),
  ('release', 'string', 'no', 'Version or commit. amebo stamps first and last release on the issue and uses it for regression context.'),
  ('environment', 'string', 'no', 'production, staging, and so on.'),
  ('server_name', 'string', 'no', 'Host or pod that produced the report.'),
  ('exceptions', 'array', 'conditional', 'At most 10, cause first and the outermost exception last. Required only when there is no message.'),
  ('breadcrumbs', 'array', 'no', 'At most 100, oldest first. What the process did before it broke.'),
  ('user', 'object', 'no', 'id, email, username, ip. Stored and counted, never used for grouping.'),
  ('tags', 'object', 'no', 'At most 32 string pairs. These become the tag distribution bars on the issue page. amebo.event_id here links the report to a business event.'),
  ('extra', 'object', 'no', 'Stored verbatim and never indexed.'),
  ('sdk', 'object', 'no', 'name and version of the client that sent this.'),
  ('trace_id', 'string', 'no', 'Correlates the report with a trace already in amebo. This is what puts a trace waterfall on the issue page.'),
  ('span_id', 'string', 'no', 'Correlates the report with one span of that trace.'),
  ('fingerprint', 'array', 'no', 'Overrides grouping entirely. Send this only when amebo groups something wrong.')] %}
{%- set exception_fields = [
  ('type', 'string', 'yes', 'The class name, for example TypeError. An exception entry without it is refused.'),
  ('value', 'string', 'no', 'The message carried by that exception.'),
  ('module', 'string', 'no', 'The module the exception class came from.'),
  ('frames', 'array', 'no', 'At most 128, oldest first, so the crash site is last.')] %}
{%- set frame_fields = [
  ('file', 'string', 'no', 'Path or URL. Normalized for grouping, stored as sent.'),
  ('function', 'string', 'no', 'Function or method name.'),
  ('line', 'integer', 'no', 'Line number. NEVER part of the fingerprint, so a refactor that shifts code does not fork the issue.'),
  ('column', 'integer', 'no', 'Column number.'),
  ('in_app', 'boolean', 'no', 'True for your code, false for a dependency. Only in_app frames are fingerprinted, and only they carry the In App badge.'),
  ('context_line', 'string', 'no', 'The source line itself. This is what renders under the frame.'),
  ('context_before', 'array', 'no', 'Source lines above.'),
  ('context_after', 'array', 'no', 'Source lines below.')] %}
<div class="section-head pin">
  <div>
    <div class="crumb"><span>Docs</span><span class="crumb-sep">/</span><span>Ingest</span></div>
    <h2>Ingest protocol</h2>
    <p class="sub">Four endpoints turn what your services are doing into traces, logs, metrics and issues. amebo stores all of it itself, in the same PostgreSQL or SQLite database as the broker: nothing is proxied to a third party and nothing leaves your network.</p>
  </div>
</div>

<div class="doc">
  <h2 class="doc-h2">Credentials</h2>
  <p>Every ingest endpoint authenticates with an application's api key, minted with
    <code>POST /v1/applications/:application/apikey</code> (or from the application's page) and sent as
    <code>Authorization: Bearer &lt;apikey&gt;</code> (<code>x-amebo-apikey: &lt;apikey&gt;</code> also works).
    The key decides which application the data belongs to: when a payload does not name its
    <code>service.name</code>, the key's application is used, and an error report is always filed under
    the key's application, so an exporter cannot report as another service.</p>
  <p>Setting <code>AMEBO_INGEST_OPEN=1</code> accepts telemetry without a key, for demos and local
    development only. Reading telemetry back (this dashboard, and <code>GET /v1/traces/:trace</code>,
    <code>/v1/logs</code>, <code>/v1/metrics/query</code>, <code>/v1/issues</code>) always needs a session
    or a credential. A superadmin sees every service; an application sees the services mapped to it
    plus unclaimed ones; a user sees the services of every application owning an action they may
    read, plus unclaimed ones.</p>

  <h2 class="doc-h2">Traces: OTLP over HTTP</h2>
  <div class="doc-ep"><span class="pill ok">POST</span> <code>/v1/traces</code></div>
  <p>This is stock OpenTelemetry. Point any OTel SDK at amebo and it works with no amebo library at all:</p>
  <pre class="codeblk">export OTEL_EXPORTER_OTLP_ENDPOINT=http://amebo:3310
export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
export OTEL_EXPORTER_OTLP_HEADERS="authorization=Bearer &lt;apikey&gt;"</pre>
  <p>The body is an <code>ExportTraceServiceRequest</code> as protobuf or OTLP JSON, gzip optional.
    Success is an empty response in the same encoding; errors are <code>google.rpc.Status</code>, per the
    OTLP spec. The resource's <code>service.name</code> stays the grouping truth, and amebo resolves it
    against the registered applications. A name nothing matches is reported as unclaimed on the
    <a class="crumb-link" onclick="dj.get('/attention', '#view', {push:true})">Needs attention</a> page
    rather than silently inventing a catalog entry for it.</p>
  {{ code_table(otlp_codes) }}

  <h3 class="doc-h3">What amebo does with a span</h3>
  <p>Every span is counted into per-minute aggregates keyed by service and route, which is where the
    percentiles come from. Those are exact at bounded storage, from base-2 exponential histograms, so a
    p99 over a week is the same arithmetic as a p99 over a minute and neither depends on sampling.
    Minutes are rolled up into hours after a day and kept for 30 days.</p>
  <p>Raw spans are sampled, because keeping every span forever buys nothing once the aggregates already
    counted them. Every error is kept, every span slower than <code>AMEBO_TRACE_SLOW_MS</code> is kept,
    and the rest keep <code>AMEBO_TRACE_SAMPLE_PCT</code> percent chosen by a hash of the trace id rather
    than a coin flip, so a kept trace is kept WHOLE and its waterfall never has holes. One exception: a
    trace carrying an amebo event is never sampled away, because those are exactly the traces the event
    detail page exists to show.</p>

  <h3 class="doc-h3">Two span attributes amebo reads</h3>
  <div class="table-wrap">
    <table class="data">
      <thead><tr><th>attribute</th><th>what it does</th></tr></thead>
      <tbody>
        <tr><td class="mono fw">amebo.event_id</td><td class="doc-note">Names the amebo event this span was processing. This is the consumer half of the correlation: it puts the trace on that event's detail page, and pins the trace against sampling. <code>messaging.message.id</code> works too.</td></tr>
        <tr><td class="mono fw">http.route</td><td class="doc-note">The route template the aggregates group by. Without it amebo groups by span name, which is usually fine and occasionally explodes the cardinality of an unparameterized URL (routes per service are capped).</td></tr>
      </tbody>
    </table>
  </div>
  <p>You often need neither. When the publisher is instrumented, amebo takes the W3C
    <code>traceparent</code> off <code>POST /v1/events</code> and stamps the same header on the outgoing
    webhook, so the consumer's SDK continues the producer's trace by itself and the correlation happens
    with no application code at all.</p>

  <h3 class="doc-h3">Exceptions with no SDK</h3>
  <p>A span event named <code>exception</code> carries the OpenTelemetry exception conventions:
    <code>exception.type</code>, <code>exception.message</code> and <code>exception.stacktrace</code>.
    amebo parses the stack trace, groups it, and you have error tracking from the tracing SDK you already
    installed. An unrecognised stack dialect degrades to message-based grouping rather than being dropped.</p>

  <h2 class="doc-h2">Logs</h2>
  <div class="doc-ep"><span class="pill ok">POST</span> <code>/v1/logs</code></div>
  <p>Either a stock OTLP <code>ExportLogsServiceRequest</code> (protobuf or JSON), which is what any
    OpenTelemetry SDK sends, or, for scripts and services with no SDK, <code>{"logs": [...]}</code> JSON
    where each line carries <code>message</code>, <code>severity</code>, an optional
    <code>timestamp</code> (seconds or nanoseconds), <code>trace_id</code>, <code>span_id</code> and
    <code>logger</code>; every other field becomes a searchable attribute. The JSON form answers
    <code>202 {"accepted": n}</code>. Lines below <code>AMEBO_LOG_MIN_SEVERITY</code> are dropped at
    ingest, and every line is counted per minute and severity, which is what the
    <a class="crumb-link" onclick="dj.get('/logs', '#view', {push:true})">Logs</a> volume chart reads.</p>
  <pre class="codeblk">curl -X POST http://amebo:3310/v1/logs \
  -H 'authorization: Bearer &lt;apikey&gt;' \
  -H 'content-type: application/json' \
  -d '{"logs": [{"message": "charged card", "severity": "info", "order": "o-8812"}]}'</pre>

  <h2 class="doc-h2">Metrics</h2>
  <div class="doc-ep"><span class="pill ok">POST</span> <code>/v1/metrics</code></div>
  <p>A stock OTLP <code>ExportMetricsServiceRequest</code>, same credential and same resource attributes
    as traces and logs. Every data point is folded into a per-minute bucket keyed by service, metric and
    attribute set, so the write rate is bounded by cardinality rather than by how often you export. A
    metric past 500 attribute sets folds its tail into one series; the
    <a class="crumb-link" onclick="dj.get('/metrics/browse', '#view', {push:true})">metric browser</a> shows
    the series count so the bill is visible.</p>

  <h2 class="doc-h2">Errors: the amebo report protocol</h2>
  <div class="doc-ep"><span class="pill ok">POST</span> <code>/v1/errors</code></div>
  <p>For everything an OTel span event cannot carry: a user, breadcrumbs, source context, your own tags.
    The body is one report object or <code>{"events": [...]}</code> carrying at most 50. Send
    <code>Content-Encoding: gzip</code> if you like; amebo refuses anything that decompresses past 1 MiB,
    and any single report past 256 KiB once normalized.</p>
  <p class="doc-warn">This is <b>not</b> <code>/v1/events</code>. That is the broker's business-event
    publish path and always will be: schema-validated against a registered action, persisted to the
    ledger, fanned out to subscribers. An error report is none of those things and lives in a different
    table with a different retention.</p>

  <h3 class="doc-h3">Smallest useful request</h3>
  <pre class="codeblk">curl -X POST http://amebo:3310/v1/errors \
  -H 'authorization: Bearer &lt;apikey&gt;' \
  -H 'content-type: application/json' \
  -d '{"message":"checkout failed for cart 8812","level":"error"}'</pre>

  <h3 class="doc-h3">A real exception</h3>
  <pre class="codeblk">curl -X POST http://amebo:3310/v1/errors \
  -H 'authorization: Bearer &lt;apikey&gt;' \
  -H 'content-type: application/json' \
  -d '{
    "level": "error",
    "release": "2026.9.1",
    "environment": "production",
    "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736",
    "tags": {"region": "eu-west-1", "amebo.event_id": "evt-order-88231"},
    "user": {"id": "u_9931"},
    "breadcrumbs": [
      {"type": "http", "category": "request", "level": "info", "message": "POST /v1/claims 201"}
    ],
    "exceptions": [{
      "type": "TypeError",
      "value": "Cannot read properties of undefined (reading planId)",
      "frames": [
        {"file": "/srv/app/server.js", "function": "main", "line": 12, "in_app": true},
        {"file": "/srv/app/pricing.js", "function": "priceClaim", "line": 88, "in_app": true,
         "context_line": "  return plan.rate * units"}
      ]
    }]
  }'</pre>
  {{ code_table(error_codes) }}

  {{ field_table('The report object', 'top level of one report', event_fields) }}
  {{ field_table('An exception entry', 'inside `exceptions`, cause first', exception_fields) }}
  {{ field_table('A stack frame', 'inside `frames`, oldest first', frame_fields) }}

  <h2 class="doc-h2">How reports become issues</h2>
  <p>Two reports with the same fingerprint are the same issue. The material is the exception type, the
    service, and the last five in-app frames as path and function. It deliberately EXCLUDES line numbers,
    so a refactor that shifts code does not fork an issue, and it deliberately INCLUDES the service, so the
    same bug in two services is two pages with two owners. With no exception, the material is the level
    and the message with its volatile parts collapsed, so "user 4231 not found" and "user 998 not found"
    group together.</p>
  <p>Send <code>fingerprint</code> to override all of that. It is the escape hatch for the case where
    amebo groups something wrong, and it should be rare. A resolved issue that sees a new report becomes
    regressed; an ignored one stays ignored. Resolving, ignoring, prioritising and assigning an issue takes
    a superadmin or the application that owns the service, and is recorded in the audit trail.</p>

  <h2 class="doc-h2">Getting alerted</h2>
  <p>Thresholds are on the Thresholds page. A crossed threshold is published as an ordinary amebo event
    against <code>{{ alert_breached }}</code>, with <code>{{ alert_recovered }}</code> for the all-clear, so
    you subscribe to it exactly the way you subscribe to anything else and get the same signed envelope,
    the same exponential backoff, the same dead-lettering and the same replay. An alert that never reached
    your pager is a row you can find, not a silence.</p>
</div>
{% endblock %}