Scores et retours

// A machine evaluation of the run. `min` and `max` are required: 0.86 on an
// undeclared scale cannot be compared to anything.
mcpd.score({
  traceId,
  scorerId: 'answer_relevance',
  value: 0.86,
  min: 0,
  max: 1,
  reason: 'Answered the question and gave the delivery date.',
})

// A human evaluation. Kept separate from a score, and never averaged with
// one, because the two answer different questions.
mcpd.feedback({
  traceId,
  kind: 'thumbs',
  value: 'up',
  comment: 'Exactly what the customer asked for.',
  authorKind: 'end_user',
})

// Anything you already count can travel with the trace.
mcpd.metric('support.tickets_handled', 1, 'counter', { labels: { queue: 'tier-1' } })

# A machine evaluation. `min` and `max` are required: a bare number with no
# declared scale cannot be compared to anything.
mcpd.score(
    "answer_relevance",
    value=round(random.uniform(0.6, 0.95), 2),
    min=0.0,
    max=1.0,
    trace_id=trace_id,
    reason="named the order and gave a delivery window, without the live lookup",
)

# A human evaluation. Kept separate from a score, because a person's
# judgement and a model's must never be averaged together.
mcpd.feedback(
    FeedbackKind.THUMBS,
    "up",
    trace_id=trace_id,
    comment="fine given the outage",
    author_kind="end_user",
)

// A score is a machine judgement. The scale is not optional: a bare 0.9 with
// no declared range cannot be compared to anything.
mcpd.score(
    Target::trace(&trace_id),
    "answer_relevance",
    0.9,
    Scale::UNIT,
    Some("Names the carrier and the delivery day the customer asked about."),
);

// Feedback is a person's judgement, kept separate from a score because the
// two answer different questions and must never be averaged together.
mcpd.feedback(Target::trace(&trace_id), FeedbackKind::Thumbs, "up", Some("fast and correct"));

// A machine evaluation of the run. min and max are required: 0.86 means
// nothing until you know the scale it sits on.
m.Score(ctx, mcpd.TraceTarget(traceID), "answer_relevance", 0.86, 0, 1,
	mcpd.ScoreReason("Answered the billing question and cited the invoice."),
	mcpd.ScoreVersion("2"),
)

// A person's judgement of the same run. Wire this to your product's
// thumbs-up control and it lands on the exact trace that produced the
// answer.
m.Feedback(ctx, mcpd.TraceTarget(traceID), mcpd.FeedbackKindThumbs, "up",
	mcpd.FeedbackComment("Fixed it first time."),
	mcpd.FeedbackAuthor("u-317", "end_user"),
)

// A machine evaluation. `min` and `max` are required: 7 is excellent out of
// 10 and poor out of 100, and a score with no scale cannot be compared.
mcpd.score(
    traceId = traceId,
    scorerId = "answer_relevance",
    value = 0.86,
    min = 0.0,
    max = 1.0,
    reason = "answered the question asked, without the delivery estimate",
)

// A person's judgement, from the thumbs-up control in your own product.
mcpd.feedback(
    traceId = traceId,
    kind = FeedbackKind.THUMBS,
    value = "up",
    comment = "fast and clear",
    authorId = "customer-4477",
)

// Nothing above touched the network. This is where it goes.
mcpd.flush()

# A machine evaluation. `min` and `max` are required: 7 is excellent out of 10
# and poor out of 100, and a score with no scale cannot be compared.
Mcpd.score(mcpd, trace_id, "answer_relevance", 0.86,
  min: 0,
  max: 1,
  reason: "answered the question asked, without the delivery estimate"
)

# A person's judgement, from the thumbs-up control in your own product.
Mcpd.feedback(mcpd, trace_id, :thumbs, "up",
  comment: "fast and clear",
  author_id: "customer-4477"
)

# Nothing above touched the network. This is where it goes.
:ok = Mcpd.flush(mcpd)

# Attach an evaluation. `min` and `max` are required: a bare number with no
# declared scale cannot be compared to anything.
curl -s -X POST "$MCPD/v1/api/scores" \
  -H 'content-type: application/json' \
  -d "{
    \"score_id\": \"\",
    \"trace_id\": \"$TRACE\",
    \"scorer_id\": \"answer_relevance\",
    \"source\": \"live\",
    \"value\": 0.94,
    \"min\": 0,
    \"max\": 1,
    \"reason\": \"Names the order and the depot.\",
    \"timestamp\": \"$NOW\",
    \"correlation\": { \"project\": \"default\" }
  }"

// Sans traceId : il se rattache à la trace du span actif.
mcpd.feedback({
  kind: 'thumbs',
  value: 'down',
  comment: 'Named the wrong depot.',
  authorKind: 'end_user',
})

curl -s -X POST "$MCPD/v1/api/feedback" \
  -H 'content-type: application/json' \
  -d "{
    \"feedback_id\": \"\",
    \"trace_id\": \"$TRACE\",
    \"kind\": \"thumbs\",
    \"value\": \"down\",
    \"comment\": \"Named the wrong depot.\",
    \"author_kind\": \"end_user\",
    \"timestamp\": \"$NOW\",
    \"correlation\": { \"project\": \"default\" }
  }"

// The same client reads. This is the query that finds bad answers: every run
// that a scorer put below the bar.
const suspect = await mcpd.listTraces({
  hasScore: 'answer_relevance',
  scoreBelow: 0.5,
  limit: 10,
})
console.log(`runs scoring under 0.5: ${suspect.traces.length}`)

const recent = await mcpd.listTraces({ limit: 5, sort: 'start_time', order: 'desc' })
for (const trace of recent.traces) {
  const seconds = ((trace.durationNs ?? 0) / 1e9).toFixed(2)
  console.log(
    `${trace.traceId.slice(0, 8)}  ${trace.name.padEnd(16)} ${trace.status.padEnd(6)} ` +
      `${seconds}s  ${trace.spanCount} spans  $${(trace.cost?.totalCost ?? 0).toFixed(6)}`,
  )
}

// One run, with every span in it, ordered so a waterfall renders directly.
const detail = await mcpd.getTrace(traceId, { include: ['scores', 'feedback'] })
for (const span of detail.spans) {
  console.log(`  ${span.kind.padEnd(18)} ${span.name}`)
}

# Nothing has been sent yet: recording appends to an in-memory queue and a
# background worker does the I/O. Flush before you read your own writes.
mcpd.flush()

detail = mcpd.get_trace(trace_id, include=["logs", "scores", "feedback"])
print(f"\n{detail.trace.name}  {detail.trace.duration_ns / 1e6:.0f} ms  "
      f"{detail.trace.span_count} spans  {detail.trace.error_count} errors")
for span in detail.spans:
    cost = f"  ${span.cost.total_cost:.6f}" if span.cost else ""
    print(f"  {span.kind.value:<18} {span.name:<14} {span.status.value}{cost}")

summary = mcpd.analytics_summary("-1h", "-0s")
print(f"\nlast hour  traces={summary.trace_count}  errors={summary.error_count}  "
      f"p95={summary.latency.p95:.0f}ms  cost=${summary.total_cost:.4f}")

// The same client reads. This is the trace you just wrote.
match mcpd.get_trace(&trace_id, Include::all()).await {
    Ok(detail) => {
        println!("{}: {} spans", detail.trace.name, detail.spans.len());
        for span in &detail.spans {
            println!(
                "  {:<22} {:>8} {}",
                span.name,
                span.duration_ms().map_or("open".to_owned(), |ms| format!("{ms:.1}ms")),
                span.status,
            );
        }
        if let Some(cost) = detail.trace.cost.as_ref().and_then(|c| c.total_cost) {
            println!("  cost: {cost:.6} USD");
        }
        for score in detail.scores.iter().flatten() {
            println!("  score {}: {}", score.scorer_id, score.value);
        }
    }
    Err(e) => eprintln!("could not read the trace back: {e}"),
}

// Recording is asynchronous, so wait for the batch to land before reading
// it back. Outside an example you would never need this.
if err := m.Flush(ctx); err != nil {
	log.Printf("flush: %v", err)
}

detail, err := m.GetTrace(ctx, traceID, "scores", "feedback")
if err != nil {
	log.Fatalf("read the run back: %v", err)
}
fmt.Printf("%s: %s in %s, %d spans\n",
	detail.Trace.TraceID, detail.Trace.Status, duration(detail.Trace.DurationNs), len(detail.Spans))
for _, span := range detail.Spans {
	fmt.Printf("  %-18s %-18s %s\n", span.Kind, span.Name, span.Status)
}

summary, err := m.AnalyticsSummary(ctx, time.Now().Add(-time.Hour), time.Now(), "model")
if err != nil {
	log.Fatalf("read the summary: %v", err)
}
fmt.Printf("last hour: %d runs, %.1f%% failed, %.4f %s\n",
	summary.TraceCount, summary.ErrorRate*100, summary.TotalCost, summary.Currency)

val detail = mcpd.getTrace(traceId, include = listOf("scores", "feedback"))
println("${detail.trace.name}: ${detail.spans.size} spans, ${detail.trace.status}")
for (span in detail.spans) {
    val ms = (span.durationNs ?: 0) / 1_000_000
    println("  ${span.kind} ${span.name} ${ms}ms ${span.error?.message ?: ""}")
}

val summary = mcpd.analyticsSummary(
    from = Clock.System.now() - 1.hours,
    to = Clock.System.now(),
)
println(
    "last hour: ${summary.traceCount} runs, " +
        "${summary.errorCount} failed, ${summary.totalCost} USD",
)

{:ok, detail} = Mcpd.get_trace(mcpd, trace_id, include: ["scores", "feedback"])

IO.puts("#{detail.trace.name}: #{length(detail.spans)} spans")

for span <- detail.spans do
  ms = div(span.duration_ns || 0, 1_000_000)
  error = (span.error && span.error.message) || ""
  IO.puts("  #{span.kind} #{span.name} #{ms}ms #{error}")
end

# Read it back. Any filter in the API works here.
curl -s "$MCPD/v1/api/traces?project=default&limit=10"

# The whole run, with its logs, scores and feedback in one request.
curl -s "$MCPD/v1/api/traces/$TRACE?include=all"

# Find the run by something the model actually said.
curl -s --get "$MCPD/v1/api/traces" \
  --data-urlencode 'project=default' \
  --data-urlencode 'q=Leeds depot'

# Find the runs a scorer marked down. This is the query that finds bad answers.
curl -s --get "$MCPD/v1/api/traces" \
  --data-urlencode 'project=default' \
  --data-urlencode 'has_score=answer_relevance' \
  --data-urlencode 'score_below=0.5'

# Where the money went, over the last week, by model.
curl -s --get "$MCPD/v1/api/analytics/cost" \
  --data-urlencode 'project=default' \
  --data-urlencode 'from=-7d' \
  --data-urlencode 'group_by=model'