Scorers API

import {
  createScorer,
  llmJudge,
  faithfulnessScorer,
  schemaAdherenceScorer,
  latencyScorer,
  aggregateScores,
  runScorersBatch,
} from "smithers-orchestrator";
import type { Scorer, ScoreResult, ScorersMap } from "smithers-orchestrator";

type ScorerBinding = { scorer: Scorer; sampling?: SamplingConfig };
type ScorersMap    = Record<string, ScorerBinding>;
type SamplingConfig =
  | { type: "all" }                  // run every time (default)
  | { type: "ratio"; rate: number }  // run with probability `rate`
  | { type: "none" };                // never run

<Task
  id="analyze"
  output={outputs.analysis}
  agent={analyst}
  scorers={{
    schema: { scorer: schemaAdherenceScorer() },
    latency: { scorer: latencyScorer({ targetMs: 5000, maxMs: 20000 }) },
    safety: { scorer: toxicityScorer(judge), sampling: { type: "ratio", rate: 0.1 } },
  }}
>
  Analyze the report.
</Task>

function createScorer(config: CreateScorerConfig): Scorer;

const wordCount = createScorer({
  id: "word-count",
  name: "Word Count",
  description: "Scores toward 1.0 as output approaches 200 words",
  score: async ({ output }) => ({
    score: Math.min(String(output).split(/\s+/).length / 200, 1),
  }),
});

function llmJudge(config: LlmJudgeConfig): Scorer;

const tone = llmJudge({
  id: "tone",
  name: "Professional Tone",
  description: "Evaluates professional tone",
  judge,
  instructions: "You evaluate text for professional tone.",
  promptTemplate: ({ output }) =>
    `Rate the professionalism (0-1 JSON):\n\n${String(output)}`,
});

const grounded = faithfulnessScorer(judge);
const onSchema = schemaAdherenceScorer();
const fast = latencyScorer({ targetMs: 5000, maxMs: 20000 });

import {
  EVAL_CASE_STATUSES,
  EVAL_PASS_THRESHOLD,
  evalAssertionScorer,
  evalCaseRunId,
  evaluateEvalCase,
  evaluateEvalCaseAsync,
  formatEvalError,
  isPlainObject,
  jsonContains,
  jsonEquals,
  normalizeEvalJudge,
  normalizeExpected,
  parseEvalDataset,
  slugifyEvalToken,
} from "smithers-orchestrator/scorers";

type EvalJudge = {
  instructions: string;
  threshold?: number; // default EVAL_PASS_THRESHOLD (0.8)
};

const grade = await evaluateEvalCaseAsync(
  {
    input,
    output,
    status: "finished",
    expected: { outputContains: { safe: true } },
    judge: {
      instructions: "The answer cites the supplied evidence.",
      threshold: 0.9,
    },
  },
  async ({ judge, input, output }) => runProjectJudge({ judge, input, output }),
);

function runScorersAsync(
  scorers: ScorersMap,
  ctx: ScorerContext,
  adapter: SmithersDb | null,
  eventBus?: EventBus | null,
): void;

function runScorersBatch(
  scorers: ScorersMap,
  ctx: ScorerContext,
  adapter: SmithersDb | null,
  eventBus?: EventBus | null,
): Promise<Record<string, ScoreResult | null>>;

const results = await runScorersBatch(
  { quality: { scorer: tone } },
  {
    runId: "RUN_ID",
    nodeId: "NODE_ID",
    iteration: 0,
    attempt: 1,
    input: "Summarize the article.",
    output: "...",
  },
  null,
);
// results.quality?.score

function aggregateScores(
  adapter: SmithersDb,
  opts?: AggregateOptions,
): Promise<AggregateScore[]>;

const stats = await aggregateScores(adapter, { runId: "RUN_ID" });

bunx smithers-orchestrator scores RUN_ID