schemacast.bench.score

Score a cast episode against its expected graph.

No model judges. Entities match by identity: the primary key of a natural type, the digest of a hash type, or any complete secondary identity. A type with no identity of its own (assigned, blank) matches by equal property values or by a quote overlapping the expected one. Edges match by relation and matched endpoints; an edge between the right endpoints the wrong way round counts as reversed. Property values compare after coercion by their field, strings ignoring case and runs of whitespace. An expected edge may end at a known entity by its ref (k1…); it matches by ref, and known entities are not counted as entities.

Every figure is a pair of integer counts, so cases pool by summing and a rate is read off the pooled counts.

Attributes

NOT_CONFORMANCE = frozenset({'quote_not_found', 'value_not_in_quote'}) | CONSEQUENTIAL module-attribute

Classes

Counts

Bases: BaseModel

Additive counts for one episode, a case, a family or a run.

Source code in schemacast/bench/score.py

class Counts(BaseModel):
    """Additive counts for one episode, a case, a family or a run."""

    episodes: int = 0
    failed: int = 0
    requests: int = 0
    repairs: int = 0
    # The first reply, before any repair, and the attempt kept.
    emitted_first: int = 0
    nonconforming_first: int = 0
    emitted_final: int = 0
    nonconforming_final: int = 0
    ungrounded_first: int = 0
    vertices_first: int = 0
    unknown_type_first: int = 0
    edges_first: int = 0
    undeclared_edge_first: int = 0
    properties_first: int = 0
    undeclared_property_first: int = 0
    unfaithful_value_first: int = 0
    direction_fixed: int = 0
    # Against the expected graph.
    entities_expected: int = 0
    entities_produced: int = 0
    entities_matched: int = 0
    mention_agree: int = 0
    edges_expected: int = 0
    edges_produced: int = 0
    edges_matched: int = 0
    edges_reversed: int = 0
    props_expected: int = 0
    props_produced: int = 0
    props_matched: int = 0
    spans_checked: int = 0
    spans_hit: int = 0
    forbidden_expected: int = 0
    forbidden_written: int = 0
    # Selection, read apart from extraction.
    sliced: int = 0
    gold_types: int = 0
    gold_types_seeded: int = 0
    gold_types_in_slice: int = 0
    gold_edges: int = 0
    gold_edges_in_slice: int = 0
    # A second cast with a fresh model call.
    recast_vertices: int = 0
    recast_added: int = 0
    recast_minted: int = 0
    recast_duplicated: int = 0
    # Pick, for the staged mentions a case gives candidates for.
    picks_expected: int = 0
    picks_offered: int = 0
    picks_correct: int = 0
    picks_linked: int = 0
    picks_wrong: int = 0
    picks_rejected: int = 0
    # Tokens, billed or replayed from the cache, and billed only.
    input_tokens: int = 0
    output_tokens: int = 0
    billed_input_tokens: int = 0
    billed_output_tokens: int = 0

    def __add__(self, other: Counts) -> Counts:
        return Counts(
            **{
                name: getattr(self, name) + getattr(other, name)
                for name in Counts.model_fields
            }
        )

Attributes

billed_input_tokens = 0 class-attribute instance-attribute

billed_output_tokens = 0 class-attribute instance-attribute

direction_fixed = 0 class-attribute instance-attribute

edges_expected = 0 class-attribute instance-attribute

edges_first = 0 class-attribute instance-attribute

edges_matched = 0 class-attribute instance-attribute

edges_produced = 0 class-attribute instance-attribute

edges_reversed = 0 class-attribute instance-attribute

emitted_final = 0 class-attribute instance-attribute

emitted_first = 0 class-attribute instance-attribute

entities_expected = 0 class-attribute instance-attribute

entities_matched = 0 class-attribute instance-attribute

entities_produced = 0 class-attribute instance-attribute

episodes = 0 class-attribute instance-attribute

failed = 0 class-attribute instance-attribute

forbidden_expected = 0 class-attribute instance-attribute

forbidden_written = 0 class-attribute instance-attribute

gold_edges = 0 class-attribute instance-attribute

gold_edges_in_slice = 0 class-attribute instance-attribute

gold_types = 0 class-attribute instance-attribute

gold_types_in_slice = 0 class-attribute instance-attribute

gold_types_seeded = 0 class-attribute instance-attribute

input_tokens = 0 class-attribute instance-attribute

mention_agree = 0 class-attribute instance-attribute

nonconforming_final = 0 class-attribute instance-attribute

nonconforming_first = 0 class-attribute instance-attribute

output_tokens = 0 class-attribute instance-attribute

picks_correct = 0 class-attribute instance-attribute

picks_expected = 0 class-attribute instance-attribute

picks_linked = 0 class-attribute instance-attribute

picks_offered = 0 class-attribute instance-attribute

picks_rejected = 0 class-attribute instance-attribute

picks_wrong = 0 class-attribute instance-attribute

properties_first = 0 class-attribute instance-attribute

props_expected = 0 class-attribute instance-attribute

props_matched = 0 class-attribute instance-attribute

props_produced = 0 class-attribute instance-attribute

recast_added = 0 class-attribute instance-attribute

recast_duplicated = 0 class-attribute instance-attribute

recast_minted = 0 class-attribute instance-attribute

recast_vertices = 0 class-attribute instance-attribute

repairs = 0 class-attribute instance-attribute

requests = 0 class-attribute instance-attribute

sliced = 0 class-attribute instance-attribute

spans_checked = 0 class-attribute instance-attribute

spans_hit = 0 class-attribute instance-attribute

undeclared_edge_first = 0 class-attribute instance-attribute

undeclared_property_first = 0 class-attribute instance-attribute

unfaithful_value_first = 0 class-attribute instance-attribute

ungrounded_first = 0 class-attribute instance-attribute

unknown_type_first = 0 class-attribute instance-attribute

vertices_first = 0 class-attribute instance-attribute

Methods:

__add__(other)

Source code in schemacast/bench/score.py

def __add__(self, other: Counts) -> Counts:
    return Counts(
        **{
            name: getattr(self, name) + getattr(other, name)
            for name in Counts.model_fields
        }
    )

Misses

Bases: BaseModel

What one episode's cast and its expected graph do not share, by quote.

Expected items no produced item matched, and produced items that matched no expected or allowed item (a forbidden match is listed and marked).

Source code in schemacast/bench/score.py

class Misses(BaseModel):
    """What one episode's cast and its expected graph do not share, by quote.

    Expected items no produced item matched, and produced items that matched
    no expected or allowed item (a forbidden match is listed and marked).
    """

    expected_entities: list[str] = []
    expected_edges: list[str] = []
    produced_entities: list[str] = []
    produced_edges: list[str] = []

    @property
    def empty(self) -> bool:
        return not (
            self.expected_entities
            or self.expected_edges
            or self.produced_entities
            or self.produced_edges
        )

Attributes

empty property

expected_edges = [] class-attribute instance-attribute

expected_entities = [] class-attribute instance-attribute

produced_edges = [] class-attribute instance-attribute

produced_entities = [] class-attribute instance-attribute

Functions:

gold_vertices(graph, schema, text, context=None)

Expected vertices with coerced properties, identity keys and spans.

Raises:

Type	Description
ValueError	a type or property the schema does not declare, or a value its field refuses.

Source code in schemacast/bench/score.py

def gold_vertices(
    graph: ExpectedGraph,
    schema: Schema,
    text: str,
    context: dict[str, str] | None = None,
) -> dict[str, _Gold]:
    """Expected vertices with coerced properties, identity keys and spans.

    Raises:
        ValueError: a type or property the schema does not declare, or a value
            its field refuses.
    """
    config = schema.core_schema.vertex_config
    out: dict[str, _Gold] = {}
    for handle, item in graph.vertices.items():
        if item.type not in config.vertex_set:
            raise ValueError(f"{handle}: no vertex type {item.type!r}")
        vertex = config[item.type]
        fields = {f.name: f for f in vertex.properties}
        props = _coerce_props(item.props, fields, handle)
        out[handle] = _Gold(
            type=item.type,
            mention=item.mention,
            keys=identity_keys(vertex, props),
            props=props,
            span=locate(item.quote, text),
            prop_spans=_prop_spans(item.props, text, context),
        )
    return out

identity_keys(vertex, props)

Every key props complete: the primary first, then secondaries.

Source code in schemacast/bench/score.py

def identity_keys(vertex: Vertex, props: dict[str, Any]) -> list[tuple[Any, ...]]:
    """Every key *props* complete: the primary first, then secondaries."""
    contract = key_contract(vertex)
    keys: list[tuple[Any, ...]] = []
    identity = primary_identity(vertex, contract, props)
    if identity is not None:
        keys.append(("id", *(_hashable(v) for v in identity.values())))
    for alternate in key_status(contract, props).alternates:
        keys.append((alternate.name, *(_hashable(props[f]) for f in alternate.fields)))
    return keys

ref_handles(outcome, expected, schema, text, context=None)

Produced mention ref → the expected vertex handle it matched.

Source code in schemacast/bench/score.py

def ref_handles(
    outcome: EpisodeOutcome,
    expected: ExpectedEpisode,
    schema: Schema,
    text: str,
    context: dict[str, str] | None = None,
) -> dict[str, str]:
    """Produced mention ref → the expected vertex handle it matched."""
    if outcome.validation is None:
        return {}
    gold = gold_vertices(expected, schema, text, context)
    entities, ref_index = _entities(outcome.validation, schema)
    pairs = _match(entities, gold, set())
    return {ref: pairs[index] for ref, index in ref_index.items() if index in pairs}

score_episode(outcome, expected, schema, text, usage=None, context=None, misses=None, known=())

Counts for one cast episode; context maps context source ids to text.

misses, when given, is filled with what the cast and the expected graph do not share. known are the episode's known entities, whose refs expected edges may name.

Source code in schemacast/bench/score.py

def score_episode(
    outcome: EpisodeOutcome,
    expected: ExpectedEpisode,
    schema: Schema,
    text: str,
    usage: UsageTracker | None = None,
    context: dict[str, str] | None = None,
    misses: Misses | None = None,
    known: Sequence[KnownEntity] = (),
) -> Counts:
    """Counts for one cast episode; *context* maps context source ids to text.

    *misses*, when given, is filled with what the cast and the expected graph
    do not share. *known* are the episode's known entities, whose refs expected
    edges may name.
    """
    misses = misses if misses is not None else Misses()
    counts = Counts(episodes=1)
    result = outcome.result
    counts.failed += result.status == "failed"
    counts.requests += result.requests
    counts.repairs += result.repairs
    if usage is not None:
        counts.input_tokens = (usage.billed.input_tokens or 0) + (
            usage.replayed.input_tokens or 0
        )
        counts.output_tokens = (usage.billed.output_tokens or 0) + (
            usage.replayed.output_tokens or 0
        )
        counts.billed_input_tokens = usage.billed.input_tokens or 0
        counts.billed_output_tokens = usage.billed.output_tokens or 0

    gold = gold_vertices(expected, schema, text, context)
    banned = gold_vertices(expected.forbidden, schema, text, context)
    allowed = gold_vertices(expected.allowed, schema, text, context)
    counts.entities_expected = len(gold)
    counts.edges_expected = len(expected.edges)
    counts.forbidden_expected = len(banned) + len(expected.forbidden.edges)
    kinds = {
        ref: entity.vertex_type for ref, entity in known_refs(known, schema).items()
    }
    kinds.update({handle: item.type for handle, item in gold.items()})
    _selection(counts, outcome, gold, expected, kinds)
    edge_props = []
    for edge in expected.edges:
        if edge.from_ not in kinds or edge.to not in kinds:
            raise ValueError(f"edge {edge.relation}: unknown handle")
        edge_id = (kinds[edge.from_], kinds[edge.to], edge.relation)
        edge_props.append(_edge_props(schema, edge_id, edge.props))

    def gold_quote(handle: str) -> str:
        item = expected.vertices[handle]
        return _named(item.type, item.quote)

    def missed(pairs_: dict[int, str], matched_edges: set[int]) -> None:
        misses.expected_entities += [
            gold_quote(h) for h in expected.vertices if h not in pairs_.values()
        ]
        misses.expected_edges += [
            f"{_named(edge.relation, edge.quote)} ({edge.from_} -> {edge.to})"
            for i, edge in enumerate(expected.edges)
            if i not in matched_edges
        ]

    validation = outcome.validation
    if outcome.first is not None:
        _reply(counts, outcome.first, first=True)
    if validation is None:
        counts.props_expected += sum(len(g.props) for g in gold.values())
        counts.props_expected += sum(len(p) for p in edge_props)
        missed({}, set())
        return counts
    _reply(counts, validation, first=False)
    counts.direction_fixed = sum(f.kind == "edge_reversed" for f in validation.fixes)

    entities, ref_index = _entities(validation, schema)
    pairs = _match(entities, gold, set())
    banned_pairs = _match(entities, banned, set(pairs))
    allowed_pairs = _match(entities, allowed, set(pairs) | set(banned_pairs))
    counts.entities_produced = len(entities) - len(allowed_pairs)
    counts.entities_matched = len(pairs)
    for index, entity in enumerate(entities):
        if index in pairs or index in allowed_pairs:
            continue
        line = _named(entity.type, entity.quotes[0] if entity.quotes else None)
        misses.produced_entities.append(
            line + (" (forbidden)" if index in banned_pairs else "")
        )
    for index, handle in pairs.items():
        entity, item = entities[index], gold[handle]
        counts.mention_agree += entity.mention == item.mention
        if item.span is not None:
            counts.spans_checked += 1
            counts.spans_hit += any(_overlap(s, item.span) for s in entity.spans)
        _score_props(
            counts, item.props, entity.props, item.prop_spans, entity.prop_spans
        )
    for handle, item in gold.items():
        if handle not in pairs.values():
            counts.props_expected += len(item.props)
    # A forbidden entity counts as written when resolution would write it.
    resolution = _resolution(outcome.graph, schema)
    resolved = {r.ref for r in resolution.resolved if not r.owned}
    written = {ref_index[ref] for ref in resolved if ref in ref_index}
    counts.forbidden_written += sum(index in written for index in banned_pairs)

    handles: dict[int, str] = {**pairs, **banned_pairs, **allowed_pairs}
    known_used = {record.ref for record in validation.known}

    def node(ref: str) -> int | str:
        # A known entity is its own handle.
        return ref if ref in known_used else ref_index[ref]

    def handle(item: int | str) -> str | None:
        return item if isinstance(item, str) else handles.get(item)

    produced: dict[
        tuple[Any, ...], tuple[dict[str, Any], list[Span], dict[str, list[Span]]]
    ] = {}
    edge_quotes: dict[tuple[Any, ...], str | None] = {}
    for valid in validation.relations:
        key = (valid.edge_id[2], node(valid.source), node(valid.target))
        props, spans, prop_spans = produced.setdefault(key, ({}, [], {}))
        edge_quotes.setdefault(key, quote_of(valid.evidence))
        props.update({k: v for k, v in valid.values().items() if k not in props})
        spans.extend(_spans(valid.evidence))
        for name, prop in valid.props.items():
            prop_spans.setdefault(name, []).extend(_spans(prop.evidence))
    by_handles = {
        (relation, handle(src), handle(tgt)): value
        for (relation, src, tgt), value in produced.items()
    }
    allowed_edges = {
        (edge.relation, edge.from_, edge.to) for edge in expected.allowed.edges
    }
    taken = {
        (edge.relation, edge.from_, edge.to) for edge in expected.edges
    } | allowed_edges
    counts.edges_produced = len(produced) - sum(
        (relation, handle(src), handle(tgt)) in allowed_edges
        for relation, src, tgt in produced
    )
    for key in produced:
        relation, src, tgt = key
        if (relation, handle(src), handle(tgt)) not in taken:
            misses.produced_edges.append(_named(relation, edge_quotes[key]))
    matched_edges: set[int] = set()
    for number, (edge, gold_props) in enumerate(
        zip(expected.edges, edge_props, strict=True)
    ):
        forward = by_handles.get((edge.relation, edge.from_, edge.to))
        if forward is None:
            counts.props_expected += len(gold_props)
            counts.edges_reversed += (edge.relation, edge.to, edge.from_) in by_handles
            continue
        matched_edges.add(number)
        props, spans, prop_spans = forward
        counts.edges_matched += 1
        span = locate(edge.quote, text)
        if span is not None:
            counts.spans_checked += 1
            counts.spans_hit += any(_overlap(s, span) for s in spans)
        _score_props(
            counts,
            gold_props,
            props,
            _prop_spans(edge.props, text, context),
            prop_spans,
        )
    counts.forbidden_written += sum(
        (edge.relation, edge.from_, edge.to) in by_handles
        for edge in expected.forbidden.edges
    )
    missed(pairs, matched_edges)
    return counts

score_picks(picks, refs, spec)

Pick counts: a pick is correct when what it links is the expected entity, or nothing when none fits. A mention the cast did not stage counts as missed.

Source code in schemacast/bench/score.py

def score_picks(picks: list[Pick], refs: dict[str, str], spec: ExpectedPicks) -> Counts:
    """Pick counts: a pick is correct when what it links is the expected entity,
    or nothing when none fits. A mention the cast did not stage counts as missed."""
    counts = Counts(picks_expected=len(spec.mentions))
    for pick in picks:
        handle = refs.get(pick.ref)
        if handle not in spec.mentions:
            continue
        wanted = spec.mentions[handle].pick
        counts.picks_offered += pick.outcome != "no_candidates"
        counts.picks_rejected += pick.outcome == "rejected"
        linked = pick.key if pick.outcome == "picked" else None
        counts.picks_linked += linked is not None
        same = (linked is None and wanted is None) or (
            linked is not None
            and wanted is not None
            and {k: str(v) for k, v in linked.items()}
            == {k: str(v) for k, v in wanted.items()}
        )
        counts.picks_correct += same
        counts.picks_wrong += linked is not None and not same
    return counts

score_recast(first, second, schema)

What a second cast of the same episode would add to a store that upserts.

A natural or hash identity the first cast lacked is an added vertex; an assigned or blank one is a duplicate, since nothing ties it to the first.

Source code in schemacast/bench/score.py

def score_recast(
    first: EpisodeOutcome, second: EpisodeOutcome, schema: Schema
) -> Counts:
    """What a second cast of the same episode would add to a store that upserts.

    A natural or hash identity the first cast lacked is an *added* vertex; an
    assigned or blank one is a *duplicate*, since nothing ties it to the first.
    """
    counts = Counts()
    before = _identities(first.graph, schema)
    after = _identities(second.graph, schema)
    config = schema.core_schema.vertex_config
    for name, keys in after.items():
        new = len(keys - before.get(name, set()))
        counts.recast_vertices += len(keys)
        if config[name].identity_mode in ("natural", "hash"):
            counts.recast_added += new
        else:
            counts.recast_minted += len(keys)
            counts.recast_duplicated += new
    return counts

total(counts)

The sum of counts.

Source code in schemacast/bench/score.py

def total(counts: list[Counts]) -> Counts:
    """The sum of *counts*."""
    out = Counts()
    for item in counts:
        out = out + item
    return out