lpgcast.bench.score

Score a cast episode against its expected graph.

No model judges. Entities match by identity: the primary key of a natural type, the digest of a hash type, or any complete secondary identity. A type with no identity of its own (assigned, blank) matches by equal property values or by a quote overlapping the expected one. Edges match by relation and matched endpoints; an edge between the right endpoints the wrong way round counts as reversed. Property values compare after coercion by their field, strings ignoring case and runs of whitespace.

Every figure is a pair of integer counts, so cases pool by summing and a rate is read off the pooled counts.

Attributes

NOT_CONFORMANCE = frozenset({'quote_not_found', 'value_not_in_quote'}) | CONSEQUENTIAL module-attribute

Classes

Counts

Bases: BaseModel

Additive counts for one episode, a case, a family or a run.

Source code in lpgcast/bench/score.py

class Counts(BaseModel):
    """Additive counts for one episode, a case, a family or a run."""

    episodes: int = 0
    failed: int = 0
    requests: int = 0
    repairs: int = 0
    # The first reply, before any repair, and the attempt kept.
    emitted_first: int = 0
    nonconforming_first: int = 0
    emitted_final: int = 0
    nonconforming_final: int = 0
    ungrounded_first: int = 0
    vertices_first: int = 0
    unknown_type_first: int = 0
    edges_first: int = 0
    undeclared_edge_first: int = 0
    properties_first: int = 0
    undeclared_property_first: int = 0
    unfaithful_value_first: int = 0
    direction_fixed: int = 0
    # Against the expected graph.
    entities_expected: int = 0
    entities_produced: int = 0
    entities_matched: int = 0
    mention_agree: int = 0
    edges_expected: int = 0
    edges_produced: int = 0
    edges_matched: int = 0
    edges_reversed: int = 0
    props_expected: int = 0
    props_produced: int = 0
    props_matched: int = 0
    spans_checked: int = 0
    spans_hit: int = 0
    forbidden_expected: int = 0
    forbidden_written: int = 0
    # Selection, read apart from extraction.
    sliced: int = 0
    gold_types: int = 0
    gold_types_seeded: int = 0
    gold_types_in_slice: int = 0
    gold_edges: int = 0
    gold_edges_in_slice: int = 0
    # A second cast with a fresh model call.
    recast_vertices: int = 0
    recast_added: int = 0
    recast_minted: int = 0
    recast_duplicated: int = 0
    # Tokens, billed or replayed from the cache, and billed only.
    input_tokens: int = 0
    output_tokens: int = 0
    billed_input_tokens: int = 0
    billed_output_tokens: int = 0

    def __add__(self, other: Counts) -> Counts:
        return Counts(
            **{
                name: getattr(self, name) + getattr(other, name)
                for name in Counts.model_fields
            }
        )

Attributes

billed_input_tokens = 0 class-attribute instance-attribute

billed_output_tokens = 0 class-attribute instance-attribute

direction_fixed = 0 class-attribute instance-attribute

edges_expected = 0 class-attribute instance-attribute

edges_first = 0 class-attribute instance-attribute

edges_matched = 0 class-attribute instance-attribute

edges_produced = 0 class-attribute instance-attribute

edges_reversed = 0 class-attribute instance-attribute

emitted_final = 0 class-attribute instance-attribute

emitted_first = 0 class-attribute instance-attribute

entities_expected = 0 class-attribute instance-attribute

entities_matched = 0 class-attribute instance-attribute

entities_produced = 0 class-attribute instance-attribute

episodes = 0 class-attribute instance-attribute

failed = 0 class-attribute instance-attribute

forbidden_expected = 0 class-attribute instance-attribute

forbidden_written = 0 class-attribute instance-attribute

gold_edges = 0 class-attribute instance-attribute

gold_edges_in_slice = 0 class-attribute instance-attribute

gold_types = 0 class-attribute instance-attribute

gold_types_in_slice = 0 class-attribute instance-attribute

gold_types_seeded = 0 class-attribute instance-attribute

input_tokens = 0 class-attribute instance-attribute

mention_agree = 0 class-attribute instance-attribute

nonconforming_final = 0 class-attribute instance-attribute

nonconforming_first = 0 class-attribute instance-attribute

output_tokens = 0 class-attribute instance-attribute

properties_first = 0 class-attribute instance-attribute

props_expected = 0 class-attribute instance-attribute

props_matched = 0 class-attribute instance-attribute

props_produced = 0 class-attribute instance-attribute

recast_added = 0 class-attribute instance-attribute

recast_duplicated = 0 class-attribute instance-attribute

recast_minted = 0 class-attribute instance-attribute

recast_vertices = 0 class-attribute instance-attribute

repairs = 0 class-attribute instance-attribute

requests = 0 class-attribute instance-attribute

sliced = 0 class-attribute instance-attribute

spans_checked = 0 class-attribute instance-attribute

spans_hit = 0 class-attribute instance-attribute

undeclared_edge_first = 0 class-attribute instance-attribute

undeclared_property_first = 0 class-attribute instance-attribute

unfaithful_value_first = 0 class-attribute instance-attribute

ungrounded_first = 0 class-attribute instance-attribute

unknown_type_first = 0 class-attribute instance-attribute

vertices_first = 0 class-attribute instance-attribute

Methods:

__add__(other)

Source code in lpgcast/bench/score.py

def __add__(self, other: Counts) -> Counts:
    return Counts(
        **{
            name: getattr(self, name) + getattr(other, name)
            for name in Counts.model_fields
        }
    )

Functions:

gold_vertices(graph, schema, text, context=None)

Expected vertices with coerced properties, identity keys and spans.

Raises:

Type	Description
ValueError	a type or property the schema does not declare, or a value its field refuses.

Source code in lpgcast/bench/score.py

def gold_vertices(
    graph: ExpectedGraph,
    schema: Schema,
    text: str,
    context: dict[str, str] | None = None,
) -> dict[str, _Gold]:
    """Expected vertices with coerced properties, identity keys and spans.

    Raises:
        ValueError: a type or property the schema does not declare, or a value
            its field refuses.
    """
    config = schema.core_schema.vertex_config
    out: dict[str, _Gold] = {}
    for handle, item in graph.vertices.items():
        if item.type not in config.vertex_set:
            raise ValueError(f"{handle}: no vertex type {item.type!r}")
        vertex = config[item.type]
        fields = {f.name: f for f in vertex.properties}
        props = _coerce_props(item.props, fields, handle)
        out[handle] = _Gold(
            type=item.type,
            mention=item.mention,
            keys=identity_keys(vertex, props),
            props=props,
            span=locate(item.quote, text),
            prop_spans=_prop_spans(item.props, text, context),
        )
    return out

identity_keys(vertex, props)

Every identity props complete: the primary first, then secondaries.

Source code in lpgcast/bench/score.py

def identity_keys(vertex: Vertex, props: dict[str, Any]) -> list[tuple[Any, ...]]:
    """Every identity *props* complete: the primary first, then secondaries."""
    keys: list[tuple[Any, ...]] = []
    mode = vertex.identity_mode
    if (
        mode == "natural"
        and vertex.identity
        and all(f in props for f in vertex.identity)
    ):
        keys.append(("id", *(_hashable(props[f]) for f in vertex.identity)))
    elif mode == "hash":
        digest = compute_vertex_identity(props, vertex)
        if digest is not None:
            keys.append(("id", digest))
    for secondary in vertex.secondary_identities:
        if all(f in props for f in secondary.fields):
            keys.append(
                (
                    secondary.name or "secondary",
                    *(_hashable(props[f]) for f in secondary.fields),
                )
            )
    return keys

score_episode(outcome, expected, schema, text, usage=None, context=None)

Counts for one cast episode; context maps context source ids to text.

Source code in lpgcast/bench/score.py

def score_episode(
    outcome: EpisodeOutcome,
    expected: ExpectedEpisode,
    schema: Schema,
    text: str,
    usage: UsageTracker | None = None,
    context: dict[str, str] | None = None,
) -> Counts:
    """Counts for one cast episode; *context* maps context source ids to text."""
    counts = Counts(episodes=1)
    result = outcome.result
    counts.failed += result.status == "failed"
    counts.requests += result.requests
    counts.repairs += result.repairs
    if usage is not None:
        counts.input_tokens = (usage.billed.input_tokens or 0) + (
            usage.replayed.input_tokens or 0
        )
        counts.output_tokens = (usage.billed.output_tokens or 0) + (
            usage.replayed.output_tokens or 0
        )
        counts.billed_input_tokens = usage.billed.input_tokens or 0
        counts.billed_output_tokens = usage.billed.output_tokens or 0

    gold = gold_vertices(expected, schema, text, context)
    banned = gold_vertices(expected.forbidden, schema, text, context)
    counts.entities_expected = len(gold)
    counts.edges_expected = len(expected.edges)
    counts.forbidden_expected = len(banned) + len(expected.forbidden.edges)
    _selection(counts, outcome, gold, expected)
    edge_props = []
    for edge in expected.edges:
        if edge.from_ not in gold or edge.to not in gold:
            raise ValueError(f"edge {edge.relation}: unknown handle")
        edge_id = (gold[edge.from_].type, gold[edge.to].type, edge.relation)
        edge_props.append(_edge_props(schema, edge_id, edge.props))

    validation = outcome.validation
    if outcome.first is not None:
        _reply(counts, outcome.first, first=True)
    if validation is None:
        counts.props_expected += sum(len(g.props) for g in gold.values())
        counts.props_expected += sum(len(p) for p in edge_props)
        return counts
    _reply(counts, validation, first=False)
    counts.direction_fixed = sum(f.kind == "edge_reversed" for f in validation.fixes)

    entities, ref_index = _entities(validation, schema)
    pairs = _match(entities, gold, set())
    banned_pairs = _match(entities, banned, set(pairs))
    counts.entities_produced = len(entities)
    counts.entities_matched = len(pairs)
    for index, handle in pairs.items():
        entity, item = entities[index], gold[handle]
        counts.mention_agree += entity.mention == item.mention
        if item.span is not None:
            counts.spans_checked += 1
            counts.spans_hit += any(_overlap(s, item.span) for s in entity.spans)
        _score_props(
            counts, item.props, entity.props, item.prop_spans, entity.prop_spans
        )
    for handle, item in gold.items():
        if handle not in pairs.values():
            counts.props_expected += len(item.props)
    counts.forbidden_written += sum(
        not entities[index].mention for index in banned_pairs
    )

    handles = {**pairs, **banned_pairs}
    produced: dict[
        tuple[Any, ...], tuple[dict[str, Any], list[Span], dict[str, list[Span]]]
    ] = {}
    for valid in validation.edges:
        key = (
            valid.edge_id[2],
            ref_index[valid.source_ref],
            ref_index[valid.target_ref],
        )
        props, spans, prop_spans = produced.setdefault(key, ({}, [], {}))
        props.update({k: v for k, v in valid.props.items() if k not in props})
        spans.append(valid.span)
        for name, span in valid.prop_spans.items():
            prop_spans.setdefault(name, []).append(span)
    counts.edges_produced = len(produced)
    by_handles = {
        (relation, handles.get(src), handles.get(tgt)): value
        for (relation, src, tgt), value in produced.items()
    }
    for edge, gold_props in zip(expected.edges, edge_props, strict=True):
        forward = by_handles.get((edge.relation, edge.from_, edge.to))
        if forward is None:
            counts.props_expected += len(gold_props)
            counts.edges_reversed += (edge.relation, edge.to, edge.from_) in by_handles
            continue
        props, spans, prop_spans = forward
        counts.edges_matched += 1
        span = locate(edge.quote, text)
        if span is not None:
            counts.spans_checked += 1
            counts.spans_hit += any(_overlap(s, span) for s in spans)
        _score_props(
            counts,
            gold_props,
            props,
            _prop_spans(edge.props, text, context),
            prop_spans,
        )
    counts.forbidden_written += sum(
        (edge.relation, edge.from_, edge.to) in by_handles
        for edge in expected.forbidden.edges
    )
    return counts

score_recast(first, second, schema)

What a second cast of the same episode would add to a store that upserts.

A natural or hash identity the first cast lacked is an added vertex; an assigned or blank one is a duplicate, since nothing ties it to the first.

Source code in lpgcast/bench/score.py

def score_recast(
    first: EpisodeOutcome, second: EpisodeOutcome, schema: Schema
) -> Counts:
    """What a second cast of the same episode would add to a store that upserts.

    A natural or hash identity the first cast lacked is an *added* vertex; an
    assigned or blank one is a *duplicate*, since nothing ties it to the first.
    """
    counts = Counts()
    before = _identities(first.resolution, schema)
    after = _identities(second.resolution, schema)
    config = schema.core_schema.vertex_config
    for name, keys in after.items():
        new = len(keys - before.get(name, set()))
        counts.recast_vertices += len(keys)
        if config[name].identity_mode in ("natural", "hash"):
            counts.recast_added += new
        else:
            counts.recast_minted += len(keys)
            counts.recast_duplicated += new
    return counts

total(counts)

The sum of counts.

Source code in lpgcast/bench/score.py

def total(counts: list[Counts]) -> Counts:
    """The sum of *counts*."""
    out = Counts()
    for item in counts:
        out = out + item
    return out