Skip to content

lpgcast.bench.oracle

The oracle: a model that replies with the expected graph.

It goes through the whole pipeline — selection, validation, repair, resolution — so its row is the ceiling the rest of the pipeline allows: a type selection leaves out of the slice costs the oracle too. It never follows an injected instruction and costs nothing.

Attributes

ORACLE = 'oracle' module-attribute

Classes

Functions:

oracle_client(expected)

A client answering every request about an episode with its expected graph.

Uncached: the oracle is free and its replies are not model output.

Source code in lpgcast/bench/oracle.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def oracle_client(expected: dict[str, ExpectedEpisode]) -> LLMClient:
    """A client answering every request about an episode with its expected graph.

    Uncached: the oracle is free and its replies are not model output.
    """
    replies = {
        source_id: json.dumps(oracle_reply(item))
        for source_id, item in expected.items()
    }

    def handle(request: httpx.Request) -> httpx.Response:
        body = json.loads(request.content)
        for message in body["messages"]:
            found = _EPISODE_ID.search(message["content"])
            if found and found.group(1) in replies:
                return httpx.Response(200, json=_completion(replies[found.group(1)]))
        return httpx.Response(
            400, json={"error": {"message": "unknown episode", "code": None}}
        )

    return LLMClient(
        LLMSettings(model=ORACLE, api_key=ORACLE, base_url="http://oracle.invalid/v1"),
        LLMCache(CacheSettings(enabled=False)),
        http_client=httpx.AsyncClient(transport=httpx.MockTransport(handle)),
    )

oracle_reply(expected)

The flat reply a perfect model would give.

Source code in lpgcast/bench/oracle.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
def oracle_reply(expected: ExpectedEpisode) -> dict[str, Any]:
    """The flat reply a perfect model would give."""
    return {
        "vertices": [
            {
                "ref": handle,
                "type": item.type,
                "quote": item.quote,
                "mention": item.mention,
                "props": _props(item.props),
            }
            for handle, item in expected.vertices.items()
        ],
        "edges": [
            {
                "relation": edge.relation,
                "from": edge.from_,
                "to": edge.to,
                "quote": edge.quote,
                "props": _props(edge.props),
            }
            for edge in expected.edges
        ],
        "unmapped": [],
    }