Skip to content

ontocast.tool.agg.triple_evaluator

Evaluate predicted vs ground-truth RDF graphs given entity alignments.

Classes

TripleSetEvaluator

Compute PR/F1 metrics for aligned predicted and ground-truth graphs.

Source code in ontocast/tool/agg/triple_evaluator.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
class TripleSetEvaluator:
    """Compute PR/F1 metrics for aligned predicted and ground-truth graphs."""

    def evaluate(
        self,
        predicted_graph: RDFGraph,
        gt_graph: RDFGraph,
        entity_matches: list[EntityMatch],
    ) -> MatchMetrics:
        predicted_to_gt = {
            as_uri_ref(matched.predicted_entity): as_uri_ref(matched.gt_entity)
            for matched in entity_matches
        }

        raw_predicted = project_triples(predicted_graph, predicted_to_gt)
        raw_ground_truth = set(gt_graph)

        predicted = prepare_metric_triples(raw_predicted)
        ground_truth = prepare_metric_triples(raw_ground_truth)

        true_positives = len(predicted & ground_truth)
        false_positives = len(predicted - ground_truth)
        false_negatives = len(ground_truth - predicted)
        precision, recall, f1 = compute_prf(
            true_positives,
            len(predicted),
            len(ground_truth),
        )

        ontology_entities = collect_ontology_entities(predicted | ground_truth)
        predicted_facts = prepare_fact_triples(predicted, ontology_entities)
        ground_truth_facts = prepare_fact_triples(ground_truth, ontology_entities)
        fact_true_positives = len(predicted_facts & ground_truth_facts)
        fact_false_positives = len(predicted_facts - ground_truth_facts)
        fact_false_negatives = len(ground_truth_facts - predicted_facts)
        fact_precision, fact_recall, fact_f1 = compute_prf(
            fact_true_positives,
            len(predicted_facts),
            len(ground_truth_facts),
        )

        predicted_entities = set(extract_instance_entities(predicted_graph))
        gt_entities = set(extract_instance_entities(gt_graph))
        instance_matches = [
            matched
            for matched in entity_matches
            if as_uri_ref(matched.predicted_entity) in predicted_entities
            and as_uri_ref(matched.gt_entity) in gt_entities
        ]
        matched_predicted = {
            as_uri_ref(matched.predicted_entity) for matched in instance_matches
        }
        matched_gt = {as_uri_ref(matched.gt_entity) for matched in instance_matches}
        entity_true_positives = len(instance_matches)
        entity_false_positives = len(predicted_entities - matched_predicted)
        entity_false_negatives = len(gt_entities - matched_gt)
        entity_precision, entity_recall, entity_f1 = compute_prf(
            entity_true_positives,
            len(predicted_entities),
            len(gt_entities),
        )
        domain_entity_matches = count_domain_entity_matches(entity_matches)

        return MatchMetrics(
            precision=precision,
            recall=recall,
            f1=f1,
            true_positives=true_positives,
            false_positives=false_positives,
            false_negatives=false_negatives,
            predicted_count=len(predicted),
            ground_truth_count=len(ground_truth),
            entity_precision=entity_precision,
            entity_recall=entity_recall,
            entity_f1=entity_f1,
            entity_true_positives=entity_true_positives,
            entity_false_positives=entity_false_positives,
            entity_false_negatives=entity_false_negatives,
            domain_entity_matches=domain_entity_matches,
            fact_precision=fact_precision,
            fact_recall=fact_recall,
            fact_f1=fact_f1,
            fact_true_positives=fact_true_positives,
            fact_false_positives=fact_false_positives,
            fact_false_negatives=fact_false_negatives,
            fact_predicted_count=len(predicted_facts),
            fact_ground_truth_count=len(ground_truth_facts),
        )

Methods:

evaluate(predicted_graph, gt_graph, entity_matches)
Source code in ontocast/tool/agg/triple_evaluator.py 
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
def evaluate(
    self,
    predicted_graph: RDFGraph,
    gt_graph: RDFGraph,
    entity_matches: list[EntityMatch],
) -> MatchMetrics:
    predicted_to_gt = {
        as_uri_ref(matched.predicted_entity): as_uri_ref(matched.gt_entity)
        for matched in entity_matches
    }

    raw_predicted = project_triples(predicted_graph, predicted_to_gt)
    raw_ground_truth = set(gt_graph)

    predicted = prepare_metric_triples(raw_predicted)
    ground_truth = prepare_metric_triples(raw_ground_truth)

    true_positives = len(predicted & ground_truth)
    false_positives = len(predicted - ground_truth)
    false_negatives = len(ground_truth - predicted)
    precision, recall, f1 = compute_prf(
        true_positives,
        len(predicted),
        len(ground_truth),
    )

    ontology_entities = collect_ontology_entities(predicted | ground_truth)
    predicted_facts = prepare_fact_triples(predicted, ontology_entities)
    ground_truth_facts = prepare_fact_triples(ground_truth, ontology_entities)
    fact_true_positives = len(predicted_facts & ground_truth_facts)
    fact_false_positives = len(predicted_facts - ground_truth_facts)
    fact_false_negatives = len(ground_truth_facts - predicted_facts)
    fact_precision, fact_recall, fact_f1 = compute_prf(
        fact_true_positives,
        len(predicted_facts),
        len(ground_truth_facts),
    )

    predicted_entities = set(extract_instance_entities(predicted_graph))
    gt_entities = set(extract_instance_entities(gt_graph))
    instance_matches = [
        matched
        for matched in entity_matches
        if as_uri_ref(matched.predicted_entity) in predicted_entities
        and as_uri_ref(matched.gt_entity) in gt_entities
    ]
    matched_predicted = {
        as_uri_ref(matched.predicted_entity) for matched in instance_matches
    }
    matched_gt = {as_uri_ref(matched.gt_entity) for matched in instance_matches}
    entity_true_positives = len(instance_matches)
    entity_false_positives = len(predicted_entities - matched_predicted)
    entity_false_negatives = len(gt_entities - matched_gt)
    entity_precision, entity_recall, entity_f1 = compute_prf(
        entity_true_positives,
        len(predicted_entities),
        len(gt_entities),
    )
    domain_entity_matches = count_domain_entity_matches(entity_matches)

    return MatchMetrics(
        precision=precision,
        recall=recall,
        f1=f1,
        true_positives=true_positives,
        false_positives=false_positives,
        false_negatives=false_negatives,
        predicted_count=len(predicted),
        ground_truth_count=len(ground_truth),
        entity_precision=entity_precision,
        entity_recall=entity_recall,
        entity_f1=entity_f1,
        entity_true_positives=entity_true_positives,
        entity_false_positives=entity_false_positives,
        entity_false_negatives=entity_false_negatives,
        domain_entity_matches=domain_entity_matches,
        fact_precision=fact_precision,
        fact_recall=fact_recall,
        fact_f1=fact_f1,
        fact_true_positives=fact_true_positives,
        fact_false_positives=fact_false_positives,
        fact_false_negatives=fact_false_negatives,
        fact_predicted_count=len(predicted_facts),
        fact_ground_truth_count=len(ground_truth_facts),
    )

Functions: