Skip to content

ontocast.tool.chunk.sections

Document section span detection and overlap-based labeling for chunk prepare.

DocumentOutline

Bases: BasePydanticModel

Ordered headings detected in a document, with the document length.

Source code in ontocast/onto/section_models.py 
46
47
48
49
50
class DocumentOutline(BasePydanticModel):
    """Ordered headings detected in a document, with the document length."""

    text_len: int
    nodes: list[HeadingNode] = Field(default_factory=list)

HeadingNode

Bases: BasePydanticModel

One detected heading in the document outline.

Attributes:

Name Type Description
text str

Raw heading line as it appears in the document text.
normalised str

Heading text after decoration/numbering stripping.
start int

Character offset of the heading line itself.
body_start int

Character offset just past the heading line.
level int

Markdown heading depth (1 = top). Docling reports a flat level for PDF conversions, so this is informational only.
sectionlike bool

Whether the heading reads as a generic section name rather than a descriptive subsection title or a document title.
label str | None

Canonical section label, when resolved.
source SectionLabelSource

How label was decided.
confidence float

Confidence in label in [0, 1].
Source code in ontocast/onto/section_models.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
class HeadingNode(BasePydanticModel):
    """One detected heading in the document outline.

    Attributes:
        text: Raw heading line as it appears in the document text.
        normalised: Heading text after decoration/numbering stripping.
        start: Character offset of the heading line itself.
        body_start: Character offset just past the heading line.
        level: Markdown heading depth (1 = top). Docling reports a flat level
            for PDF conversions, so this is informational only.
        sectionlike: Whether the heading reads as a generic section name rather
            than a descriptive subsection title or a document title.
        label: Canonical section label, when resolved.
        source: How ``label`` was decided.
        confidence: Confidence in ``label`` in ``[0, 1]``.
    """

    text: str
    normalised: str
    start: int
    body_start: int
    level: int = 1
    sectionlike: bool = True
    label: str | None = None
    source: SectionLabelSource = SectionLabelSource.OUTLINE_UNRESOLVED
    confidence: float = 0.0

SectionSpan

Bases: BasePydanticModel

Character span of a document section with a normalised label.

label is None for a region whose section type is not (yet) known — for example an unrecognised but section-like heading. Such a span is explicitly unresolved rather than absent, which is what stops a neighbouring label from being smeared across it.

Source code in ontocast/onto/section_models.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
class SectionSpan(BasePydanticModel):
    """Character span of a document section with a normalised label.

    ``label`` is ``None`` for a region whose section type is not (yet) known —
    for example an unrecognised but section-like heading. Such a span is
    explicitly unresolved rather than absent, which is what stops a neighbouring
    label from being smeared across it.
    """

    label: str | None = None
    start: int
    end: int
    source: SectionLabelSource = SectionLabelSource.OUTLINE_UNRESOLVED
    confidence: float = 0.0

build_section_spans_from_labels(text, labeled_headings)

Build section spans from explicit (offset, label) pairs.

Source code in ontocast/tool/chunk/sections.py 
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
def build_section_spans_from_labels(
    text: str, labeled_headings: list[tuple[int, str]]
) -> list[SectionSpan]:
    """Build section spans from explicit ``(offset, label)`` pairs."""
    outline = DocumentOutline(
        text_len=len(text),
        nodes=[
            HeadingNode(
                text=label,
                normalised=label,
                start=start,
                body_start=start,
                label=label,
                source=SectionLabelSource.HEADING_PATTERN,
                confidence=0.95,
            )
            for start, label in sorted(labeled_headings, key=lambda item: item[0])
        ],
    )
    return [span for span in outline_to_spans(outline) if span.label is not None]

detect_section_spans(text, schema=None, *, include_text_headings=False)

Detect document sections and return a partition of the text into spans.

Every heading closes the preceding span, so an unrecognised heading yields an explicitly unresolved (label=None) span instead of letting the previous label run on to the next recognised heading.

Parameters:

Name Type Description Default
text str

Document text (the markdown export).

 required
schema SectionLabelSchema | None

Section label schema; the manifest default when omitted.

 None
include_text_headings bool

Enable the plain-text heading heuristic for documents with no markdown heading structure.

 False

Returns:

Type Description
list[SectionSpan]

Section spans tiling text, ordered by start offset.
Source code in ontocast/tool/chunk/sections.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
def detect_section_spans(
    text: str,
    schema: SectionLabelSchema | None = None,
    *,
    include_text_headings: bool = False,
) -> list[SectionSpan]:
    """Detect document sections and return a partition of the text into spans.

    Every heading closes the preceding span, so an unrecognised heading yields an
    explicitly unresolved (``label=None``) span instead of letting the previous
    label run on to the next recognised heading.

    Args:
        text: Document text (the markdown export).
        schema: Section label schema; the manifest default when omitted.
        include_text_headings: Enable the plain-text heading heuristic for
            documents with no markdown heading structure.

    Returns:
        Section spans tiling ``text``, ordered by start offset.
    """
    if not text:
        return []
    active = schema or get_default_section_schema()
    outline = build_document_outline(
        text, active, include_text_headings=include_text_headings
    )
    spans = outline_to_spans(outline)
    return inject_front_matter_spans(spans, text, active)

document_text_for_section_tagging(doc)

Export document text used for section heading detection.

Source code in ontocast/tool/chunk/sections.py 
23
24
25
def document_text_for_section_tagging(doc: DoclingDocument) -> str:
    """Export document text used for section heading detection."""
    return doc.export_to_markdown()

inject_front_matter_spans(spans, text, schema, *, min_gap_chars=80, max_gap_chars=ABSTRACT_FRONT_MATTER_MAX_CHARS)

Label unheaded front matter before the first labeled section as abstract.

Leading unresolved spans (a title block, an unrecognised banner heading) are skipped when locating the first labeled section, so front matter is still recovered on documents whose first recognised section is not an IMRaD opener -- papers that jump straight to Results are common.

Source code in ontocast/tool/chunk/sections.py 
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def inject_front_matter_spans(
    spans: list[SectionSpan],
    text: str,
    schema: SectionLabelSchema,
    *,
    min_gap_chars: int = 80,
    max_gap_chars: int = ABSTRACT_FRONT_MATTER_MAX_CHARS,
) -> list[SectionSpan]:
    """Label unheaded front matter before the first labeled section as abstract.

    Leading unresolved spans (a title block, an unrecognised banner heading) are
    skipped when locating the first labeled section, so front matter is still
    recovered on documents whose first recognised section is not an IMRaD
    opener -- papers that jump straight to ``Results`` are common.
    """
    if "abstract" not in canonical_labels(schema):
        return spans
    if any(span.label == "abstract" for span in spans):
        return spans
    if not spans:
        return spans

    ordered = sorted(spans, key=lambda span: span.start)
    first_labeled = next(
        (span for span in ordered if span.label is not None),
        None,
    )
    if first_labeled is None or first_labeled.start <= 0:
        return spans

    gap = text[: first_labeled.start].strip()
    gap_len = len(gap)
    if gap_len < min_gap_chars or gap_len > max_gap_chars:
        return spans

    # The front matter may already be covered by unresolved spans; replace them
    # rather than overlapping, so the result stays a partition.
    tail = [span for span in ordered if span.start >= first_labeled.start]
    abstract_span = SectionSpan(
        label="abstract",
        start=0,
        end=first_labeled.start,
        source=SectionLabelSource.FRONT_MATTER,
        confidence=0.5,
    )
    return [abstract_span, *tail]

label_from_headings(headings, schema)

Return the first schema label matched from the heading breadcrumb (most-specific first).

This uses docling's structural metadata directly — no substring search required — so it is reliable even when the markdown export differs from the hybrid-chunker text.

Source code in ontocast/tool/chunk/sections.py 
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
def label_from_headings(
    headings: list[str] | None,
    schema: SectionLabelSchema,
) -> str | None:
    """Return the first schema label matched from the heading breadcrumb (most-specific first).

    This uses docling's structural metadata directly — no substring search
    required — so it is reliable even when the markdown export differs from
    the hybrid-chunker text.
    """
    if not headings:
        return None
    for heading in reversed(headings):
        label = match_heading_line(heading, schema)
        if label is not None:
            return label
    return None

label_text_from_spans(text, document_text, spans, search_from)

Assign section label via span overlap; return label and next search offset.

Source code in ontocast/tool/chunk/sections.py 
171
172
173
174
175
176
177
178
def label_text_from_spans(
    text: str,
    document_text: str,
    spans: list[SectionSpan],
    search_from: int,
) -> tuple[str | None, int]:
    """Assign section label via span overlap; return label and next search offset."""
    return resolve_section_label(text, document_text, spans, search_from)

resolve_section_label(chunk_text, document_text, spans, search_from=0)

Return section label with max overlap; second value is next search offset.

When the chunk text cannot be located in document_text the cursor is preserved at search_from (not reset to 0) so that subsequent segments are not mis-anchored to the start of the document.

Source code in ontocast/tool/chunk/sections.py 
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def resolve_section_label(
    chunk_text: str,
    document_text: str,
    spans: list[SectionSpan],
    search_from: int = 0,
) -> tuple[str | None, int]:
    """Return section label with max overlap; second value is next search offset.

    When the chunk text cannot be located in ``document_text`` the cursor is
    preserved at ``search_from`` (not reset to 0) so that subsequent segments
    are not mis-anchored to the start of the document.
    """
    start, end = _chunk_char_range(chunk_text, document_text, search_from)
    if end <= start or not spans:
        # Preserve cursor instead of returning 0 so subsequent finds stay ordered.
        return None, search_from

    best_label: str | None = None
    best_overlap = 0
    for span in spans:
        overlap_start = max(start, span.start)
        overlap_end = min(end, span.end)
        overlap = max(0, overlap_end - overlap_start)
        if overlap > best_overlap:
            best_overlap = overlap
            best_label = span.label
    return best_label, end