ontocast.tool.chunk.segment

Mutable text segments during chunk preparation (before sizing).

PrepareSegment dataclass

One logical segment prior to min/max sizing.

section_label_source records how the label was decided. It is load bearing, not bookkeeping: forward-fill refuses to overwrite a segment whose section is explicitly unresolved, which is what stops a neighbouring label from being smeared across an unrecognised section.

Source code in ontocast/tool/chunk/segment.py

@dataclass
class PrepareSegment:
    """One logical segment prior to min/max sizing.

    ``section_label_source`` records *how* the label was decided. It is load
    bearing, not bookkeeping: forward-fill refuses to overwrite a segment whose
    section is explicitly unresolved, which is what stops a neighbouring label
    from being smeared across an unrecognised section.
    """

    text: str
    headings: list[str] | None = None
    doc_item_refs: tuple[str, ...] = ()
    section_label: str | None = None
    section_label_source: SectionLabelSource | None = None
    section_label_confidence: float = 0.0

coalesce_small_segments_right(segments, min_chars, schema)

Merge undersized segments into the right neighbor; otherwise into the left.

Source code in ontocast/tool/chunk/segment.py

def coalesce_small_segments_right(
    segments: list[PrepareSegment],
    min_chars: int,
    schema: SectionLabelSchema,
) -> list[PrepareSegment]:
    """Merge undersized segments into the right neighbor; otherwise into the left."""
    if min_chars <= 0 or len(segments) <= 1:
        return segments

    merge_count = 0
    index = 0
    while index < len(segments):
        if (
            is_abstract_exempt(segments[index], schema)
            or _effective_length(segments[index], schema) >= min_chars
        ):
            index += 1
            continue
        merged = False
        if index + 1 < len(segments) and not segments_differ_in_structure(
            segments[index], segments[index + 1], schema
        ):
            segments[index + 1] = merge_into_right(segments[index], segments[index + 1])
            del segments[index]
            merge_count += 1
            merged = True
        elif _can_merge_small_into_left(segments, index, schema):
            segments[index - 1] = merge_into_left(segments[index - 1], segments[index])
            del segments[index]
            merge_count += 1
            index -= 1
            merged = True
        if not merged:
            index += 1

    if merge_count:
        logger.debug(
            "Coalesced %s undersized segment(s) (min_chars=%s)",
            merge_count,
            min_chars,
        )
    return segments

is_abstract_exempt(segment, schema)

Preserve short abstract segments instead of merging them into neighbors.

Source code in ontocast/tool/chunk/segment.py

def is_abstract_exempt(segment: PrepareSegment, schema: SectionLabelSchema) -> bool:
    """Preserve short abstract segments instead of merging them into neighbors."""
    first_line = _first_non_empty_line(segment.text)
    if first_line and match_heading_line(first_line, schema) == "abstract":
        return True
    if segment.headings:
        for heading in segment.headings:
            if match_heading_line(heading, schema) == "abstract":
                return True
    return False

segments_differ_in_structure(left, right, schema)

True when merging would join distinct document sections.

Source code in ontocast/tool/chunk/segment.py

def segments_differ_in_structure(
    left: PrepareSegment,
    right: PrepareSegment,
    schema: SectionLabelSchema,
) -> bool:
    """True when merging would join distinct document sections."""
    if starts_with_section_heading(right, schema):
        return True
    if left.headings and right.headings and left.headings != right.headings:
        return True
    return False

starts_with_section_heading(segment, schema)

True when the segment opens with a recognised section heading line.

Source code in ontocast/tool/chunk/segment.py

def starts_with_section_heading(
    segment: PrepareSegment, schema: SectionLabelSchema
) -> bool:
    """True when the segment opens with a recognised section heading line."""
    first_line = _first_non_empty_line(segment.text)
    return bool(first_line and match_heading_line(first_line, schema) is not None)