ontocast.tool.chunk.segment

Mutable text segments during chunk preparation (before sizing).

Attributes

logger = logging.getLogger(__name__) module-attribute

Classes

PrepareSegment dataclass

One logical segment prior to min/max sizing.

section_label_source records how the label was decided. It is load bearing, not bookkeeping: forward-fill refuses to overwrite a segment whose section is explicitly unresolved, which is what stops a neighbouring label from being smeared across an unrecognised section.

Source code in ontocast/tool/chunk/segment.py

@dataclass
class PrepareSegment:
    """One logical segment prior to min/max sizing.

    ``section_label_source`` records *how* the label was decided. It is load
    bearing, not bookkeeping: forward-fill refuses to overwrite a segment whose
    section is explicitly unresolved, which is what stops a neighbouring label
    from being smeared across an unrecognised section.
    """

    text: str
    headings: list[str] | None = None
    doc_item_refs: tuple[str, ...] = ()
    section_label: str | None = None
    section_label_source: SectionLabelSource | None = None
    section_label_confidence: float = 0.0

Attributes

doc_item_refs = () class-attribute instance-attribute

headings = None class-attribute instance-attribute

section_label = None class-attribute instance-attribute

section_label_confidence = 0.0 class-attribute instance-attribute

section_label_source = None class-attribute instance-attribute

text instance-attribute

Methods:

__init__(text, headings=None, doc_item_refs=(), section_label=None, section_label_source=None, section_label_confidence=0.0)

Functions:

coalesce_small_segments_right(segments, min_chars, schema)

Merge undersized segments into the right neighbor; otherwise into the left.

Source code in ontocast/tool/chunk/segment.py

def coalesce_small_segments_right(
    segments: list[PrepareSegment],
    min_chars: int,
    schema: SectionLabelSchema,
) -> list[PrepareSegment]:
    """Merge undersized segments into the right neighbor; otherwise into the left."""
    if min_chars <= 0 or len(segments) <= 1:
        return segments

    merge_count = 0
    index = 0
    while index < len(segments):
        if (
            is_abstract_exempt(segments[index], schema)
            or _effective_length(segments[index], schema) >= min_chars
        ):
            index += 1
            continue
        merged = False
        if index + 1 < len(segments) and not segments_differ_in_structure(
            segments[index], segments[index + 1], schema
        ):
            segments[index + 1] = merge_into_right(segments[index], segments[index + 1])
            del segments[index]
            merge_count += 1
            merged = True
        elif _can_merge_small_into_left(segments, index, schema):
            segments[index - 1] = merge_into_left(segments[index - 1], segments[index])
            del segments[index]
            merge_count += 1
            index -= 1
            merged = True
        if not merged:
            index += 1

    if merge_count:
        logger.debug(
            "Coalesced %s undersized segment(s) (min_chars=%s)",
            merge_count,
            min_chars,
        )
    return segments

is_abstract_exempt(segment, schema)

Preserve short abstract segments instead of merging them into neighbors.

Source code in ontocast/tool/chunk/segment.py

def is_abstract_exempt(segment: PrepareSegment, schema: SectionLabelSchema) -> bool:
    """Preserve short abstract segments instead of merging them into neighbors."""
    first_line = _first_non_empty_line(segment.text)
    if first_line and match_heading_line(first_line, schema) == "abstract":
        return True
    if segment.headings:
        for heading in segment.headings:
            if match_heading_line(heading, schema) == "abstract":
                return True
    return False

merge_doc_item_refs(left, right)

Source code in ontocast/tool/chunk/segment.py

def merge_doc_item_refs(
    left: tuple[str, ...], right: tuple[str, ...]
) -> tuple[str, ...]:
    seen: set[str] = set()
    merged: list[str] = []
    for ref in left + right:
        if ref not in seen:
            seen.add(ref)
            merged.append(ref)
    return tuple(merged)

merge_into_left(left, right)

Source code in ontocast/tool/chunk/segment.py

def merge_into_left(left: PrepareSegment, right: PrepareSegment) -> PrepareSegment:
    left_text = left.text.strip()
    right_text = right.text.strip()
    combined_text = (
        f"{left_text}{DEFAULT_PART_SEPARATOR}{right_text}"
        if left_text and right_text
        else left_text or right_text
    )
    winner = left if left.section_label is not None else right
    return PrepareSegment(
        text=combined_text,
        headings=left.headings or right.headings,
        doc_item_refs=merge_doc_item_refs(left.doc_item_refs, right.doc_item_refs),
        section_label=winner.section_label,
        section_label_source=winner.section_label_source,
        section_label_confidence=winner.section_label_confidence,
    )

merge_into_right(left, right)

Source code in ontocast/tool/chunk/segment.py

def merge_into_right(left: PrepareSegment, right: PrepareSegment) -> PrepareSegment:
    left_text = left.text.strip()
    right_text = right.text.strip()
    combined_text = (
        f"{left_text}{DEFAULT_PART_SEPARATOR}{right_text}"
        if left_text and right_text
        else left_text or right_text
    )
    winner = right if right.section_label is not None else left
    return PrepareSegment(
        text=combined_text,
        headings=right.headings or left.headings,
        doc_item_refs=merge_doc_item_refs(left.doc_item_refs, right.doc_item_refs),
        section_label=winner.section_label,
        section_label_source=winner.section_label_source,
        section_label_confidence=winner.section_label_confidence,
    )

segments_differ_in_structure(left, right, schema)

True when merging would join distinct document sections.

Source code in ontocast/tool/chunk/segment.py

def segments_differ_in_structure(
    left: PrepareSegment,
    right: PrepareSegment,
    schema: SectionLabelSchema,
) -> bool:
    """True when merging would join distinct document sections."""
    if starts_with_section_heading(right, schema):
        return True
    if left.headings and right.headings and left.headings != right.headings:
        return True
    return False

starts_with_section_heading(segment, schema)

True when the segment opens with a recognised section heading line.

Source code in ontocast/tool/chunk/segment.py

def starts_with_section_heading(
    segment: PrepareSegment, schema: SectionLabelSchema
) -> bool:
    """True when the segment opens with a recognised section heading line."""
    first_line = _first_non_empty_line(segment.text)
    return bool(first_line and match_heading_line(first_line, schema) is not None)