Skip to content

ontocast.tool.chunk.segment

Mutable text segments during chunk preparation (before sizing).

PrepareSegment dataclass

One logical segment prior to min/max sizing.

Source code in ontocast/tool/chunk/segment.py 
14
15
16
17
18
19
20
21
@dataclass
class PrepareSegment:
    """One logical segment prior to min/max sizing."""

    text: str
    headings: list[str] | None = None
    doc_item_refs: tuple[str, ...] = ()
    section_label: str | None = None

coalesce_small_segments_right(segments, min_chars, schema)

Merge undersized segments into the right neighbor; otherwise into the left.

Source code in ontocast/tool/chunk/segment.py 
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
def coalesce_small_segments_right(
    segments: list[PrepareSegment],
    min_chars: int,
    schema: SectionLabelSchema,
) -> list[PrepareSegment]:
    """Merge undersized segments into the right neighbor; otherwise into the left."""
    if min_chars <= 0 or len(segments) <= 1:
        return segments

    merge_count = 0
    index = 0
    while index < len(segments):
        if (
            is_abstract_exempt(segments[index], schema)
            or _effective_length(segments[index], schema) >= min_chars
        ):
            index += 1
            continue
        merged = False
        if index + 1 < len(segments) and not segments_differ_in_structure(
            segments[index], segments[index + 1], schema
        ):
            segments[index + 1] = merge_into_right(segments[index], segments[index + 1])
            del segments[index]
            merge_count += 1
            merged = True
        elif _can_merge_small_into_left(segments, index, schema):
            segments[index - 1] = merge_into_left(segments[index - 1], segments[index])
            del segments[index]
            merge_count += 1
            index -= 1
            merged = True
        if not merged:
            index += 1

    if merge_count:
        logger.debug(
            "Coalesced %s undersized segment(s) (min_chars=%s)",
            merge_count,
            min_chars,
        )
    return segments

is_abstract_exempt(segment, schema)

Preserve short abstract segments instead of merging them into neighbors.

Source code in ontocast/tool/chunk/segment.py 
44
45
46
47
48
49
50
51
52
53
def is_abstract_exempt(segment: PrepareSegment, schema: SectionLabelSchema) -> bool:
    """Preserve short abstract segments instead of merging them into neighbors."""
    first_line = _first_non_empty_line(segment.text)
    if first_line and match_heading_line(first_line, schema) == "abstract":
        return True
    if segment.headings:
        for heading in segment.headings:
            if match_heading_line(heading, schema) == "abstract":
                return True
    return False

segments_differ_in_structure(left, right, schema)

True when merging would join distinct document sections.

Source code in ontocast/tool/chunk/segment.py 
86
87
88
89
90
91
92
93
94
95
96
def segments_differ_in_structure(
    left: PrepareSegment,
    right: PrepareSegment,
    schema: SectionLabelSchema,
) -> bool:
    """True when merging would join distinct document sections."""
    if starts_with_section_heading(right, schema):
        return True
    if left.headings and right.headings and left.headings != right.headings:
        return True
    return False

starts_with_section_heading(segment, schema)

True when the segment opens with a recognised section heading line.

Source code in ontocast/tool/chunk/segment.py 
56
57
58
59
60
61
def starts_with_section_heading(
    segment: PrepareSegment, schema: SectionLabelSchema
) -> bool:
    """True when the segment opens with a recognised section heading line."""
    first_line = _first_non_empty_line(segment.text)
    return bool(first_line and match_heading_line(first_line, schema) is not None)