Skip to content

ontocast.tool.chunk

Document chunking tools for OntoCast.

Modules:

Name Description
assignment

Section label assignment: every tier adds evidence, one rule decides.

bibliography

Reference-list detection at block level, and trimming of citation runs.

chunker
contents

Tables of contents and lists of figures or tables, recognised from their content.

density

Content-based section classification for regions with no usable heading.

embedding

Embedding helpers shared by the heading tiers: the embedder type and cosine.

heading_embedding

Embedding heading tier: a heading no pattern or keyword named votes for its nearest label.

non_content

Non-content unit detection: front and back matter that carries no domain facts.

outline

Document outline detection for chunk preparation.

prepare

Prepare content units: segment, label, select, size, trim and route.

proposition

Lightweight text splitting helpers (no ML dependencies).

routing

Route each prepared unit before extraction: extract, citations, non-content or skip.

schema_detect

Detect which document-type schema a document belongs to.

section_llm

LLM section tier: one call labels the document's outline, as weak votes.

sections

Document section spans for chunk prepare: outline spans plus front matter.

segment

Mutable text segments during chunk preparation (before sizing).

sizing

Shared text sizing helpers for OntoCast chunking.

structure

Document structure: the blocks of the markdown export and their kinds.

util

Attributes

NormalizedChunk = PreparedChunk module-attribute

__all__ = ['ChunkerTool', 'NormalizedChunk', 'PreparedChunk', 'PrepareOptions', 'SectionSelectionEmptyError', 'prepare_content_units', 'size_bounded_text'] module-attribute

Classes

ChunkerTool

Bases: Tool

Tool for semantic chunking of documents.

Falls back to naive chunking if sentence-transformers is not available. Includes caching to avoid re-chunking the same text with the same parameters.

Source code in ontocast/tool/chunk/chunker.py
class ChunkerTool(Tool):
    """Tool for semantic chunking of documents.

    Falls back to naive chunking if sentence-transformers is not available.
    Includes caching to avoid re-chunking the same text with the same parameters.
    """

    config: ChunkConfig = Field(
        default_factory=ChunkConfig, description="Chunking configuration parameters"
    )
    chunking_mode: Literal["semantic", "naive"] = Field(
        default="semantic",
        description=(
            "Effective chunking mode: CHUNK_CHUNKING_MODE, or 'naive' when "
            "semantic chunking is unavailable in this environment"
        ),
    )
    cache: Any = Field(default=None, exclude=True)

    def __init__(
        self,
        chunk_config: ChunkConfig | None = None,
        cache: Cacher | None = None,
        **kwargs: Any,
    ):
        """Initialize the ChunkerTool.

        Args:
            chunk_config: Chunking configuration. If None, uses default ChunkConfig.
            cache: Optional shared Cacher instance. If None, creates a new one.
            **kwargs: Additional keyword arguments passed to the parent class.
        """
        super().__init__(**kwargs)
        # The model itself is process-shared and its construction is locked by
        # get_shared_encoder; all this holds is the per-tool adapter around it.
        self._embeddings: SharedSentenceTransformerEmbeddings | None = None
        self._embeddings_unavailable = False

        # Initialize cache - use shared cacher or create new one
        if cache is not None:
            self.cache = ToolCacher(cache, CHUNKER_CACHE_SUBDIR)
        else:
            # Standalone use (CLI helpers, direct library use): fall back to a
            # private Cacher on the configured/default directory.
            shared_cache = Cacher()
            self.cache = ToolCacher(shared_cache, CHUNKER_CACHE_SUBDIR)

        # Override config if provided
        if chunk_config is not None:
            self.config = chunk_config
        self.chunking_mode = self.config.chunking_mode

        # Probe heavy deps only when semantic mode is requested
        if self.chunking_mode == "semantic" and not _semantic_chunking_available():
            self.chunking_mode = "naive"
            logger.warning(
                "Semantic chunking not available (needs the 'semantic-chunking' "
                "extra: sentence-transformers, hdbscan, umap-learn). "
                "Falling back to naive chunking."
            )

    def embeddings(self) -> SharedSentenceTransformerEmbeddings | None:
        """Embeddings over the process-shared encoder, or ``None`` if unavailable.

        The encoder is shared with retrieval and entity clustering when their
        model names match, so this loads no weights of its own in that case, and
        its inference is serialised against theirs.
        """
        if self._embeddings is not None or self._embeddings_unavailable:
            return self._embeddings
        if not _embedding_model_available():
            self._embeddings_unavailable = True
            return None
        try:
            self._embeddings = SharedSentenceTransformerEmbeddings(
                get_shared_encoder(
                    self.config.embedding_model,
                    feature=(
                        "Semantic chunking and schema detection. Install the "
                        "'semantic-chunking' extra"
                    ),
                ),
                normalize=False,
            )
        except Exception as exc:
            # Record the failure rather than retrying the load on every call:
            # a missing or broken checkpoint does not become available later in
            # the same process.
            logger.error("Failed to initialize chunker embedding model: %s", exc)
            self._embeddings_unavailable = True
            return None
        return self._embeddings

    def embed_texts(self, texts: list[str]) -> list[list[float]] | None:
        """Embed short texts with the chunker's model, or ``None`` if unavailable.

        Exposed so document-type detection can reuse the model already loaded
        for semantic chunking instead of constructing a second one. Returns
        ``None`` -- rather than raising -- when the semantic extras are absent,
        so callers degrade to their deterministic tiers exactly as chunking
        itself degrades to ``naive``.

        Args:
            texts: Short strings to embed (headings or sampled paragraphs).

        Returns:
            One embedding per input, or ``None`` when no model is available.
        """
        if not texts:
            return []
        embeddings = self.embeddings()
        if embeddings is None:
            return None
        try:
            return embeddings.embed_documents(texts)
        except Exception as exc:  # pragma: no cover - environment dependent
            logger.warning("Embedding failed, skipping semantic tier: %s", exc)
            return None

    def naive_split(self, doc: str) -> list[str]:
        """Split text by paragraph/sentence boundaries up to ``max_size``.

        Unlike :meth:`_naive_chunk`, does not enforce ``min_size`` filtering.
        """
        paragraphs = re.split(r"\n\s*\n", doc.strip())

        chunks: list[str] = []
        current_chunk = ""

        for paragraph in paragraphs:
            paragraph = paragraph.strip()
            if not paragraph:
                continue

            if (
                current_chunk
                and len(current_chunk) + len(paragraph) + 2 > self.config.max_size
            ):
                if current_chunk:
                    chunks.append(current_chunk.strip())
                current_chunk = paragraph
            else:
                if current_chunk:
                    current_chunk += "\n\n" + paragraph
                else:
                    current_chunk = paragraph

            if len(current_chunk) > self.config.max_size:
                if len(current_chunk) - len(paragraph) - 2 > 0:
                    prev_chunk = current_chunk[
                        : len(current_chunk) - len(paragraph) - 2
                    ].strip()
                    if prev_chunk:
                        chunks.append(prev_chunk)

                sentences = re.split(r"(?<=[.!?])\s+", paragraph)
                temp_chunk = ""

                for sentence in sentences:
                    if len(temp_chunk) + len(sentence) + 1 > self.config.max_size:
                        if temp_chunk:
                            chunks.append(temp_chunk.strip())
                        temp_chunk = sentence
                    else:
                        if temp_chunk:
                            temp_chunk += " " + sentence
                        else:
                            temp_chunk = sentence

                current_chunk = temp_chunk

        if current_chunk:
            chunks.append(current_chunk.strip())

        return chunks

    def size_text(self, doc: str) -> list[str]:
        """Split ``doc`` to respect ``min_size`` / ``max_size`` using naive boundaries."""
        return size_bounded_text(doc, self.config, self.naive_split)

    def _naive_chunk(self, doc: str) -> list[str]:
        """Naive chunking fallback when semantic chunking is not available.

        Args:
            doc: The document text to chunk.

        Returns:
            List of text chunks.
        """
        chunks = self.size_text(doc)

        logger.info(f"Naive chunking produced {len(chunks)} chunks")
        return chunks

    def __call__(self, doc: str) -> list[str]:
        """Chunk a document into semantic segments.

        Args:
            doc: The document text to chunk.

        Returns:
            List of text chunks.
        """
        # Prepare configuration for caching. The "model" key name is kept even
        # though its source moved to ChunkConfig -- the dict is hashed, so
        # renaming it would invalidate every cached chunking for no reason.
        config_dict = {
            "model": self.config.embedding_model,
            "chunking_mode": self.chunking_mode,
            "max_size": self.config.max_size,
            "min_size": self.config.min_size,
            "cache_format_version": CHUNKER_CACHE_FORMAT_VERSION,
        }

        # Check cache first
        cached_result = self.cache.get(doc, config=config_dict)
        if cached_result is not None:
            logger.debug("Cache hit for document chunking")
            return cached_result

        # Perform chunking
        embeddings = None if self.chunking_mode == "naive" else self.embeddings()
        if embeddings is None or not _semantic_chunking_available():
            if self.chunking_mode != "naive":
                logger.warning(
                    "Semantic chunking requested but not available. "
                    "Falling back to naive chunking."
                )
            result = self._naive_chunk(doc)
        else:
            from ontocast.tool.chunk.util import SemanticChunker

            text_splitter = SemanticChunker(
                embeddings=embeddings,
                chunk_config=self.config,
                sentence_split_regex=SENTENCE_SPLIT_REGEX,
            )

            try:
                # SemanticChunker now handles max_size internally
                result_docs = text_splitter.create_documents([doc])
                result = [chunk.page_content for chunk in result_docs]
            except ValueError as exc:
                # Degenerate inputs (too few distinct sentences for the
                # HDBSCAN neighborhood) must not fail chunking outright.
                logger.warning(
                    "Semantic chunking failed (%s); falling back to "
                    "naive chunking for this text.",
                    exc,
                )
                result = self._naive_chunk(doc)

            # Log chunk lengths for debugging
            lens = [len(chunk) for chunk in result]
            logger.info(
                f"Semantic chunking produced {len(result)} chunks with lengths: {lens}"
            )

        # Cache the result
        self.cache.set(doc, result, config=config_dict)
        logger.debug("Cached document chunking result")

        return result

Attributes

cache = Field(default=None, exclude=True) class-attribute instance-attribute
chunking_mode = self.config.chunking_mode class-attribute instance-attribute
config = Field(default_factory=ChunkConfig, description='Chunking configuration parameters') class-attribute instance-attribute

Methods:

__call__(doc)

Chunk a document into semantic segments.

Parameters:

Name Type Description Default
doc str

The document text to chunk.

required

Returns:

Type Description
list[str]

List of text chunks.

Source code in ontocast/tool/chunk/chunker.py
def __call__(self, doc: str) -> list[str]:
    """Chunk a document into semantic segments.

    Args:
        doc: The document text to chunk.

    Returns:
        List of text chunks.
    """
    # Prepare configuration for caching. The "model" key name is kept even
    # though its source moved to ChunkConfig -- the dict is hashed, so
    # renaming it would invalidate every cached chunking for no reason.
    config_dict = {
        "model": self.config.embedding_model,
        "chunking_mode": self.chunking_mode,
        "max_size": self.config.max_size,
        "min_size": self.config.min_size,
        "cache_format_version": CHUNKER_CACHE_FORMAT_VERSION,
    }

    # Check cache first
    cached_result = self.cache.get(doc, config=config_dict)
    if cached_result is not None:
        logger.debug("Cache hit for document chunking")
        return cached_result

    # Perform chunking
    embeddings = None if self.chunking_mode == "naive" else self.embeddings()
    if embeddings is None or not _semantic_chunking_available():
        if self.chunking_mode != "naive":
            logger.warning(
                "Semantic chunking requested but not available. "
                "Falling back to naive chunking."
            )
        result = self._naive_chunk(doc)
    else:
        from ontocast.tool.chunk.util import SemanticChunker

        text_splitter = SemanticChunker(
            embeddings=embeddings,
            chunk_config=self.config,
            sentence_split_regex=SENTENCE_SPLIT_REGEX,
        )

        try:
            # SemanticChunker now handles max_size internally
            result_docs = text_splitter.create_documents([doc])
            result = [chunk.page_content for chunk in result_docs]
        except ValueError as exc:
            # Degenerate inputs (too few distinct sentences for the
            # HDBSCAN neighborhood) must not fail chunking outright.
            logger.warning(
                "Semantic chunking failed (%s); falling back to "
                "naive chunking for this text.",
                exc,
            )
            result = self._naive_chunk(doc)

        # Log chunk lengths for debugging
        lens = [len(chunk) for chunk in result]
        logger.info(
            f"Semantic chunking produced {len(result)} chunks with lengths: {lens}"
        )

    # Cache the result
    self.cache.set(doc, result, config=config_dict)
    logger.debug("Cached document chunking result")

    return result
__init__(chunk_config=None, cache=None, **kwargs)

Initialize the ChunkerTool.

Parameters:

Name Type Description Default
chunk_config ChunkConfig | None

Chunking configuration. If None, uses default ChunkConfig.

None
cache Cacher | None

Optional shared Cacher instance. If None, creates a new one.

None
**kwargs Any

Additional keyword arguments passed to the parent class.

{}
Source code in ontocast/tool/chunk/chunker.py
def __init__(
    self,
    chunk_config: ChunkConfig | None = None,
    cache: Cacher | None = None,
    **kwargs: Any,
):
    """Initialize the ChunkerTool.

    Args:
        chunk_config: Chunking configuration. If None, uses default ChunkConfig.
        cache: Optional shared Cacher instance. If None, creates a new one.
        **kwargs: Additional keyword arguments passed to the parent class.
    """
    super().__init__(**kwargs)
    # The model itself is process-shared and its construction is locked by
    # get_shared_encoder; all this holds is the per-tool adapter around it.
    self._embeddings: SharedSentenceTransformerEmbeddings | None = None
    self._embeddings_unavailable = False

    # Initialize cache - use shared cacher or create new one
    if cache is not None:
        self.cache = ToolCacher(cache, CHUNKER_CACHE_SUBDIR)
    else:
        # Standalone use (CLI helpers, direct library use): fall back to a
        # private Cacher on the configured/default directory.
        shared_cache = Cacher()
        self.cache = ToolCacher(shared_cache, CHUNKER_CACHE_SUBDIR)

    # Override config if provided
    if chunk_config is not None:
        self.config = chunk_config
    self.chunking_mode = self.config.chunking_mode

    # Probe heavy deps only when semantic mode is requested
    if self.chunking_mode == "semantic" and not _semantic_chunking_available():
        self.chunking_mode = "naive"
        logger.warning(
            "Semantic chunking not available (needs the 'semantic-chunking' "
            "extra: sentence-transformers, hdbscan, umap-learn). "
            "Falling back to naive chunking."
        )
embed_texts(texts)

Embed short texts with the chunker's model, or None if unavailable.

Exposed so document-type detection can reuse the model already loaded for semantic chunking instead of constructing a second one. Returns None -- rather than raising -- when the semantic extras are absent, so callers degrade to their deterministic tiers exactly as chunking itself degrades to naive.

Parameters:

Name Type Description Default
texts list[str]

Short strings to embed (headings or sampled paragraphs).

required

Returns:

Type Description
list[list[float]] | None

One embedding per input, or None when no model is available.

Source code in ontocast/tool/chunk/chunker.py
def embed_texts(self, texts: list[str]) -> list[list[float]] | None:
    """Embed short texts with the chunker's model, or ``None`` if unavailable.

    Exposed so document-type detection can reuse the model already loaded
    for semantic chunking instead of constructing a second one. Returns
    ``None`` -- rather than raising -- when the semantic extras are absent,
    so callers degrade to their deterministic tiers exactly as chunking
    itself degrades to ``naive``.

    Args:
        texts: Short strings to embed (headings or sampled paragraphs).

    Returns:
        One embedding per input, or ``None`` when no model is available.
    """
    if not texts:
        return []
    embeddings = self.embeddings()
    if embeddings is None:
        return None
    try:
        return embeddings.embed_documents(texts)
    except Exception as exc:  # pragma: no cover - environment dependent
        logger.warning("Embedding failed, skipping semantic tier: %s", exc)
        return None
embeddings()

Embeddings over the process-shared encoder, or None if unavailable.

The encoder is shared with retrieval and entity clustering when their model names match, so this loads no weights of its own in that case, and its inference is serialised against theirs.

Source code in ontocast/tool/chunk/chunker.py
def embeddings(self) -> SharedSentenceTransformerEmbeddings | None:
    """Embeddings over the process-shared encoder, or ``None`` if unavailable.

    The encoder is shared with retrieval and entity clustering when their
    model names match, so this loads no weights of its own in that case, and
    its inference is serialised against theirs.
    """
    if self._embeddings is not None or self._embeddings_unavailable:
        return self._embeddings
    if not _embedding_model_available():
        self._embeddings_unavailable = True
        return None
    try:
        self._embeddings = SharedSentenceTransformerEmbeddings(
            get_shared_encoder(
                self.config.embedding_model,
                feature=(
                    "Semantic chunking and schema detection. Install the "
                    "'semantic-chunking' extra"
                ),
            ),
            normalize=False,
        )
    except Exception as exc:
        # Record the failure rather than retrying the load on every call:
        # a missing or broken checkpoint does not become available later in
        # the same process.
        logger.error("Failed to initialize chunker embedding model: %s", exc)
        self._embeddings_unavailable = True
        return None
    return self._embeddings
naive_split(doc)

Split text by paragraph/sentence boundaries up to max_size.

Unlike :meth:_naive_chunk, does not enforce min_size filtering.

Source code in ontocast/tool/chunk/chunker.py
def naive_split(self, doc: str) -> list[str]:
    """Split text by paragraph/sentence boundaries up to ``max_size``.

    Unlike :meth:`_naive_chunk`, does not enforce ``min_size`` filtering.
    """
    paragraphs = re.split(r"\n\s*\n", doc.strip())

    chunks: list[str] = []
    current_chunk = ""

    for paragraph in paragraphs:
        paragraph = paragraph.strip()
        if not paragraph:
            continue

        if (
            current_chunk
            and len(current_chunk) + len(paragraph) + 2 > self.config.max_size
        ):
            if current_chunk:
                chunks.append(current_chunk.strip())
            current_chunk = paragraph
        else:
            if current_chunk:
                current_chunk += "\n\n" + paragraph
            else:
                current_chunk = paragraph

        if len(current_chunk) > self.config.max_size:
            if len(current_chunk) - len(paragraph) - 2 > 0:
                prev_chunk = current_chunk[
                    : len(current_chunk) - len(paragraph) - 2
                ].strip()
                if prev_chunk:
                    chunks.append(prev_chunk)

            sentences = re.split(r"(?<=[.!?])\s+", paragraph)
            temp_chunk = ""

            for sentence in sentences:
                if len(temp_chunk) + len(sentence) + 1 > self.config.max_size:
                    if temp_chunk:
                        chunks.append(temp_chunk.strip())
                    temp_chunk = sentence
                else:
                    if temp_chunk:
                        temp_chunk += " " + sentence
                    else:
                        temp_chunk = sentence

            current_chunk = temp_chunk

    if current_chunk:
        chunks.append(current_chunk.strip())

    return chunks
size_text(doc)

Split doc to respect min_size / max_size using naive boundaries.

Source code in ontocast/tool/chunk/chunker.py
def size_text(self, doc: str) -> list[str]:
    """Split ``doc`` to respect ``min_size`` / ``max_size`` using naive boundaries."""
    return size_bounded_text(doc, self.config, self.naive_split)

PrepareOptions dataclass

Options for the chunk preparation pipeline.

Source code in ontocast/tool/chunk/prepare.py
@dataclass
class PrepareOptions:
    """Options for the chunk preparation pipeline."""

    section_schema_id: str | None = None
    document_type_hint: str | None = None
    target_sections: list[str] | None = None
    summarize_sections: list[str] | None = None
    exclude_sections: list[str] | None = None

    def needs_section_prepare(self) -> bool:
        """True when a request option explicitly requires section labels.

        Section tagging itself is default-on (see ``CHUNK_SECTION_CLASSIFIER``);
        this only reports whether the request carries section-dependent options.
        """
        return (
            self.target_sections is not None
            or self.summarize_sections is not None
            or self.exclude_sections is not None
        )

    def filter_allowlist(self) -> list[str] | None:
        """The extraction allowlist: ``target_sections`` only.

        ``summarize_sections`` chooses which units are summarized and never
        what is extracted.
        """
        return self.target_sections

Attributes

document_type_hint = None class-attribute instance-attribute
exclude_sections = None class-attribute instance-attribute
section_schema_id = None class-attribute instance-attribute
summarize_sections = None class-attribute instance-attribute
target_sections = None class-attribute instance-attribute

Methods:

__init__(section_schema_id=None, document_type_hint=None, target_sections=None, summarize_sections=None, exclude_sections=None)
filter_allowlist()

The extraction allowlist: target_sections only.

summarize_sections chooses which units are summarized and never what is extracted.

Source code in ontocast/tool/chunk/prepare.py
def filter_allowlist(self) -> list[str] | None:
    """The extraction allowlist: ``target_sections`` only.

    ``summarize_sections`` chooses which units are summarized and never
    what is extracted.
    """
    return self.target_sections
needs_section_prepare()

True when a request option explicitly requires section labels.

Section tagging itself is default-on (see CHUNK_SECTION_CLASSIFIER); this only reports whether the request carries section-dependent options.

Source code in ontocast/tool/chunk/prepare.py
def needs_section_prepare(self) -> bool:
    """True when a request option explicitly requires section labels.

    Section tagging itself is default-on (see ``CHUNK_SECTION_CLASSIFIER``);
    this only reports whether the request carries section-dependent options.
    """
    return (
        self.target_sections is not None
        or self.summarize_sections is not None
        or self.exclude_sections is not None
    )

PreparedChunk dataclass

A prepared text chunk with optional structural metadata, label and route.

section_label_source and section_label_strength record which tier decided the label and whether it may take part in a drop; section_label_evidence holds every tier's vote. route and route_reason record what routing decided and why.

Source code in ontocast/tool/chunk/prepare.py
@dataclass(frozen=True)
class PreparedChunk:
    """A prepared text chunk with optional structural metadata, label and route.

    ``section_label_source`` and ``section_label_strength`` record which tier
    decided the label and whether it may take part in a drop;
    ``section_label_evidence`` holds every tier's vote. ``route`` and
    ``route_reason`` record what routing decided and why.
    """

    text: str
    headings: list[str] | None
    #: Docling items whose blocks the unit holds whole, set after trimming.
    doc_item_refs: tuple[str, ...] = ()
    section_label: str | None = None
    section_label_source: SectionLabelSource | None = None
    section_label_strength: LabelStrength | None = None
    section_label_evidence: tuple[LabelEvidence, ...] = ()
    route: UnitRoute = UnitRoute.EXTRACT
    route_reason: RouteReason = RouteReason.CONTENT

Attributes

doc_item_refs = () class-attribute instance-attribute
headings instance-attribute
route = UnitRoute.EXTRACT class-attribute instance-attribute
route_reason = RouteReason.CONTENT class-attribute instance-attribute
section_label = None class-attribute instance-attribute
section_label_evidence = () class-attribute instance-attribute
section_label_source = None class-attribute instance-attribute
section_label_strength = None class-attribute instance-attribute
text instance-attribute

Methods:

__init__(text, headings, doc_item_refs=(), section_label=None, section_label_source=None, section_label_strength=None, section_label_evidence=(), route=UnitRoute.EXTRACT, route_reason=RouteReason.CONTENT)

SectionSelectionEmptyError

Bases: ValueError

A section selection matched no segment in this document.

Distinct from a malformed parameter: target_sections=["reslts"] is syntactically fine and only turns out to be wrong once this document has been classified. Under the default CHUNK_SECTION_FILTER_ON_EMPTY=warn this is a log line and the run continues to an empty graph, which reads exactly like a document that genuinely had nothing to extract -- telling those two apart is the whole point of the error mode.

Subclasses :class:ValueError so existing parameter guards keep their shape. Deliberately not an api-layer error: nothing under tool/ imports ontocast.api, and the parameter here is well-formed.

Source code in ontocast/tool/chunk/prepare.py
class SectionSelectionEmptyError(ValueError):
    """A section selection matched no segment in this document.

    Distinct from a malformed parameter: ``target_sections=["reslts"]`` is
    syntactically fine and only turns out to be wrong once *this* document has
    been classified. Under the default ``CHUNK_SECTION_FILTER_ON_EMPTY=warn``
    this is a log line and the run continues to an empty graph, which reads
    exactly like a document that genuinely had nothing to extract -- telling
    those two apart is the whole point of the ``error`` mode.

    Subclasses :class:`ValueError` so existing parameter guards keep their
    shape. Deliberately *not* an ``api``-layer error: nothing under ``tool/``
    imports ``ontocast.api``, and the parameter here is well-formed.
    """

    def __init__(self, param: str, message: str) -> None:
        super().__init__(message)
        self.param = param

Attributes

param = param instance-attribute

Methods:

__init__(param, message)
Source code in ontocast/tool/chunk/prepare.py
def __init__(self, param: str, message: str) -> None:
    super().__init__(message)
    self.param = param

Functions:

prepare_content_units(docling_doc, splitter, config, options, tools=None) async

Segment, tag, select, size, trim and route document text into units.

Section tagging is default-on: the sections-first flow runs unless CHUNK_SECTION_CLASSIFIER=off (which leaves only the content rules of routing; explicit section options are ignored with a warning in that case).

Parameters:

Name Type Description Default
docling_doc DoclingDocument

Converted source document.

required
splitter ChunkerTool

Chunker used to size oversized sections.

required
config ChunkConfig

Chunk configuration, including the classifier tier.

required
options PrepareOptions

Per-request section schema and filters.

required
tools 'ToolBox | None'

ToolBox providing the LLM. Required only when config.section_classifier == "llm"; the deterministic tiers need no LLM, so callers that only inspect sections may omit it.

None

Returns:

Type Description
PreparedDocument

The kept and dropped units, each with its route and reason, and how the

PreparedDocument

schema was chosen.

Raises:

Type Description
ValueError

section_classifier is "llm" but no ToolBox was supplied.

SectionSelectionEmptyError

target_sections or the exclusions removed every unit and CHUNK_SECTION_FILTER_ON_EMPTY=error.

Source code in ontocast/tool/chunk/prepare.py
async def prepare_content_units(
    docling_doc: DoclingDocument,
    splitter: ChunkerTool,
    config: ChunkConfig,
    options: PrepareOptions,
    tools: "ToolBox | None" = None,
) -> PreparedDocument:
    """Segment, tag, select, size, trim and route document text into units.

    Section tagging is default-on: the sections-first flow runs unless
    ``CHUNK_SECTION_CLASSIFIER=off`` (which leaves only the content rules of
    routing; explicit section options are ignored with a warning in that
    case).

    Args:
        docling_doc: Converted source document.
        splitter: Chunker used to size oversized sections.
        config: Chunk configuration, including the classifier tier.
        options: Per-request section schema and filters.
        tools: ToolBox providing the LLM. Required only when
            ``config.section_classifier == "llm"``; the deterministic tiers
            need no LLM, so callers that only inspect sections may omit it.

    Returns:
        The kept and dropped units, each with its route and reason, and how the
        schema was chosen.

    Raises:
        ValueError: ``section_classifier`` is ``"llm"`` but no ToolBox was
            supplied.
        SectionSelectionEmptyError: ``target_sections`` or the exclusions
            removed every unit and ``CHUNK_SECTION_FILTER_ON_EMPTY=error``.
    """
    if config.section_classifier == "llm" and tools is None:
        raise ValueError(
            "CHUNK_SECTION_CLASSIFIER=llm requires a ToolBox providing an LLM"
        )
    # Segmentation is CPU-bound and runs local embedding models, but this
    # coroutine is awaited on the event loop -- inline, it would freeze every
    # concurrent document's in-flight provider sockets for its whole duration.
    structure = await asyncio.to_thread(build_document_structure, docling_doc)
    document_text = structure.text
    block_kinds = structure.block_kinds

    if config.section_classifier == "off":
        if options.needs_section_prepare():
            logger.warning(
                "Section options requested but CHUNK_SECTION_CLASSIFIER=off; "
                "section filters are ignored"
            )
        chunks = await asyncio.to_thread(
            _simple_prepare, document_text, splitter, config
        )
        routed = _route_chunks(
            _trim_citation_runs(chunks, None, block_kinds),
            schema=None,
            schema_fallback=True,
            exclude_sections=None,
            config=config,
            block_kinds=block_kinds,
        )
        return PreparedDocument(routed=routed)

    decision = await asyncio.to_thread(
        resolve_prepare_schema, document_text, config, options, splitter
    )
    schema = decision.schema
    spans = await asyncio.to_thread(
        detect_section_spans,
        document_text,
        schema,
        include_text_headings=config.section_text_headings,
        embed=heading_embedder(splitter, config),
    )

    segments = await asyncio.to_thread(
        _semantic_section_segments, document_text, spans, splitter, config.max_size
    )
    if not segments:
        return PreparedDocument(
            routed=[], schema_id=schema.id, schema_source=decision.source
        )

    segments = await asyncio.to_thread(
        coalesce_small_segments_right,
        segments,
        config.section_tag_min_chars,
        schema,
    )
    if config.section_classifier in ("heuristic", "llm"):
        await asyncio.to_thread(
            _add_content_evidence, segments, schema, config, block_kinds
        )
    outline_labels: OutlineLabelReport | None = None
    if config.section_classifier == "llm" and tools is not None:
        outline_labels = await llm_label_outline(
            segments,
            tools,
            # The resolved schema is threaded, not re-derived: re-resolving from
            # the raw request would ignore a text-based detection and validate
            # LLM labels against a different schema, silently dropping them.
            schema,
            document_type_hint=options.document_type_hint,
            window=config.section_llm_batch_size,
        )
    segments = _attach_bare_headings(segments)

    unlabeled = sum(1 for s in segments if s.section_label is None)
    if unlabeled:
        # Normal for text with no headings (plain-text and JSON inputs); the
        # manifest's section_label_histogram is the record.
        logger.info(
            "%s segment(s) remain without section_label after classification",
            unlabeled,
        )

    on_empty = config.section_filter_on_empty

    excluded_chars = 0
    excluded_unlabeled_chars = 0
    allowlist = options.filter_allowlist()
    if allowlist is not None:
        before = len(segments)
        kept = _filter_segments(segments, allowlist)
        excluded_chars, excluded_unlabeled_chars = _allowlist_report(
            segments, kept, allowlist
        )
        segments = kept
        _guard_empty_selection(
            param="target_sections",
            selection=allowlist,
            before=before,
            after=len(segments),
            on_empty=on_empty,
        )

    chunks = await asyncio.to_thread(_size_segments, segments, splitter, config)
    chunks = _trim_citation_runs(chunks, schema, block_kinds)
    routed = _route_chunks(
        chunks,
        schema=schema,
        schema_fallback=decision.source == "default",
        exclude_sections=options.exclude_sections,
        config=config,
        block_kinds=block_kinds,
    )
    document = PreparedDocument(
        routed=routed,
        schema_id=schema.id,
        schema_source=decision.source,
        allowlist_excluded_chars=excluded_chars,
        allowlist_excluded_unlabeled_chars=excluded_unlabeled_chars,
        outline_labels=outline_labels,
    )
    _guard_excluded_everything(document, options, schema, on_empty)
    return document

size_bounded_text(text, config, split_fn, *, separator=DEFAULT_PART_SEPARATOR)

Split text when needed, then enforce OntoCast chunk size bounds.

Source code in ontocast/tool/chunk/sizing.py
def size_bounded_text(
    text: str,
    config: ChunkConfig,
    split_fn: Callable[[str], list[str]],
    *,
    separator: str = DEFAULT_PART_SEPARATOR,
) -> list[str]:
    """Split ``text`` when needed, then enforce OntoCast chunk size bounds."""
    text = text.strip()
    if not text:
        return []

    if len(text) > config.max_size:
        parts = [part.strip() for part in split_fn(text) if part.strip()]
        if not parts:
            parts = [text]
    else:
        parts = [text]

    return size_text_parts(
        parts,
        config.min_size,
        config.max_size,
        separator=separator,
    )