@click.command("sections")
@click.option(
"--input-path",
type=click.Path(exists=True, path_type=pathlib.Path),
required=True,
help="Document to inspect (PDF, PowerPoint, markdown, text or JSON).",
)
@click.option(
"--section-schema-id",
default=None,
help="Section label schema: academic, financial, legal, clinical, manual, "
"fiction, patent, standard, news, general. Omit to detect it.",
)
@click.option(
"--document-type-hint",
default=None,
help="Free-text document type; overrides detection when it matches.",
)
@click.option(
"--target-sections",
default=None,
help="Comma-separated labels to keep, to preview what a filtered run selects.",
)
@click.option(
"--exclude-sections",
default=None,
help="Comma-separated labels to drop; empty string disables exclusion.",
)
@click.option(
"--section-classifier",
type=click.Choice(["off", "heading", "heuristic", "llm"]),
default=None,
help="Override CHUNK_SECTION_CLASSIFIER. Only 'llm' makes LLM calls.",
)
@click.option("--as-json", "as_json", is_flag=True, help="Emit JSON instead of tables.")
def main(
input_path: pathlib.Path,
section_schema_id: str | None,
document_type_hint: str | None,
target_sections: str | None,
exclude_sections: str | None,
section_classifier: str | None,
as_json: bool,
) -> None:
"""Show the detected section outline and per-chunk labels for a document."""
config = Config()
tool_config = config.get_tool_config()
chunk_config = tool_config.chunk_config
if section_classifier is not None:
chunk_config.section_classifier = cast(
Literal["llm", "heuristic", "heading", "off"], section_classifier
)
# A diagnostic must survive the condition it diagnoses: the empty-selection
# gate is a pipeline policy, and "0 chunks kept" is exactly the answer this
# command exists to show.
chunk_config.section_filter_on_empty = "warn"
# Only the 'llm' tier needs an LLM, and building a full ToolBox requires
# provider credentials. Inspecting a document must stay free, so the
# deterministic tiers get just the converter and chunker.
cache = Cacher(config=config)
converter = ConverterTool(
cache=cache, converter_config=tool_config.converter_config
)
chunker = ChunkerTool(chunk_config=chunk_config, cache=cache)
tools = ToolBox(config) if chunk_config.section_classifier == "llm" else None
docling_doc = _load_document(input_path, converter)
document_text = document_text_for_section_tagging(docling_doc)
hint = parse_document_type_hint_param(document_type_hint)
schema_id = parse_section_schema_id_param(section_schema_id)
# The parsers are shared with the HTTP layer and signal bad input by
# raising; surface that as a click usage error rather than a traceback.
try:
options = PrepareOptions(
section_schema_id=schema_id,
document_type_hint=hint,
# None and [] mean different things downstream (schema defaults
# versus exclusion disabled), so an omitted flag must stay None.
target_sections=(
parse_sections_list_param(target_sections, param="target-sections")
if target_sections is not None
else None
),
exclude_sections=(
parse_sections_list_param(exclude_sections, param="exclude-sections")
if exclude_sections is not None
else None
),
)
except ValueError as exc:
raise click.BadParameter(str(exc)) from exc
# Resolved through the same helper the pipeline uses, so the schema reported
# here is necessarily the schema the chunks were actually labeled against.
decision = resolve_prepare_schema(document_text, chunk_config, options, chunker)
schema = decision.schema
resolved_id = decision.schema_id
prepared = asyncio.run(
prepare_content_units(docling_doc, chunker, chunk_config, options, tools)
)
outline = _outline_rows(
document_text,
schema,
chunk_config.section_text_headings,
heading_embedder(chunker, chunk_config),
)
chunk_rows = _chunk_rows(prepared.routed)
labeled = sum(1 for row in chunk_rows if row["label"] is not None)
dropped_chars: dict[str, int] = {}
for unit in prepared.dropped:
reason = unit.route_reason.value
dropped_chars[reason] = dropped_chars.get(reason, 0) + len(unit.text)
schema_rows = [
{
"candidate": ev.schema_id,
"score": round(ev.score, 1),
"share": f"{ev.share:.0%}",
"evidence": ", ".join(ev.examples[:3]),
}
for ev in (decision.detection.evidence[:5] if decision.detection else [])
if ev.score > 0
]
summary = {
"input": str(input_path),
"schema": resolved_id,
"schema_source": decision.source,
"schema_tier": decision.detection.tier if decision.detection else None,
"schema_margin": (
round(decision.detection.margin, 2) if decision.detection else None
),
"classifier": chunk_config.section_classifier,
"density": chunk_config.section_density,
"document_chars": len(document_text),
"headings": len(outline),
"chunks": len(chunk_rows),
"labeled_chunks": labeled,
"kept_chunks": len(prepared.units),
"dropped_chars_by_reason": dropped_chars,
"allowlist_excluded_chars": prepared.allowlist_excluded_chars,
"allowlist_excluded_unlabeled_chars": (
prepared.allowlist_excluded_unlabeled_chars
),
"outline_labels": (
prepared.outline_labels.model_dump()
if prepared.outline_labels is not None
else None
),
}
if as_json:
click.echo(
json.dumps(
{
"summary": summary,
"schema_candidates": schema_rows,
"outline": outline,
"chunks": chunk_rows,
},
indent=2,
)
)
return
provenance = decision.source
if decision.detection is not None:
margin = decision.detection.margin
# An infinite margin means no other schema scored at all -- "infx" reads
# as a formatting bug rather than as the strongest possible evidence.
margin_text = (
"uncontested" if margin == float("inf") else f"{margin:.1f}x margin"
)
provenance = (
f"{decision.source} via {decision.detection.tier} tier, {margin_text}"
)
click.echo(click.style(f"{input_path}", bold=True))
click.echo(
f" schema={resolved_id} ({provenance}) "
f"classifier={chunk_config.section_classifier} "
f"density={chunk_config.section_density} chars={len(document_text)}"
)
if schema_rows:
_print_table(
"Schema candidates",
schema_rows,
[("candidate", 12), ("score", 6), ("share", 6), ("evidence", 60)],
)
_print_table(
"Outline",
outline,
[
("offset", 7),
("number", 6),
("parent", 6),
("kind", 8),
("label", 16),
("source", 18),
("strength", 6),
("heading", 60),
],
)
_print_table(
"Chunks",
chunk_rows,
[
("index", 5),
("chars", 6),
("label", 16),
("source", 18),
("strength", 6),
("route", 11),
("reason", 20),
("items", 5),
("preview", 50),
],
)
_print_table(
"Evidence (label:source:strength per tier that voted)",
[row for row in chunk_rows if row["evidence"]],
[("index", 5), ("label", 16), ("evidence", 100)],
)
click.echo(
f"\n {labeled}/{len(chunk_rows)} chunks labeled "
f"({len(chunk_rows) - labeled} unresolved); "
f"{len(prepared.units)} kept for extraction"
)
if dropped_chars:
click.echo(
" dropped: "
+ ", ".join(
f"{reason} {chars} chars" for reason, chars in dropped_chars.items()
)
)
if options.target_sections is not None:
click.echo(
f" target_sections removed {prepared.allowlist_excluded_chars} chars "
f"({prepared.allowlist_excluded_unlabeled_chars} unlabeled)"
)
report = prepared.outline_labels
if report is not None:
line = (
f" model tier: {report.windows} window(s), {report.windows_failed} failed"
)
if report.errors:
line += (
" (" + ", ".join(f"{k} x{v}" for k, v in report.errors.items()) + ")"
)
click.echo(line)