Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion mineru/doclib/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ async def startup() -> None:
from .background.watch import WatchLoop
from .core.db import DatabaseManager
from .core.fts import FTSManager
from .core.fts_content import migrate_fts_contents
from .services.cleanup_svc import CleanupService
from .services.config_svc import ConfigService
from .services.ingest_svc import IngestService
Expand All @@ -90,6 +91,7 @@ async def startup() -> None:
state.db_ready.set()

state.fts = FTSManager(state.db)
await migrate_fts_contents(state.db, state.fts, data_dir=data_dir, latex_delimiters=cfg.render.latex_delimiters)
state.config_svc = ConfigService(state.db)
state.telemetry_svc = TelemetryService(TelemetryStore(state.db))
await state.telemetry_svc.initialize()
Expand All @@ -107,7 +109,13 @@ async def startup() -> None:
telemetry_svc=state.telemetry_svc,
)
state.parse_execution_svc = ParseExecutionService(
state.db, state.fts, state.config_svc, data_dir, health=state.health, telemetry_svc=state.telemetry_svc
state.db,
state.fts,
state.config_svc,
data_dir,
health=state.health,
latex_delimiters=cfg.render.latex_delimiters,
telemetry_svc=state.telemetry_svc,
)
state.scan_svc = ScanService(
state.db,
Expand Down Expand Up @@ -404,6 +412,7 @@ def __init__(self) -> None:
"exclude_rules",
"files",
"fts_contents",
"fts_rebuild_queue",
"fts_filenames",
"parses",
"parsing_rules",
Expand Down
2 changes: 1 addition & 1 deletion mineru/doclib/core/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from ...errors import ServerBusyError
from ..config_defaults import CONFIG_DEFAULTS

SCHEMA_VERSION = 2
SCHEMA_VERSION = 3
_MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations"

_T = TypeVar("_T")
Expand Down
10 changes: 9 additions & 1 deletion mineru/doclib/core/fts.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ async def replace(
"INSERT INTO fts_contents (sha256, tier, text, title, author) VALUES (?, ?, ?, ?, ?)",
(sha256, tier, tokenized, title or "", author or ""),
),
("DELETE FROM fts_rebuild_queue WHERE sha256=?", (sha256,)),
]
)

Expand All @@ -82,6 +83,7 @@ async def search(self, query: str, limit: int = 200) -> list[FtsContentSearchRow
"SELECT sha256, title, author, tier, "
"snippet(fts_contents, 2, '<mark>', '</mark>', '...', 40) AS snippet, rank "
"FROM fts_contents WHERE fts_contents MATCH ? "
"AND sha256 NOT IN (SELECT sha256 FROM fts_rebuild_queue) "
"ORDER BY rank LIMIT ?",
(fts_query, limit),
),
Expand All @@ -94,7 +96,13 @@ async def get_tier(self, sha256: str) -> Tier | None:
return cast(Tier | None, row["tier"]) if row else None

async def delete(self, sha256: str) -> None:
await self.db.execute("DELETE FROM fts_contents WHERE sha256=?", (sha256,))
"""同时移除正文索引和待迁移标记,防止已失效文档被重新索引。"""
await self.db.execute_atomic(
[
("DELETE FROM fts_contents WHERE sha256=?", (sha256,)),
("DELETE FROM fts_rebuild_queue WHERE sha256=?", (sha256,)),
]
)

# ── fts_filenames ────────────────────────────────────────────

Expand Down
111 changes: 111 additions & 0 deletions mineru/doclib/core/fts_content.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
"""Doclib 检索文本构造与缓存重建,不把展示用图片载荷写入索引。"""

from __future__ import annotations

import logging
from typing import cast

from docvortex.options import LatexDelimitersConfig

from ...errors import MineruError
from ...parser.page_range import parse_page_range_set
from ...types import TIER_ORDER, BlockBase, MiddleJson, Tier
from ..rows import ParseRow
from ..types import PARSE_STATUS_DONE
from .db import DatabaseManager
from .fts import FTSManager, truncate_head_tail
from .middle_json import read_cached_middle_json
from .parse_artifacts import load_pages_from_done_batches, parse_batch_is_readable, parse_batch_json_path

logger = logging.getLogger("mineru.doclib.search_index")


def _omit_image(_block: BlockBase) -> str:
"""索引只保留视觉块的文字内容,图片通过 Doclib read 按需读取。"""
return ""


def render_fts_text(middle_json: MiddleJson, *, latex_delimiters: LatexDelimitersConfig) -> str:
"""沿用正文渲染和跨页续接规则,抑制图片后才应用索引字符预算。"""
from docvortex.render.markdown import render_markdown

return truncate_head_tail(render_markdown(middle_json, latex_delimiters=latex_delimiters, image_renderer=_omit_image))


async def rebuild_fts_from_cache(
db: DatabaseManager,
fts: FTSManager,
*,
data_dir: str,
sha256: str,
latex_delimiters: LatexDelimitersConfig,
require_complete_cache: bool = False,
) -> None:
"""按最高可读档位重建索引;升级迁移要求完整缓存,且不依赖源文件仍然存在。"""
doc = await db.fetchone("SELECT title, author FROM docs WHERE sha256=?", (sha256,))
rows = cast(
list[ParseRow],
await db.fetchall(
"SELECT * FROM parses WHERE sha256=? AND status=? ORDER BY done_at DESC, id DESC",
(sha256, PARSE_STATUS_DONE),
),
)
tiers: list[Tier] = sorted({row["tier"] for row in rows}, key=lambda tier: TIER_ORDER[tier], reverse=True)
for tier in tiers:
tier_rows = [row for row in rows if row["tier"] == tier]
readable = [row for row in tier_rows if parse_batch_is_readable(data_dir, sha256, tier, row)]
pages = load_pages_from_done_batches(data_dir, sha256, tier, readable)
if require_complete_cache:
expected = set().union(*(parse_page_range_set(row["page_range"]) for row in tier_rows))
if not pages or expected - {page.page_idx + 1 for page in pages}:
raise MineruError("stale_cache", "Search index rebuild requires readable cached parse pages.")
if not pages:
continue
# 保留真实批次的文档身份与元数据,用按最新记录合并的页树进行整本文本渲染。
newest = readable[0]
template = read_cached_middle_json(
parse_batch_json_path(data_dir, sha256, tier, newest["page_range"], newest["done_at"])
)
middle_json = template.model_copy(update={"pages": pages})
await fts.replace(
sha256=sha256,
tier=tier,
text=render_fts_text(middle_json, latex_delimiters=latex_delimiters),
title=(doc or {}).get("title") or "",
author=(doc or {}).get("author") or "",
)
return
if require_complete_cache:
raise MineruError("stale_cache", "Search index rebuild requires a completed parse cache.")
await fts.delete(sha256)


async def migrate_fts_contents(
db: DatabaseManager, fts: FTSManager, *, data_dir: str, latex_delimiters: LatexDelimitersConfig
) -> None:
"""启动接单前分批迁移旧索引;成功记录随替换原子清除,失败保留原因供下次重试。"""
cursor = ""
while rows := await db.fetchall("SELECT sha256 FROM fts_rebuild_queue WHERE sha256>? ORDER BY sha256 LIMIT 100", (cursor,)):
for row in rows:
sha256 = row["sha256"]
try:
await rebuild_fts_from_cache(
db,
fts,
data_dir=data_dir,
sha256=sha256,
latex_delimiters=latex_delimiters,
require_complete_cache=True,
)
except (OSError, ValueError, TypeError, MineruError) as exc:
await db.execute("UPDATE fts_rebuild_queue SET error=? WHERE sha256=?", (str(exc)[:500], sha256))
logger.warning(
"Search index rebuild deferred for doc=%s: %s. The old index is excluded from search; "
"restore or reparse its cache and restart Doclib to retry.",
sha256,
exc,
)
cursor = rows[-1]["sha256"]


__all__ = ["render_fts_text", "rebuild_fts_from_cache", "migrate_fts_contents"]
10 changes: 10 additions & 0 deletions mineru/doclib/migrations/003_rebuild_search_text.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
-- v003: Rebuild parsed-document indexes using image-free search text.
-- Pending rows hide old payload-bearing indexes until replacement succeeds.
CREATE TABLE IF NOT EXISTS fts_rebuild_queue (
sha256 TEXT PRIMARY KEY NOT NULL REFERENCES docs(sha256) ON DELETE CASCADE,
error TEXT
);

INSERT OR IGNORE INTO fts_rebuild_queue (sha256)
SELECT f.sha256 FROM fts_contents AS f JOIN docs AS d ON d.sha256=f.sha256
WHERE f.tier IS NOT NULL;
15 changes: 8 additions & 7 deletions mineru/doclib/services/parse_execution_svc.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
from typing import TYPE_CHECKING, cast
from urllib.parse import urlparse

from docvortex.options import LatexDelimitersConfig

from ...filetypes import (
OFD_EXTENSIONS,
OFFICE_EXTENSIONS,
Expand All @@ -25,13 +27,13 @@
parse_page_range_set,
)
from ...types import TIER_ORDER, Tier
from ..core import fts as fts_utils
from ..core.db import DatabaseManager
from ..parse_server_state import ParseServerHealth
from ..core.file_io import (
metadata_to_doclib,
)
from ..core.fts import FTSManager
from ..core.fts_content import render_fts_text
from ..core.parse_artifacts import (
accessible_file_for_sha256,
parse_artifact_directory,
Expand Down Expand Up @@ -72,10 +74,12 @@ def __init__(
data_dir: str,
*,
health: ParseServerHealth,
latex_delimiters: LatexDelimitersConfig,
telemetry_svc: TelemetryService | None = None,
) -> None:
"""保存此服务所需的显式依赖,不隐式组装另一侧服务。"""
self.health = health
self.latex_delimiters = latex_delimiters.model_copy(deep=True)
self.db = db
self.fts = fts
self.config_svc = config_svc
Expand Down Expand Up @@ -231,10 +235,8 @@ async def process_doc(self, task: ParseRow) -> bool:
return False

try:
# generate markdown for FTS (not persisted, only for search indexing)
md_text = ""
md = result.markdown(add_markers=False) if hasattr(result, "markdown") else ""
md_text += md + "\n"
# 新解析与缓存重建使用相同的检索表示,在截断前排除图片载荷。
md_text = render_fts_text(result.middle_json, latex_delimiters=self.latex_delimiters)

# 原文件属性不受推理档位限制,先更新数据库和已有索引,再处理正文索引。
await self._update_source_metadata(sha256, result)
Expand Down Expand Up @@ -473,12 +475,11 @@ async def _record_write(
await self._record_duration("parse_task.write_duration_bucket.count", start_ms, dimensions=dimensions)

async def _maybe_update_fts(self, sha256: str, tier: Tier, text: str, file_row: FileRow) -> None:
"""仅以相同或更高档位的正文更新索引,并保留首尾截断。"""
"""只用更高档位替换已有索引;调用方传入已按统一预算处理的检索文本。"""
existing_tier = await self.fts.get_tier(sha256)
if existing_tier and TIER_ORDER.get(existing_tier, -1) >= TIER_ORDER.get(tier, -1):
return # current FTS data is from a higher or equal tier

text = fts_utils.truncate_head_tail(text)
await self.fts.replace(
sha256=sha256,
tier=tier,
Expand Down
65 changes: 5 additions & 60 deletions mineru/doclib/services/parse_queue_svc.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,17 @@
from ...parser.page_range import (
format_page_range as _page_numbers_to_range_str,
)
from ...types import QUALITY_TIERS, TIER_ORDER, Tier
from ..core import fts as fts_utils
from ...types import QUALITY_TIERS, Tier
from ..core.db import DatabaseManager
from ..parse_server_state import ParseServerHealth
from ..core.fts import FTSManager
from ..core.fts_content import rebuild_fts_from_cache
from ..core.parse_artifacts import (
load_pages_from_done_batches,
parse_batch_is_readable,
parse_coverage,
)
from ..rows import DocRow, FileRow, ParseRow
from ..types import (
FILE_STATUS_ACTIVE,
PARSE_STATUS_DONE,
PARSE_STATUS_PARSING,
PARSE_STATUS_PENDING,
Expand Down Expand Up @@ -348,63 +346,10 @@ async def invalidate(self, sha256: str, tier: Tier | None = None) -> int:
return cursor.rowcount

async def _rebuild_fts_after_invalidate(self, sha256: str) -> None:
# FTS rebuild reads only cached parse batches, so any active row works;
# keep it deterministic and tolerate rows whose file moved away.
"""按档位和缓存可读性重建正文索引,不修改历史缓存或源文件。"""
file_row = cast(
FileRow | None,
await self.db.fetchone(
"SELECT * FROM files WHERE sha256=? AND status=? ORDER BY updated_at DESC",
(sha256, FILE_STATUS_ACTIVE),
),
"""按共享规则从剩余缓存重建正文索引,不依赖源文件仍然存在。"""
await rebuild_fts_from_cache(
self.db, self.fts, data_dir=self.data_dir, sha256=sha256, latex_delimiters=self.latex_delimiters
)
if file_row is None:
await self.fts.delete(sha256)
return
doc_row = cast(DocRow | None, await self.db.fetchone("SELECT title, author FROM docs WHERE sha256=?", (sha256,)))

rows = cast(
list[ParseRow],
await self.db.fetchall(
"SELECT * FROM parses WHERE sha256=? AND status=? ORDER BY done_at DESC",
(sha256, PARSE_STATUS_DONE),
),
)
tier_set: set[Tier] = {row["tier"] for row in rows}
tiers = sorted(tier_set, key=lambda item: TIER_ORDER.get(item, -1), reverse=True)
for tier in tiers:
tier_rows = [
row for row in rows if row["tier"] == tier and parse_batch_is_readable(self.data_dir, sha256, tier, row)
]
pages = load_pages_from_done_batches(self.data_dir, sha256, tier, tier_rows)
if not pages:
continue

from docvortex.render.markdown import render_single_block

delimiters = self.latex_delimiters
parts: list[str] = []
for page in pages:
for block in page.blocks:
rendered = render_single_block(
block,
delimiters=delimiters,
asset_base_url="",
image_renderer=lambda _block: "",
)
if rendered and rendered.strip():
parts.append(rendered.strip())
text = fts_utils.truncate_head_tail("\n\n".join(parts))
await self.fts.replace(
sha256=sha256,
tier=tier,
text=text,
title=(doc_row or {}).get("title") or "",
author=(doc_row or {}).get("author") or "",
)
return

await self.fts.delete(sha256)

async def _record_count(self, metric_name: str, *, value: int = 1, dimensions: dict[str, str] | None = None) -> None:
"""通过显式指标接口记录计数,未配置时跳过。"""
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ classifiers = [
"Programming Language :: Python :: 3.14",
]
dependencies = [
"docvortex>=0.6.21,<0.7",
"docvortex>=0.6.24,<0.7",
"gradio>=6.21,<7",
"mineru[torch] ; sys_platform == 'darwin' and platform_machine == 'arm64'",
"mineru-vl-utils>=2.0.8,<3",
Expand Down
9 changes: 9 additions & 0 deletions tests/fixtures/issue5776/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Issue #5776 搜索图片载荷回归样本

来源:https://github.com/opendatalab/MinerU/issues/5776

原附件:https://github.com/user-attachments/files/34912771/1.pdf

SHA-256:`ea5d3ef8fc5499e8ae846caaf2dac264a59b248e658d5576bd2c2332e9753aff`

一页 PDF,包含“测试”文字与图片。自动化测试使用 Flash 原生文本模式验证真实图片载荷不会进入搜索索引,无需下载 OCR 模型;完整 auto 路径另作真实模型烟测。
Binary file added tests/fixtures/issue5776/source.pdf
Binary file not shown.
10 changes: 9 additions & 1 deletion tests/unittest/doclib_test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,15 @@ def create_parse_services(
parse_lock_timeout_sec=parse_lock_timeout_sec,
telemetry_svc=telemetry_svc,
)
execution = execution_type(db, fts, config_svc, data_dir, health=health, telemetry_svc=telemetry_svc)
execution = execution_type(
db,
fts,
config_svc,
data_dir,
health=health,
latex_delimiters=config.render.latex_delimiters,
telemetry_svc=telemetry_svc,
)
return ParseServices(ingest, queue, execution, health, config)


Expand Down
2 changes: 1 addition & 1 deletion tests/unittest/test_doclib_migrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ async def _run() -> None:
await db.initialize()

rows = await db.fetchall("SELECT version FROM _migrations ORDER BY version")
assert rows == [{"version": 1}, {"version": 2}]
assert rows == [{"version": 1}, {"version": 2}, {"version": 3}]

asyncio.run(_run())

Expand Down
Loading
Loading