From 97178455c9955c3b225d65b32d9f60322887f372 Mon Sep 17 00:00:00 2001 From: myhloli Date: Mon, 16 Nov 2026 20:55:40 +0800 Subject: [PATCH] fix(doclib): keep image payloads out of search indexes Unify image-free index rendering, retain document continuation rules, and migrate legacy indexes from cached parse results with atomic progress and retryable failures. Require DocVortex 0.6.24 and cover the original issue attachment, migration, and search output regressions. Refs opendatalab/MinerU#5776 --- mineru/doclib/app.py | 11 +- mineru/doclib/core/db.py | 2 +- mineru/doclib/core/fts.py | 10 +- mineru/doclib/core/fts_content.py | 111 +++++++ .../migrations/003_rebuild_search_text.sql | 10 + mineru/doclib/services/parse_execution_svc.py | 15 +- mineru/doclib/services/parse_queue_svc.py | 65 +--- pyproject.toml | 2 +- tests/fixtures/issue5776/README.md | 9 + tests/fixtures/issue5776/source.pdf | Bin 0 -> 27118 bytes tests/unittest/doclib_test_utils.py | 10 +- tests/unittest/test_doclib_migrations.py | 2 +- tests/unittest/test_doclib_parse_services.py | 4 +- tests/unittest/test_doclib_search_index.py | 307 ++++++++++++++++++ tests/unittest/test_doclib_telemetry_core.py | 31 +- 15 files changed, 496 insertions(+), 93 deletions(-) create mode 100644 mineru/doclib/core/fts_content.py create mode 100644 mineru/doclib/migrations/003_rebuild_search_text.sql create mode 100644 tests/fixtures/issue5776/README.md create mode 100644 tests/fixtures/issue5776/source.pdf create mode 100644 tests/unittest/test_doclib_search_index.py diff --git a/mineru/doclib/app.py b/mineru/doclib/app.py index f942966842..06ada0f50c 100644 --- a/mineru/doclib/app.py +++ b/mineru/doclib/app.py @@ -69,6 +69,7 @@ async def startup() -> None: from .background.watch import WatchLoop from .core.db import DatabaseManager from .core.fts import FTSManager + from .core.fts_content import migrate_fts_contents from .services.cleanup_svc import CleanupService from .services.config_svc import ConfigService from .services.ingest_svc import IngestService @@ -90,6 +91,7 @@ async def startup() -> None: state.db_ready.set() state.fts = FTSManager(state.db) + await migrate_fts_contents(state.db, state.fts, data_dir=data_dir, latex_delimiters=cfg.render.latex_delimiters) state.config_svc = ConfigService(state.db) state.telemetry_svc = TelemetryService(TelemetryStore(state.db)) await state.telemetry_svc.initialize() @@ -107,7 +109,13 @@ async def startup() -> None: telemetry_svc=state.telemetry_svc, ) state.parse_execution_svc = ParseExecutionService( - state.db, state.fts, state.config_svc, data_dir, health=state.health, telemetry_svc=state.telemetry_svc + state.db, + state.fts, + state.config_svc, + data_dir, + health=state.health, + latex_delimiters=cfg.render.latex_delimiters, + telemetry_svc=state.telemetry_svc, ) state.scan_svc = ScanService( state.db, @@ -404,6 +412,7 @@ def __init__(self) -> None: "exclude_rules", "files", "fts_contents", + "fts_rebuild_queue", "fts_filenames", "parses", "parsing_rules", diff --git a/mineru/doclib/core/db.py b/mineru/doclib/core/db.py index 9662947b5b..a9b64e5da1 100644 --- a/mineru/doclib/core/db.py +++ b/mineru/doclib/core/db.py @@ -14,7 +14,7 @@ from ...errors import ServerBusyError from ..config_defaults import CONFIG_DEFAULTS -SCHEMA_VERSION = 2 +SCHEMA_VERSION = 3 _MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" _T = TypeVar("_T") diff --git a/mineru/doclib/core/fts.py b/mineru/doclib/core/fts.py index 86a45b5a22..a12c43fea7 100644 --- a/mineru/doclib/core/fts.py +++ b/mineru/doclib/core/fts.py @@ -67,6 +67,7 @@ async def replace( "INSERT INTO fts_contents (sha256, tier, text, title, author) VALUES (?, ?, ?, ?, ?)", (sha256, tier, tokenized, title or "", author or ""), ), + ("DELETE FROM fts_rebuild_queue WHERE sha256=?", (sha256,)), ] ) @@ -82,6 +83,7 @@ async def search(self, query: str, limit: int = 200) -> list[FtsContentSearchRow "SELECT sha256, title, author, tier, " "snippet(fts_contents, 2, '', '', '...', 40) AS snippet, rank " "FROM fts_contents WHERE fts_contents MATCH ? " + "AND sha256 NOT IN (SELECT sha256 FROM fts_rebuild_queue) " "ORDER BY rank LIMIT ?", (fts_query, limit), ), @@ -94,7 +96,13 @@ async def get_tier(self, sha256: str) -> Tier | None: return cast(Tier | None, row["tier"]) if row else None async def delete(self, sha256: str) -> None: - await self.db.execute("DELETE FROM fts_contents WHERE sha256=?", (sha256,)) + """同时移除正文索引和待迁移标记,防止已失效文档被重新索引。""" + await self.db.execute_atomic( + [ + ("DELETE FROM fts_contents WHERE sha256=?", (sha256,)), + ("DELETE FROM fts_rebuild_queue WHERE sha256=?", (sha256,)), + ] + ) # ── fts_filenames ──────────────────────────────────────────── diff --git a/mineru/doclib/core/fts_content.py b/mineru/doclib/core/fts_content.py new file mode 100644 index 0000000000..9653806306 --- /dev/null +++ b/mineru/doclib/core/fts_content.py @@ -0,0 +1,111 @@ +"""Doclib 检索文本构造与缓存重建,不把展示用图片载荷写入索引。""" + +from __future__ import annotations + +import logging +from typing import cast + +from docvortex.options import LatexDelimitersConfig + +from ...errors import MineruError +from ...parser.page_range import parse_page_range_set +from ...types import TIER_ORDER, BlockBase, MiddleJson, Tier +from ..rows import ParseRow +from ..types import PARSE_STATUS_DONE +from .db import DatabaseManager +from .fts import FTSManager, truncate_head_tail +from .middle_json import read_cached_middle_json +from .parse_artifacts import load_pages_from_done_batches, parse_batch_is_readable, parse_batch_json_path + +logger = logging.getLogger("mineru.doclib.search_index") + + +def _omit_image(_block: BlockBase) -> str: + """索引只保留视觉块的文字内容,图片通过 Doclib read 按需读取。""" + return "" + + +def render_fts_text(middle_json: MiddleJson, *, latex_delimiters: LatexDelimitersConfig) -> str: + """沿用正文渲染和跨页续接规则,抑制图片后才应用索引字符预算。""" + from docvortex.render.markdown import render_markdown + + return truncate_head_tail(render_markdown(middle_json, latex_delimiters=latex_delimiters, image_renderer=_omit_image)) + + +async def rebuild_fts_from_cache( + db: DatabaseManager, + fts: FTSManager, + *, + data_dir: str, + sha256: str, + latex_delimiters: LatexDelimitersConfig, + require_complete_cache: bool = False, +) -> None: + """按最高可读档位重建索引;升级迁移要求完整缓存,且不依赖源文件仍然存在。""" + doc = await db.fetchone("SELECT title, author FROM docs WHERE sha256=?", (sha256,)) + rows = cast( + list[ParseRow], + await db.fetchall( + "SELECT * FROM parses WHERE sha256=? AND status=? ORDER BY done_at DESC, id DESC", + (sha256, PARSE_STATUS_DONE), + ), + ) + tiers: list[Tier] = sorted({row["tier"] for row in rows}, key=lambda tier: TIER_ORDER[tier], reverse=True) + for tier in tiers: + tier_rows = [row for row in rows if row["tier"] == tier] + readable = [row for row in tier_rows if parse_batch_is_readable(data_dir, sha256, tier, row)] + pages = load_pages_from_done_batches(data_dir, sha256, tier, readable) + if require_complete_cache: + expected = set().union(*(parse_page_range_set(row["page_range"]) for row in tier_rows)) + if not pages or expected - {page.page_idx + 1 for page in pages}: + raise MineruError("stale_cache", "Search index rebuild requires readable cached parse pages.") + if not pages: + continue + # 保留真实批次的文档身份与元数据,用按最新记录合并的页树进行整本文本渲染。 + newest = readable[0] + template = read_cached_middle_json( + parse_batch_json_path(data_dir, sha256, tier, newest["page_range"], newest["done_at"]) + ) + middle_json = template.model_copy(update={"pages": pages}) + await fts.replace( + sha256=sha256, + tier=tier, + text=render_fts_text(middle_json, latex_delimiters=latex_delimiters), + title=(doc or {}).get("title") or "", + author=(doc or {}).get("author") or "", + ) + return + if require_complete_cache: + raise MineruError("stale_cache", "Search index rebuild requires a completed parse cache.") + await fts.delete(sha256) + + +async def migrate_fts_contents( + db: DatabaseManager, fts: FTSManager, *, data_dir: str, latex_delimiters: LatexDelimitersConfig +) -> None: + """启动接单前分批迁移旧索引;成功记录随替换原子清除,失败保留原因供下次重试。""" + cursor = "" + while rows := await db.fetchall("SELECT sha256 FROM fts_rebuild_queue WHERE sha256>? ORDER BY sha256 LIMIT 100", (cursor,)): + for row in rows: + sha256 = row["sha256"] + try: + await rebuild_fts_from_cache( + db, + fts, + data_dir=data_dir, + sha256=sha256, + latex_delimiters=latex_delimiters, + require_complete_cache=True, + ) + except (OSError, ValueError, TypeError, MineruError) as exc: + await db.execute("UPDATE fts_rebuild_queue SET error=? WHERE sha256=?", (str(exc)[:500], sha256)) + logger.warning( + "Search index rebuild deferred for doc=%s: %s. The old index is excluded from search; " + "restore or reparse its cache and restart Doclib to retry.", + sha256, + exc, + ) + cursor = rows[-1]["sha256"] + + +__all__ = ["render_fts_text", "rebuild_fts_from_cache", "migrate_fts_contents"] diff --git a/mineru/doclib/migrations/003_rebuild_search_text.sql b/mineru/doclib/migrations/003_rebuild_search_text.sql new file mode 100644 index 0000000000..a6d54976f1 --- /dev/null +++ b/mineru/doclib/migrations/003_rebuild_search_text.sql @@ -0,0 +1,10 @@ +-- v003: Rebuild parsed-document indexes using image-free search text. +-- Pending rows hide old payload-bearing indexes until replacement succeeds. +CREATE TABLE IF NOT EXISTS fts_rebuild_queue ( + sha256 TEXT PRIMARY KEY NOT NULL REFERENCES docs(sha256) ON DELETE CASCADE, + error TEXT +); + +INSERT OR IGNORE INTO fts_rebuild_queue (sha256) +SELECT f.sha256 FROM fts_contents AS f JOIN docs AS d ON d.sha256=f.sha256 +WHERE f.tier IS NOT NULL; diff --git a/mineru/doclib/services/parse_execution_svc.py b/mineru/doclib/services/parse_execution_svc.py index 1cef70a007..c84cf28f64 100644 --- a/mineru/doclib/services/parse_execution_svc.py +++ b/mineru/doclib/services/parse_execution_svc.py @@ -10,6 +10,8 @@ from typing import TYPE_CHECKING, cast from urllib.parse import urlparse +from docvortex.options import LatexDelimitersConfig + from ...filetypes import ( OFD_EXTENSIONS, OFFICE_EXTENSIONS, @@ -25,13 +27,13 @@ parse_page_range_set, ) from ...types import TIER_ORDER, Tier -from ..core import fts as fts_utils from ..core.db import DatabaseManager from ..parse_server_state import ParseServerHealth from ..core.file_io import ( metadata_to_doclib, ) from ..core.fts import FTSManager +from ..core.fts_content import render_fts_text from ..core.parse_artifacts import ( accessible_file_for_sha256, parse_artifact_directory, @@ -72,10 +74,12 @@ def __init__( data_dir: str, *, health: ParseServerHealth, + latex_delimiters: LatexDelimitersConfig, telemetry_svc: TelemetryService | None = None, ) -> None: """保存此服务所需的显式依赖,不隐式组装另一侧服务。""" self.health = health + self.latex_delimiters = latex_delimiters.model_copy(deep=True) self.db = db self.fts = fts self.config_svc = config_svc @@ -231,10 +235,8 @@ async def process_doc(self, task: ParseRow) -> bool: return False try: - # generate markdown for FTS (not persisted, only for search indexing) - md_text = "" - md = result.markdown(add_markers=False) if hasattr(result, "markdown") else "" - md_text += md + "\n" + # 新解析与缓存重建使用相同的检索表示,在截断前排除图片载荷。 + md_text = render_fts_text(result.middle_json, latex_delimiters=self.latex_delimiters) # 原文件属性不受推理档位限制,先更新数据库和已有索引,再处理正文索引。 await self._update_source_metadata(sha256, result) @@ -473,12 +475,11 @@ async def _record_write( await self._record_duration("parse_task.write_duration_bucket.count", start_ms, dimensions=dimensions) async def _maybe_update_fts(self, sha256: str, tier: Tier, text: str, file_row: FileRow) -> None: - """仅以相同或更高档位的正文更新索引,并保留首尾截断。""" + """只用更高档位替换已有索引;调用方传入已按统一预算处理的检索文本。""" existing_tier = await self.fts.get_tier(sha256) if existing_tier and TIER_ORDER.get(existing_tier, -1) >= TIER_ORDER.get(tier, -1): return # current FTS data is from a higher or equal tier - text = fts_utils.truncate_head_tail(text) await self.fts.replace( sha256=sha256, tier=tier, diff --git a/mineru/doclib/services/parse_queue_svc.py b/mineru/doclib/services/parse_queue_svc.py index 4a09e2c513..661ddcdcc7 100644 --- a/mineru/doclib/services/parse_queue_svc.py +++ b/mineru/doclib/services/parse_queue_svc.py @@ -24,19 +24,17 @@ from ...parser.page_range import ( format_page_range as _page_numbers_to_range_str, ) -from ...types import QUALITY_TIERS, TIER_ORDER, Tier -from ..core import fts as fts_utils +from ...types import QUALITY_TIERS, Tier from ..core.db import DatabaseManager from ..parse_server_state import ParseServerHealth from ..core.fts import FTSManager +from ..core.fts_content import rebuild_fts_from_cache from ..core.parse_artifacts import ( - load_pages_from_done_batches, parse_batch_is_readable, parse_coverage, ) from ..rows import DocRow, FileRow, ParseRow from ..types import ( - FILE_STATUS_ACTIVE, PARSE_STATUS_DONE, PARSE_STATUS_PARSING, PARSE_STATUS_PENDING, @@ -348,63 +346,10 @@ async def invalidate(self, sha256: str, tier: Tier | None = None) -> int: return cursor.rowcount async def _rebuild_fts_after_invalidate(self, sha256: str) -> None: - # FTS rebuild reads only cached parse batches, so any active row works; - # keep it deterministic and tolerate rows whose file moved away. - """按档位和缓存可读性重建正文索引,不修改历史缓存或源文件。""" - file_row = cast( - FileRow | None, - await self.db.fetchone( - "SELECT * FROM files WHERE sha256=? AND status=? ORDER BY updated_at DESC", - (sha256, FILE_STATUS_ACTIVE), - ), + """按共享规则从剩余缓存重建正文索引,不依赖源文件仍然存在。""" + await rebuild_fts_from_cache( + self.db, self.fts, data_dir=self.data_dir, sha256=sha256, latex_delimiters=self.latex_delimiters ) - if file_row is None: - await self.fts.delete(sha256) - return - doc_row = cast(DocRow | None, await self.db.fetchone("SELECT title, author FROM docs WHERE sha256=?", (sha256,))) - - rows = cast( - list[ParseRow], - await self.db.fetchall( - "SELECT * FROM parses WHERE sha256=? AND status=? ORDER BY done_at DESC", - (sha256, PARSE_STATUS_DONE), - ), - ) - tier_set: set[Tier] = {row["tier"] for row in rows} - tiers = sorted(tier_set, key=lambda item: TIER_ORDER.get(item, -1), reverse=True) - for tier in tiers: - tier_rows = [ - row for row in rows if row["tier"] == tier and parse_batch_is_readable(self.data_dir, sha256, tier, row) - ] - pages = load_pages_from_done_batches(self.data_dir, sha256, tier, tier_rows) - if not pages: - continue - - from docvortex.render.markdown import render_single_block - - delimiters = self.latex_delimiters - parts: list[str] = [] - for page in pages: - for block in page.blocks: - rendered = render_single_block( - block, - delimiters=delimiters, - asset_base_url="", - image_renderer=lambda _block: "", - ) - if rendered and rendered.strip(): - parts.append(rendered.strip()) - text = fts_utils.truncate_head_tail("\n\n".join(parts)) - await self.fts.replace( - sha256=sha256, - tier=tier, - text=text, - title=(doc_row or {}).get("title") or "", - author=(doc_row or {}).get("author") or "", - ) - return - - await self.fts.delete(sha256) async def _record_count(self, metric_name: str, *, value: int = 1, dimensions: dict[str, str] | None = None) -> None: """通过显式指标接口记录计数,未配置时跳过。""" diff --git a/pyproject.toml b/pyproject.toml index 0870f96474..00f68e4c78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] dependencies = [ - "docvortex>=0.6.21,<0.7", + "docvortex>=0.6.24,<0.7", "gradio>=6.21,<7", "mineru[torch] ; sys_platform == 'darwin' and platform_machine == 'arm64'", "mineru-vl-utils>=2.0.8,<3", diff --git a/tests/fixtures/issue5776/README.md b/tests/fixtures/issue5776/README.md new file mode 100644 index 0000000000..eff1954fd3 --- /dev/null +++ b/tests/fixtures/issue5776/README.md @@ -0,0 +1,9 @@ +# Issue #5776 搜索图片载荷回归样本 + +来源:https://github.com/opendatalab/MinerU/issues/5776 + +原附件:https://github.com/user-attachments/files/34912771/1.pdf + +SHA-256:`ea5d3ef8fc5499e8ae846caaf2dac264a59b248e658d5576bd2c2332e9753aff` + +一页 PDF,包含“测试”文字与图片。自动化测试使用 Flash 原生文本模式验证真实图片载荷不会进入搜索索引,无需下载 OCR 模型;完整 auto 路径另作真实模型烟测。 diff --git a/tests/fixtures/issue5776/source.pdf b/tests/fixtures/issue5776/source.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dd3de3f04e44d3aae7a12089f330cf4f5053f5a7 GIT binary patch literal 27118 zcmdS9by!@@)+gGyLvRw@-JL)PZoyp>q=Cj7cS~>&?(V_e-ARz(?hqtsaGi$tyx%$J z`)2OUoqOkxY4(1qcI{QQ>{n}To>kOJ;*!klEWBvQ)OC;_8ZtW>8yVQh0u5PEkX6mi z7RV}Q=xAsKHe*#XGy^)2aX?2Nqm-q69 z_Led>reMe#&t0Gi8H5b@?j5T<(9zHYvLG2J*9)Nr2ndQGp^dbqX7}&`M!jM%CWa6Mp#`(OQKH0y{dmj2P^B`(K7&-n!4;7#T z*vTHk79yx5*v9d>VJG`b2T2Yx_7`m-VoSI>N~t;;IszdCQmX81WRUCG4v;=O8RUZW zAyit5MixNh=S5_!A;x0oB7<2xby*?iW>o{aI{rl>`d1_NS0nQ*Kg2{P zAVX2Gt1e_J8yPo0Hwz~R86Our3qOZG8ANLvM~M6mWPjOD${y@w3t{-;sh0;tVw|IPoBR>bYt}lKV4} zC!d8=?-&_^osWyLku6=8IQ^0L$$iqf83{&$E%C&FWcBJZ8AIj)x;CEmeY0=)=s|hk z)`Q&+Yk^y&#QJG>GETMdOaUy?W_U1eDa|^L0pX~P!nj?15nq@BlDMda8b;d1Bo-__ zbeyDxO0<^RB3lE4OpX`Iy!k+k^LA*>BGaY2%}&7Ru~NESJ?c-*G{dITP8ZI@AM(5{ zEKlYdQa^BXMoCf-_I6W0-BNyd!2TdZi$ji-sNjv%a<9_o5gaQP%x)+X3`@_5j%4hj zCD8i&*YO>x9{8Z1JYoYWGwdDU1FP%J3sV0!#NVKO#?aqZd~wb*yjWG8j2xf$)bj`% ztEiy^@MTz3Ok6@jf>9h|S}l;F4XcC=#MB@gGgcWBh_6A8Zp_jUCps8I7B{qUe4gS! z#sguX!m0*Vw*fu-g^cqBk^kz*XNngQbjf(1iS^0;xu>27{ub|lD8}E?iphvS^L!SX z<1eJWNDUcQ153$>%NyGMhoG$DnvlK0#>NVHsp{t72(*3<17v(J!amP>R>K}->j<_d z<9`{{B!j^FWt03K5I5+Q)kQOL>G)(U9- zys`fc%g-$SUg$r(@J|t5_Pm0jHIVf`djNuXGe|7rV3mOcKajDAjhPjYjEz;~*$0kf zeC%xh@P{KAGY5o9%+OXE2r@JO>y?PJnI^~tvPrnOcp>r6^{)qZ4t|aohMIp-azU)5 z^^aK`+-#673AEOLOnIRZeF*@}>|E?G3?MOvoBf$BBmz8J5wb6V9AupT2t5BZ+)G&c zH--Nra+VqZy) ze?zmNP5%*qUJ?}RKY+>2&-xD>vh%V-GK>}2Ue(sn82Hb` z!72)JbWj4?i-E0e!8R{j0OE(2+#oKd_K(aU1IbMGw&3Sn@sjTUP8B@dY%iPg?-2Pk z_p}ChB`YZ-34nrv0(^q}0iM4$q01yFiFi-$ENDm5f%fi6Cw9kEj z@~gl4&kxF&09Z%^>7w^4WBs%LOb37p%mM(aqMjB3VgN*Vcm#MjL<9r`BqT&+R2(!^ z6cp6gSlAdi#BWGRh~E$qkx{YGl2I^G5)sky(lfEKb8&Hz((()OaR{<-a&bH_0)>Qx zgo=VnfQClEK~6-@@&Eew)Cs^u1e5?WVW21g(3ns#m{3o>08)sga1cv88{;28C}8Cg{|bq!4|Z5?A1psATT z$il(V$=Su#&E3yGATTI6Bs3;AELZakUIl(ou|A~x8#kEd-`mEZE zX8&u7efi(g>^~Iyr(O#HR0uGkF<~$P!hoyXPMSOblrrzKuxyGiGF{HgjTNf%RtZ{J z0tbK`cRK~XR&3PKO!^a`ulRSlzt6m#phSI$_`qu-lIdS5l|SL;62Ad@gC* zJin7b(D2s9HxAqvH_yj=dQ2LXM5tNIJ4s|E;>j^+y7N$$-R=A|QDrTBcASpeqW>_GWWm$CVT&DW!n_j zI;dd8b_VHEmw-A5=`szE+84pr%2wXJ&=*l3L&=m6My+*7_+q{YrZ9AW#SIurpBp2s zYZY3GBo@AQYp1H2bT2#u`^a-j{uQoax{z1LGrdrHEUwpQmFxlCvU!^6LN4`?M-ENK z+IlV3Ye|x9#ohSG?_TgK3r+T|@vP7m z@9#5~*KrG#cZofBeMDL}^zAKP^z_)V!sZvDgQ-@`EcslEOP54h!@6YIq`dF!ngpsNOQWtP@Hc9%lJ-axM`nBR&9?q(}(erm216Xd`IUNi!-fTUTSyHJ&0B%a+aJ919{`y?iTQv#nGPyqTl! zh>cQ~3qz)Q&!F9e5FgbA@!sj@z0z9ZuFGR9Rz^Xs`4c0xef?&Vec(IllyS?uF11Tp z7MYKY&G2o4iLbOP9Rx}$qljYI1iXdw&4nXaOs5i7mi3Q#eYX+eXb_!h_CMo=;2I zzG6oHtu8I*2iAIbZiYTj)qo0MZ%pG=X<(0sGH-s{thuJIB&Ir~Wn;=Ykk=?0w~p47ME zfw$Q$qgdS{?EbtGvP<)rzPu~kQ?DC*^XvT-nf3=sIS{_9{-S+hnoAiP2IoZK$wf{6 z#(}%~u-<|guu%MTo4MC8PmFh@$1cWoOZ`_(=xQe&&FKdv(3MNfh=+?A3MNBKm5Er4 zn$zoFp??rB@;rpz0=E<`jfw0R!;(wCBd7zr!fZD|M>2yIa*dNmPCkKb7Ks8sqELfp ztMkgObIp_4w@F#Q=9cphWg%EJ7y5x~zY`L(oE;7SQgD<*36rCj(`^F4-8GU$W?hVc zE%A9yExd{Arnnc2oDg?Jjq3D69JMh2RPd>&T~&~dcTHtiV|(R;D_BI4-t3%k{NRAM zWpBbigKqXAF=t4*46NR3E+})Rs;m36^;H7rv-(-~m6-!}j9tILI6@gqmHF-O6gJa4 z6X%WXz!e@m2L4+=wYn$*^Ay~DZ3}#bL=4g-N@)_Y2EMUj4EK%m2W(}j}40cwLkBH~5GPSd>jv(W!Y zoJvM-AQc8FG`UeVAWu+spT?u55wVd3)-#UW(`8W9U4QnnF3OKf<+?M;P_`Z27KydQ z_zK(fo7WmL*t7kb+4b+(%O^4RAGia*L`Jzm`J3{(yIXy1E_&R5YekTzS()ux7J-4Kw@31NVpg%~jwajoteA%AHKoYZ z5$WJ;+pav}u7HQQ!|R6d3W81ZKF>tr#09*KC8_!q_PZeCA)$c{%L5EFXhF{WFKiNd z+9a8D75wU#bwtT$-t)7?3{lP>!Oh-rH#ypCF*KvIDupD0NzX~bV!S5UZ{0ZzE=Gs? zvXVmltP}qTo;}hpbjAlBq-x%56V1CoGN!0RAw~~X$(CQ$gE?i^=E$XlK=6*h)d_f~ zf^sp1g2#K{MwS-CWns{wE^Xvj6G%X(4IThLEg??6+#MS4ABW%x*cV}V1~@zLv`qT%}OMsMPdK0A&J zb`9wRtfmBhcbRxk^;YL&Qe3#lFJepj(H2?OPsIX%^s491{HOFOZkW)07Hov$y%g6k znb!~bgS&Ws#vp*>G@~*6Q?__tN6Kj%((Y(e^AF78{!Vdf(E-)ZjLgu*qYDMnuTRB# z4yJKY;~jXcSLpDcd<%H4!{=V~YYQOm6g)%UPC0Ub$3T>k+G=z$vbr^Xc zvCtSFeT*M0>Q=+c+Ve~fWd}qnN`cHtUGT;*csK0o*pi;)gT>0$ zBvpP-a&7W{IDd-MYg~~gQ%iFJFWReo?@j!a|Ef0p9qn;d3uT6ISm<&VbSPaG^jDP$ zWPncJ=|>Wh#&-u#fNbQ2QboKpSNmW49k;LYnL{IYPe}1xJCv$hV-@Q};Kw;xRXQVA z;=4xNxW6aUB*^GpnDKMZc}mv5{e6s2v_pA6a8XXImf7lhUPf1RwzS~V7(u?PxGmHA zS%rDMDBGwoxW@e^QN!nHk5`+Tnx;pjBQ${|805>L_#57{TQmyEcUBl1Cs|v4^7m2#vrD9?IHUyelttUs!g)z~eU$J! zC~-4+13c+{%Jud%-uPd>@qZQ6IHj?ze>7ng7!hXXvV70ir+Ru(qIdtkR-RM7=}x~h zX2Nnl$>09+xSnc^;7EFtuUzt2Dr59gpJ%g&CqdH^JryGZ74KRbAc0#Rr3gv-|`qadPj| z?V8WO8H297a?5r$Cn=&*)Cxg2>X zea?|pTUf)|!@i0|T_XVZieOI<(E34LeXQKH4bcdZ#Col%Vrrw+rw2GSzYWt&yOJS8 zlIXJf!Jb4sbXyrpIJ@Y{G9%Y^L&FUr$+t-};|6=BMlF#?%1yZ@$oj8LUX6qo3u;2G zhaKp@)xE>HI&GD%lA#(ZGGI+6H*TxMkFn(=!*Cb@c###_`yIT`wyVXQ#DM^<`}Ffb zpiS__#|K1>L&NKqOths3%_o2;PJWfrb@W<0|3_g7|IPT+sKMFP?ayT4m~*ogmMMY@ z%_6Awd3Ka|_L4F}RkyR$8yKi1`l%&m4}-|nENu3CRI2%0xu#BP1zSgv3UsDThD7Ww zO}}%wtHA5TqE7H)MEsAZLl2C#lfMmXs*%80{_F8+(}^w z`yH=~8oL&}^IdzaTXMCZk+X)Mp-;5@b|%E1mws7@TJkvgrmY}-#xfD@iyJW`zgk>; zvUfk5i$r`lN&Ige);G*wp8y4n_a%e5$Y}1D5l;XzOX1s%j^R*`w7K+X!}h&fwP}=R zWR@6*?_5M7N|fyvdpH#DBa%zL5}!%&t*gvPxCs2{SPdxp^J-4h?3MF*gu4-W>O=}5&gFw_5u0uJsM74yC8`ec|yd7?rhA&h^@f^(5 z7j%ot6H1gOmX3Am8SyiQycqfb?RW=Y=_o;yN2l4J?-6q?0*3eqhO%B_NAIcGeORKf zI%G=*SkKUg32*5)-c~&N)5{rUpHhzaTfOfy+{cS7;ritD`>S_ddr-Jzsh19jCT_q~ zeY*iL+9s{B2gx?DsX|;6OJ%Tb4N6OC4$2P+1lt@8ht33iLKYshGHc6`Qm`6c7gL#5 zXGV(69AjfR&2ziWb>CG)aGErg)|`bH9(++?=#+hzOORM;|84KccB5V*;h(X* zERyZ-)brf_m42!`A(*AO7?}1sC)ZWS9jWED<-*4-jpkj^*aZnGP1ZF1`4uAg(K@&P zM3-a4-4&T1>By(M=@@1ooBQE>qAGr?=iTO2P1@|Z{K;`QdS7bK3|z<%4MWCa=OaP2 zsiig6C`Au!sp|+|KCUrH!+PnIke6qnY>GYolzpk7O!`xhXSgi{A#J6us-l`K&H9zb zQ@?j&6p?m0S?0M~Oi2P5`9+U*SNLk4#@;Aicf$J(Q*cQ3!tD_tYD_pDO!1z zMXXfPawFYi`#_sVLKpLmy7$fs)CRQTT4HH4Zt z)+Q`Ro^^hy#7?x__i~xWG4;1$)|sJxP0kriyW(-Rt>pV7&KX}J+0Y+B>WvBd{DiVC;rhK3(jyIc zy#VJ9AqBgBcQwYc4SoU)gr`tWN+UezsEnvAUtqb-CNKo%*Lo4qSRel>TwLO8Vr6t zq{K%Q`Ov3PcRE^JdL61O|3;3Vg7<83ae#rif=Ia{sjqA8$oBmJnQ>^@x)8;qy!J&2 zqi#}}$<(h%t@7y{vpUyt_=Tk{4h(O3)sy<*Ti!@xU9WfMj(3*JQl^yF*{HUMzg7`Q z8ne*CDu@o=2t|biR#=-~x$AV2-=Y70D~rC`?m?u9Q0!@Oio+3{^<{y^6<6KF-v;YM zLE+wUtj?KzL2AS89^`rJwOgA&N3uBfdEG3AZjnF*iqA?j;X3yH4C_TBOPS-c#OA}G z)n54`Pxq^|=)u9g?x$(3$ww3%WI&zlIvp;LC;ElIf_0oeD zYOKyjXcZ?lgs;U|%LV0ehZeaKRFcAOdP~3vHULPj_ON6VXL1FY%)&f0E*##u_y5rJFE-pk>- z`;0UENajV(Q6e|WweA?hzBD=63qlCnl0SEa82~A!n%9gCMG<^}Xua^=#uZ@mA0sRH6Q#jk=Z zk1f=e+ga2W!D;Nj5NRPbsB`|O2v{=OFfPJR^Z(tNG{=9bgDDzWs5)A+DnROsf7R8< zcp>%2|6$Rei|wUu_;(@4H7#C6-m?4st2X?53c=;ZZzy)+&Xk?GS#*)|1vI7cZFW4#lGl^#seK z*h%@mVbPW0q+Bl~^9k@a_#kdUUr@BU{z*sszN30^gw?5{Q8|SO?H#lSH_^do#A75K z+^+Wn14Qv&S{-wi+*BY{M^(G2vqX5i>^dMy`nuYY47*XL+-A~VLT@f2{dha7Q(OA_ zkirRSYYquNOFnb;YhtmwCVsiZ<{4T%L#ug$ZFqSQ_Ks0UF_SuhHMNuU=L~z)i2@zS zC+=RNqvX1md5x}RTg%}WiXi5kd>ZD|;kn(E)z#*j9SoJ^y*#|&YDd+07JIqLd}_%Z z%@LJUazclhS$_4yV%HM+n;7#MvCrQ$INt6AZP1^+#Wdv@==tUdb`Lq5ZH_HFJx7}u z12q5lz5cJYenm*d)D7(9$f{%oDeyfP%vnXPz{ZxW&jn{Pj+Z+9zm`s4MqWz8x@7!p zWISyAEZp2=9FT^CjF*#_g_rwZPZeCq`fLWce;AsK)Zzysih6#2yqK$VVZ!wL&&H4{8?jWr`pcVMfw6xV|K2Bo9EBK1sSl0|(Ap zPYR^^`iNJ&{lKPWMltFg@+N!Wr+_nEr&hWR!;`zpXS**gP6~W=HYXnt*D|NlRP_us zU$1V$Y##soRBPkE?6Mqvl<>;iRx~l|lggK_&+u*EP6}vXGpf8Si@=l3v^!&`b@2Yi zGeMRFhiIcBpzz^{L2(gkkya&{z|s2y(h4=<-LTLPf-^vgqrw6~IT{6Z)fo48yoEnj z^cLf+=RL&RdZqWyBzdE@J>zo+o)?cACMcPBo3qek@{2fL9J~8uhn&E$LI(Ep( zBd&jxmLX@a_#r;ydG^5H%ekigPDgT8onAPeCE;+)&?OuHjg1$ z313`~Tsn?j$6Sv8JU(iBR9uw1%~@8+dyycCA)W@HDDD^H_4j)5UZnFP8Zvb0Sd>*w zt$N*F^K=uXA@Zy=XURx&`ID0Wpv}a}@vLlZ&)M1hP3npCXI76cuPLXw>3;2zzT(x6 z2{w6kSiEx>v-z*cimcL_H7v%iwHCw28l{<6h#1ACARQddk>f8Xy8ir?waudz+KrZ7 z-~DgOQ@(caPHEvyvtIYJqHbh+Q%udJGU3lekjTv$mzdvKFmqq_o}0N~T@EHHt$w?0 z?k_mHli}~3yhOP`yS%+noKc|oAqB?lhQnUaG*$bYUG8&2)KMjj9Ur5F{%$#{|7*p^ ztGAO6eRXmqA)!*_1Ru00psLRLmZSY$t$eSKg|PZ!Zj~ys68c$G#PTS#Hx!Kqz1Es0 z8?EqrdvJ#q&s}?lq3)~3_fdG>OizVZupG2} z2B`l}#(6}X)3Vjx;`2x;y>@BX77n{-n*SpeKO!t$`V{Z^+ot;B;}Xt`{cZdmT0YwX z*KGV+XqPg{1(Ok7KITv{MaOoTaP)Fdeb8~3+J4Lhyh zQ?)k_T6M=&UEE(gKg9M;x~B<^6JBc10=8iw1sVQU00j-M1IjtXCa*1jjAPgI zYZpw4n}2YYUd*Oi!JL%k?EJO7Y3z@rlrz4qINYgSXxU8AP|>@EBl~>>o1EK_GJT_> zGK|U`Pv%jj@DnazXJf$c$dD4h}`2!5dLLN4*FXkK_ zt%jkaXRwrkiT&_2h`QP$r8pKi7e3+WFz_mv^YF8}F-{p%oeCuyBG-Ewb~euRQk4%C z3&vD=65-ntegyTRtm3KF1O2Zo@fqju^FwQmBqX&?W>^eD*)Q{Ux(ec~m~-S|`=R<* zcIi18B1j~`GeH4A3knn}TLDpD5X79_Ip2kUgR#dA-eD=o@98OuQK@~8oIBO6E-_oe z!oYrbPV-g1Ra9ax4)bR+@o${;om=-Gyj_)M&TOiV`5P%)6r6Bc5emlHAq7n8U7a90{eO;Om(@n^SR)dY_5TRLpn@x z0^enw)^P7@h_oG#n7Fu+4nH*wq6bEq^heu`yEHXSsv0Z0xPvA#m?rH2)M?emo$^Yq!0ckQ*|6qm8n!I z*grGv%oX=w;Un4%K&Pyz(_-cldv_&l}_l3t4xAGA`(+ znENE1-NL33@a7~TZd`Cu_5(UNwZGsq6!#lqyxn@)$`p0<8g-&igBbFZlsUlJt5;We z-4nE&6FTULET7}zErzLJyaH|N)t9X3g4DdMQ;2lZUCnx1D5YXSemGOwRto5Nn*N_S zM_RXH*QzlEp95-fV;g=Ds1j)?$=%ATs(b(7`BJ_x!q4(I7HDv5K6;8u- z^XKp#?&u*Vpu#XL~;ata}mS4efz%CyJ>KWsZ1Q5 zn7glHhzsxsndb1^%IX?%c)(pB2eEpK>pNu&{JzSInbJwi?qbSdi+Z0HI?e0~n9@^J z==Oe0SCB2mkqbz`O-geUlEK-)5Ny~>vzns|&Pz8#UL)=d!S97Bk=+N?;=^-fnZ{KCxmP1mO0wJeCqLKEQ4QoXinc)W*Oa1W|*dx%y=gE z{-%G#F-ircO;OKBz#Nmb36*OGbJ=6s z?51S5>Fs$lERUt@ZRKDl;f;1UUzY??S{bT&S8RDx+vY`~`q#mONo$>P@%V3HJNVy0Z* z6l{!a#QYS%-(M=xYGxU~ z=x5Cv%gZsNisu^rNl1?!rb}0bR(99?zOWU@5_Bf4nLHtss(`^Bc_gW`EnE>AOG77> zQck47z%R8x+xe|dfXLDEAZc4JQweMhn1jg%4UG?)xG)-89r+e){rnc6BZ1nzS&%gn zUBEnnLUB)i=nVpSh~x>^ns&&_!gUDQx_`_%kYPrHu3h9*(fM^`?qUOSVwp?mX7MqmY|N zf0>?V-m)9GT7i=pijlaOcQ>_ns8hcnB1=IcNTD>M{8S+QuCm#G)FxF+FIC~yx3#qR zu1*k&L7e)zI`#OJY+2(F|5~RiuqycO>vh#nj8PT1bAOayju9Q5;nc?Yig*!smVgx~ z&pc2m!#rx?6U%G^%*cdBU$-2|5?2t}HNyKiF+~3{s9*MWuFH0Be!_$8nr6~NFF)}M zrl|<>CZV!0Yi;nq!w7L;XCsGm_qZMj1>#PQJiI zzmHHn?umZPeu~FGR5KM+PxrPoZJGF&H#c7!ts&+4!S$!p!XJ64z>fYXJfo-jQgmbI zDf>&yNiBudKen^#p1>Pt4}m>*odn^^#2Y8ho5`BZ|Ft((qZJ)1!f^hT72OqboX6|o zHq=K4|MgMdC#JdG2h^$fPQpPgI{FM=$Au~ENeZ$bC+zpq*U*}O1ke5bV)zD^(VEId z$}QJAV5uLVA3t_U9(VX2>TYZ(_&wGB1dc{I7yofg-Y;&(yTD186Vkq_70~3>6_UZP z7=68xGfHx#qUe9;Z^+^_Q)%S1Q%P`fZ3r{{T8waNPfUNmHk;XEa~#LXB2G?7;XtKO zym`u)vTmPU482$i!>%|Mp=BzUi^WN$7v9OpUoM@DNM!(prH*_lKN1D3JP@d;elX4A z6z=ZlzD1Vk-uph$eb8L}DLJL9X4+3ZXbKZX&my}+t_(d&IVh-k%CB2*|Ff+o^%ZOC zPq3r;d^pq;zN8oV1Eh>ahZKs+gdBX%ip+JF>Q%cU!NL4-%X(+T8@FA;Kxd31K*OMy zDu&`?O(FXf6EAj6kcM_IXr3ZJncG1!H+;TlNvvRx#))39@+i)YGwlb1nF)DYw`QPf z`>;%?tZ9+}!H&(JGpB>WtaF)B4`lgtA}CfO!TE4cRfMr+YZ`%^$-Q7*frgG>!{^aB zM^&<*E(f`f6|0BTZM=i?Q@o|dDMcN{yzNyo-w3@Qww$r3uE-houc31&dY!%vI60Ju zeIiACj6J#LiV4VrS5y2=whyM!U{{k3l_myJIMamwfXRfbMK!V<{2;9?|7OrgnwL~X zLrgX-Y34)E*Od>$(zyN(on}F5roj~yu|tmRVB97&S<&ZzO13g7QYD^ZD8s19^R!!6 z01bYyv5Wc&v)ZnOS53}Rp>D<@SJMi!0Y2Ks30LhcC2RWc#Lmko0A8 z>Va)fXk&UDK@C6aS5L0!+z?VLCI|MB(guWG(v3;{Rgt9b2j3f$^?2AvppMx`r1`** zPQCidnjUsug<81&1>O>S0~^gQo4m{3G`nG$mYIQEnJ_+%>Z++aowey4C^|KM?O#*` zdn^2>&dH~==&KYK|0&I9s?fZc4}4{6*naFk65=giXVSn@!(HpN#@W}KxT>gBfYKNVqOa?I zJU<$7^E+d$a)`3XnF$^Qx(&B(twF!j!9=j1cfWg$~@YmgbLB_eu!j^x=RaOEZ=4gs4`C2RG6_Jbe2p$1o^z z9HoEy^@8;q*-W*}(lJ(vYXa}uO-_GMbMo6?yq}Lr%L~iz>)~+HBR**QF;E!2?2=gX zd%LfFnbkhyduxTwgB3x=G&9S3i(}QgGF*6JTqJwp4>Z$wd(u@oee$Z`ddEP=odma! zi?MNWiLLaSrn(a;E%xsOT?X_O3f?#)M$!$Aqvz~y3=jwrWEg5Chns!$txzRt*HKk9 z+%oJc&6~^wys3TUs559E<)G*qI0u;Xx_Ovj5e1wQNxL(?!ZDdK98oW1e zu(FWYI*pT)l6ncCP2O&k8_FyLT^xU*&sHr}Cng>G4O~uQe5(IFvN^CfLmj2>raoH2 z)IiHi^O2v`vTa}U-v2aG?Wpkk_uW-keceXJO3ENiwB*aQ3YQM1jpe=p0x#Tk5d{~_tfbT=- z-oY~+r+J_FrH5+)HiBq|Ar%MpPTWYeRfAK@M(EGumqQuBh7WEnw8BTlH^=F%f1-tj z+BQQjFQt+TeN4N};-bP0n;3f%<7#PUM-0DIJY4+Q42mMUgO+qD`95T`Wk4fB)gBv* zATsHK4Gz##Xyn2;!go#Aqs-`4v?`!Y=St^$V6$VZ9B;1ly`~d04<^TSUaNeyknq*F zP!P}cw1=3xC&)ZDaV34g`IoT%Vr%t~@#ldm{0e=$<0}Ib9OMAeY$dw?<4w`io)O>w z4wqZfQkee*@rs^G1jgVW$*b>4K~KUzasOiF6OKsj+p5UdeL}L9iK1}E*-I)KlS$pa zh2D@!DO!drwci_prmyHitf9;pFErc|- zpn>H?O#j}w;o~<(zo}^Vi4Sk2af<?8Ih#4VCwPb z;dkBNvMSvDpUb;cYL9yq9 z5q~~4)l{KULG|Q`I31=bVr4yMbr~n7sv#EYQ!lxPn*X`treMM&<151z-Q=D}dcio=GW3jHP`uzpuhD>nW{m-=xS{)0*$%Fla zoPhA1z`i#SwTk+7?|gmY{FA-_3djIze|`LR#Mx z4l2Nt?%#z1E0P*F@JS`RY^=jt3&3Hmy&NA(GgqXJl$(txjDNW%L}NZ!+bgE0ttZ!0 zC9F5-T;O>*l78>BUT20G(o$A64BqvV;^h}m+AKP+;>8_A;cQ=6Q@&X{DKm5tyzJt= z8Wc>B1gE%;*K#<7J%B+$kB_o@GdvZqZ+Ra5+ao+Y7za3*2u;Yu>s<~0G&dOBO{}Q=T(nOWA*$fa zS#vwp)z8^Jytg<-MNZZOS{GTCGJ>%<_TTIuykU}8|7u)hAh+2>=CtaXs!vz?ttdbx zpf#L}99^Io1LIFf*z7=mfB)V^NYyFzs}H_eE5uX059Egq0}dV76<}ldD`GkVpYn)P z%1>7&iv)qKO}+dHPdBM%(Af3cp8S$L4u?~U8{0cQ+>?t>T~tT(oFze1rHgGl{_EMm zbOa4~pPONXCB_SU;Wsx<*SlXpl-^8y^WWZj&DvN_nM>H1>R~%r?4}&A5+yjYzPGeCs{a`5QXqDfkUY@VxFj*f14mEQT)P!|ccR1M7J#gyx zmg!wc)XwT-@}BtAz(;*|*X^6VRc2)#R#NrG5%w+HM{zID2B!yymT!zc?GJOKn|osT zAC6X=ve(;M;1!5f+iqk|67(~B4wVwxJ*jqoCcC;L>lOCH3 zx7v?MeSf-lTwWA5&`NgLPv#nRSMvI88~f?NuGewv$o&Vhq;=>t1*X*j~IPZ4IjfZNY#j( z`{qvjC)P=a+e^Y$Y|jey2F9e`A+}hKOfU5PaQ~IkS&YqL!|BaiIC-kw?F)n87*)b? z=?oUqYIFA$0vciYP`liP%6Ol8o zmW3Wx{pyBvR1cqrx>Mb_!b3uIDg5O+aL8G&^l~hy*?SC}s+OAmmR0=pCi?25KC$;= zzt|XaYVCn=9BEt4evg|Wp%j>}R9@&#-sRqJG73E<-mA*tdNB9ggaAw`ZseDZ)$%6f^`L9d`yD@FyT~u$& zCtdWiLbv3N((hM#a-T$P)-DrKOz&GCt`jX=X;|)UZ~Su~JyrzUHt)C{qy|kB+M6t| z!jPH^w^^HN zZ%4;PT>ap4zFzmPcye*!(`u-+Fsy2J$?nCp3-p;rzj_Pg-P0w_j=;N~k-wd}J1;x% zKro{!1Y1uRo~!k7WLI_Qs;}hzCeTfGP-L9ML zT0Bom0VSE%q0FjP)2g5H2T2Y0xR=MWU^qlz6+%=??WD4tTmqqQ1Sj&Q-TW0J|3Mb#({} zkZ+O(jJ0kQR_M$;c;_jpn%%6S^!a8v{KA)WgAk-y)LbMT2%BjcXL{u zb$NGqwcR?B`c997Yqd#$w&P)azac~O7wSow`~FSBY)!vqR_YEGwv(h0pfxw{tKKIDJ!y;G5l^RC4{K|a zdzjIYpDAU96?D?;-Cc`T*}u<4#OK5P5>7iApTXO_A}>Q3u)e9TT=i`g$>p&|XXS;J zpblczNp^J|wsa8iQAk#5y!)=2>8h;~NZP`BYpXKEXY6j(vheZ7;xxfa%yhdZ>W8(r z@Nd7(JKmKtEh4wY>%wh^j+@2QkLD!pK97V|)-d8^8j(5CU&Uc8W6{MayKkQ%@6sQCOymAP^7t$!aZJ)1s0$VX- zY}@zsek8+uw|P4*bgyvu=Snx0ba|=iZE^GVX8*#~PHnpPX`9)AV`-YpDuyoplM>eC z@;jfw(d0W-6;kVlL7D*5h)m~8;PhsBcFD|xw)6Dmd_cpQ+a12J(50WJxq;GLrHgDW zX@%>w?eBP$d`gQ(m#~nFg*my#}Zgraxn?I zNpsBcw&%*6Fc#i86&iD>GGkd{6?nOor0eNgFCcvCy=7~6dsN9RJciVGRlk?`hvuGb zXSY_n06*cFFX54yk#*Gsd~0Ibk;NzE`JKA&_4(}_tXKD-a!4SS&}ugaetkU&2rN_z z_PWDxv!J+mv$tuuc5z+%#sem;$G`Z(nO@!4D(;<%ES`a`mNu}W9%&L8CE!OleXdl* zY|kP+xvH>{r@g0&qMP38S%LcWoD>d^w#9ycs7KEih7vDJ-A;b5RMGKYpD>K2uaL7j4av77GoVDStt882FY3>!lSa2%2J9f$3IA4}QF7A1`R zJ+%CK@Gkv6pZ9(LdCp&FzUR8H`?{}t&N=rvGuQX)xGZ(xl_AHaujkvlgNIzmxZbvR z3xoctMIAXEBczhfx~Z8i9>eopix|DOmRH0$5uJdq4nxJltdkx7l7T*xN`zI+f#%Ho z#gSU?yrhGE_;Z)6U0bZYX_dA7o`0z~aR1Se=typ<(Fa+C zcU6dK2WmGqy>GTB5e}wLB}#%8It^?V1708kD*JiYXk?xn1dJaiK`63j{e4#lgNFAm z&aYCPO7mSAV|Q^*=`9_PjaKb79F}=&FIS+<)qie9J;Ea=pfj?Qyr$T& z^461unj(Fnm#ay{0G(zJx%Iw^31V6R*^9iGHoa&*;E~$wQz4fxfb5faDeRGaw{6+~ zYjA&+pI`=;T)%Usvig(((^pFe_czu-1r-5qChD&W1MGcS9?_3sErdIH{BrEaMesKj zD{O~Cp4Le8meq6FQttLex;3jIylPv8)hy3?;l>1SOK5YYw0Y+W`%)TuNPb=I7ZjqB)P)t_@b6iSc&4CV{=)jGgM|{alMx zfZSWF2}GfMY8b`)h5d_Gy*2W6PSJbKb5TAxOHb^khdLU8v=)Y9C+@Sd9c-zf%=9W8 z@spxIwA|GUU$xuIYBfSRIK%MD!oT!D6?Kq6Vyx!=$aUd39yZSBcD3tWFE7Bt$^$>SfpN)76VbJ6AfR0)E8Q>V_*;1fE;#Qf3{HL8biWhu9z z{G;)3e48Uyq#|rN?}@+hMJ+QWOL@m>&ShfzT1ocG1MV=of>N5ROl%juGc;DR+47=! zm75e5J4S9HyEKF~?p9XiVxAQqF3`f&?~dMeqLcU+>;CFtlWHxRm$fS8YxsLD%A0Ho$4#r7nafC=v(g|9Hj4*FOhs zA_*PVX$nYKSN@VhR^?iZZ|E3$R@Z=F^e(Kv(IKD2LG^Mdl$}zNl642;B@u9Xm))W% zTm4J~tG(#+uBX8o7K{3%&)UtYAt%x}J`kRemYDDMpOUxC-%tN13s^KJ9VKm-OFtP_ z$TBc1=dE_<+_B^yp>cF_-VD`9F!+Is11~N7k#C>!;JYHK{Nsnis~W3P&XHHC5x0*K=@0sTSQYO1oxP)5LX; zc3cdRy~X5alcYk|f@odRiMl_8OP+m0)!$L^RMm;FR`ei>X!(k=2DTR3I9uoW zq{8y10AupH_PZ?M>BD4$cUMfRKjE3fueS3MG%gj9+^@wJRJ`t#EWU>!yYsqMcD|Fz zTubE6kDK&96^CEQmv|zN;s_3EB4dhGpMQdu)LZBEr&}jU-kH_b?YAxHXQhGmuEXA&%6Dmvx+6Dl`T42lZZ|Dn}D9r`4O3%{44$zTfvn(2NiGMbUHhlubWM5HzXz zjwAk+O!M(K@mcUQyXc-W%PT)GIifvOaZY1D$JxGvRxU&hE9UXZvv)t>Nq;zZr{T)% zGyZmF)dn7HM?VXFZW_$_wgEzussrb9#_eRCrKd>#1StbW4t%E_UCyHT#N%|tNe(&v z<4R85!`58x%;){W@#T&hWqhKJ(_swi<(K^_iDPsQcW1eiHsCCJxdrzHyAr$15(K+Rv)Lxi-_krhVTQ#%=HnMWLpfeXkm9S=Fh`4(8}GB4trV5T(n$?w8oK@kqi8q0%)njB zMAV4{A|c(LME4*T_+!QF9nwRJUlh}oKk^Un3vZqvG`M}n>XdZ3i_Z#@bxlZC*b9_m zN~1#DpJLR&sh{~ymVIs9b#%6OFE0AJqwv@1D^J?RGL;u{$=o?a(d0HE$NHH#xNLS_ zHdTiOiJ{wEqK4j8nnAfZqrUo@mbdYveq2uR&`P11qINYO*F>u0DBW&knNt{wcG)Wn z(kVw}JU-O>x>E5Ft}Jnzz>_EK{PNk>yEuquVoiXTw>9 zx>xzWG1#1!g!neAndVhrlZ&%ZoLxO4M<&9HkA&&LSl8S|q{2LjPaA)L?A zCukdO9;XfOE(@p^XiaXlK5ZFo=rp{_r&%$5#42O;umBCEhRPQtC&QY$E<|&ww5nTJ zLzUQqo9u4My2Q#UB6N^_f2t1Un%7c36(j9ivGKi;N0shOR#5Hdx=$@eulmORYiz%pjiH{pm5iz*NyWG3(=bs_;_2@!kXU+0t_Qfj zFzcl`Rsc%CTimm2++V*;Vo?LFe(ZpzpJ_eWcty=%ao`2|s+z}*wW~qy?|l@%^rjcv zeo(0%q82G@Dsnl(!-I=q8hgPpDb~esV4h<>CeQGlJ=fUnq9uD-Ml}S$pc8CpE^$DI z*{nzJtyYALVH%eTFTv82%v9Usc_~r*bNgd(qTF@O%g0MTPT8JkdLo5*D{D98QfwPg zbZho==iL4qxCOQAT2kDCZ?#pjkLkKMo_}L+G+0~PZj!9(O*tzAfhD#OGs9yF3GYoy zlF~x(hM|H?G*%qXrx^2nU#n%E3wT4Y)25oWt@o0V_O3~m`E=RRgE$e!NR@x0-GA2y zhk(ep;l;aWF?xHV<0B3PIS|9xXB_Bbm4ILKX`$1#`b%=oxkC!RSLb=4U;` zvWG}rOR3RmnZ`I)o7d^|-Nts&8OM@(0{O5}pLeexWC!ubNhT*)ca8B6)8ZaJ=T*eX8h-V{Wt3Mh_;vs2XpJRLjX^ULtU!epqLX|4pd;f%m&{bEDKW%TH5~mg|l8}(iJMLASR8NYTDB^Cxj1R|fUtA1F81n&&?o9ksEP)Cqi* zBJhwDK}6&%5=OHVUB(OaMcA(>a@*K7m=I@-$fBomEtlEi4d{dk33JEt674a{JTl_~ z)Yr7ini&E{3#4LV-3nu*_ZBp^%|x*@`}{dUDI>TVR5ka6O1V+$@a5y>l0oCp}!qEwjxe|3hS$*iop6ib6({vYftn^!Cp6y$W^SKTxuF+RW z7&=}HU2P#E+DE3w>seA>BSl_(GI_ivQXpVhTFp6Kkd`9=9D)uuo}dc=Tc@)O|8#i;KL}?n!*7Jtf@rxa+SudNeBEs@zS{a$ zz7AGsYkoOdX@V5N$<+xU2?i0I9GyL+2r~SZ8yuff!2Sjd<_B$>csj`N!^NRu5)dfA zETgo$wXKw%lFIi+z$Y1gdrwbSDKOaE+gsclF7D!P2Zo~2XfOl@hQY)D3o#ELXHQFl zn6rn#4-Y?xL4lee(1u6L3qYlm3_m~UyICYe4uGNJaPWpBU|;G- z>65G$&dS}z!^PIKDi37j;%*HZ3k?2nGiSK$2I%J@5FCY+-f;1+X42q|?tcmA`w)LY zb_~$k>gTv!z1$r)M{R8dw)sYl>H)+E-Hh?ybPr6;Pyc@#KLKEXg^EKqeFH{P-z&BB zU0fW0BL!PqO8*3GHVUw0Hwv`GpinU=N*{`rLPDfAtF)vb5NYrh%gvTR-4@Q)=YP`j z7t4P)^FRPvTK{!&oSc5-M@vge$;HZRqxw!=N!H5?XDtPnL?TgWBwP%s2uF({RA7o? z3J?XT7+M95gh@!iV2TQ;&0+q%?Z!X>MBUlL(-L6M{&&3n(sqZzgZ>XT#{YvpK>u{| zTUq`mUH_!(w=(cs#{Yb~{z=zwW#G4r|M_B_JQTIEU?1(vHq6**ZLhO(3au% z23F@gKW_Z0bfARuaCNlw0aiEqI8T5v|C{Q^FOL*IH3@!bRv7Eq*n%-QUqHYB1vUa6 zR{&v)4V{Z`Y9brL0N_mpflX=wU|-2p8VLthN=UdQ9F0_jDI!!h7D~UalVtyUiwzq6 z|CHqd3g5zVVRe32@cKGe(0vdMO&FFb!#PAm=qRL?hbjXU+&ECl(V)-Z#j075MuTxY z$yU#(AVY`S2brP5RndBCAcN4G2g);`W16Hfp{Q3P=qSv~lDI2Jk;@Y#`^K7!(CCfo#EGP=LH)3x?dFGuVPjB7r&Ff}th1^#O%! zNWuKw4hogn)-M!>*yaz4klc*;O@s9v~l3KM<1 zU`YJ0UnRc1(6Ds({C2XzP!I$RP)ZV1)=>d8Z@!;^5N5{zkJ!L~Y7!o_84`HoDhd$! R0IEDd%C-fKhNHg+{U7|xF=qe( literal 0 HcmV?d00001 diff --git a/tests/unittest/doclib_test_utils.py b/tests/unittest/doclib_test_utils.py index b1492ba640..6a60804877 100644 --- a/tests/unittest/doclib_test_utils.py +++ b/tests/unittest/doclib_test_utils.py @@ -68,7 +68,15 @@ def create_parse_services( parse_lock_timeout_sec=parse_lock_timeout_sec, telemetry_svc=telemetry_svc, ) - execution = execution_type(db, fts, config_svc, data_dir, health=health, telemetry_svc=telemetry_svc) + execution = execution_type( + db, + fts, + config_svc, + data_dir, + health=health, + latex_delimiters=config.render.latex_delimiters, + telemetry_svc=telemetry_svc, + ) return ParseServices(ingest, queue, execution, health, config) diff --git a/tests/unittest/test_doclib_migrations.py b/tests/unittest/test_doclib_migrations.py index 0fdd50325f..511f9192ba 100644 --- a/tests/unittest/test_doclib_migrations.py +++ b/tests/unittest/test_doclib_migrations.py @@ -18,7 +18,7 @@ async def _run() -> None: await db.initialize() rows = await db.fetchall("SELECT version FROM _migrations ORDER BY version") - assert rows == [{"version": 1}, {"version": 2}] + assert rows == [{"version": 1}, {"version": 2}, {"version": 3}] asyncio.run(_run()) diff --git a/tests/unittest/test_doclib_parse_services.py b/tests/unittest/test_doclib_parse_services.py index 01b63f6365..2fd636d9de 100644 --- a/tests/unittest/test_doclib_parse_services.py +++ b/tests/unittest/test_doclib_parse_services.py @@ -131,7 +131,9 @@ async def scenario() -> None: health=ingest.health, latex_delimiters=LatexDelimitersConfig(), ) - execution = ParseExecutionService(db, fts, config, str(tmp_path), health=ingest.health) + execution = ParseExecutionService( + db, fts, config, str(tmp_path), health=ingest.health, latex_delimiters=LatexDelimitersConfig() + ) pool = ParseWorkerPool(queue, execution, num_workers=1) acquire = queue.acquire_task diff --git a/tests/unittest/test_doclib_search_index.py b/tests/unittest/test_doclib_search_index.py new file mode 100644 index 0000000000..7685692fad --- /dev/null +++ b/tests/unittest/test_doclib_search_index.py @@ -0,0 +1,307 @@ +"""验证无图片索引、真实服务写入以及旧库可恢复迁移,关联 Issue #5776。""" + +from __future__ import annotations + +import asyncio +import sqlite3 +from pathlib import Path + +import pytest +from docvortex.options import LatexDelimitersConfig +from fastapi.testclient import TestClient + +from doclib_test_utils import DOCLIB_AUTH_HEADERS, authenticated_doclib_config, create_parse_services +from mineru.cli.commands.search import _render_search_results +from mineru.config import Config +from mineru.doclib import app as app_module +from mineru.doclib.core import db as db_module +from mineru.doclib.core.db import DatabaseManager +from mineru.doclib.core.fts import FTSManager, strip_sep, tokenize_for_index +from mineru.doclib.core.fts_content import migrate_fts_contents, render_fts_text +from mineru.doclib.core.parse_artifacts import parse_batch_json_path +from mineru.doclib.rows import FileRow +from mineru.doclib.services.config_svc import ConfigService +from mineru.doclib.services.parse_execution_svc import ParseExecutionService +from mineru.doclib.services.search_svc import SearchService +from mineru.doclib.types import SearchResponse, SearchResult +from mineru.parser.base import ParseResult +from mineru.types import MiddleJson, Tier + +_SHA = "a" * 64 +_TEXT_SHA = "b" * 64 +_DELIMITERS = LatexDelimitersConfig() +_URI = "data:image/png;base64," + "A" * 40000 + + +def _result(text: str = "needle", page_idx: int = 0) -> ParseResult: + """构造含超预算图片、图注、公式与图表识别内容的真实协议对象。""" + blocks = [ + {"type": "text", "content": [{"type": "text", "content": text}]}, + { + "type": "image", + "content": [ + {"type": "image_body", "content": "recognized", "image_base64": _URI}, + {"type": "image_caption", "content": [{"type": "text", "content": "caption"}]}, + ], + }, + {"type": "text", "content": [{"type": "text", "content": "middlekeyword"}]}, + {"type": "table", "content": [{"type": "table_body", "content": "", "image_base64": _URI}]}, + {"type": "equation", "content": "", "image_base64": _URI}, + {"type": "chart", "content": [{"type": "chart_body", "content": "charttext", "image_base64": _URI}]}, + {"type": "text", "content": [{"type": "text", "content": "tailkeyword"}]}, + ] + for index, block in enumerate(blocks): + block["index"] = index + if block["type"] in {"image", "table", "chart"}: + for child in block["content"]: + child["index"] = index + return ParseResult.from_dict( + { + "schema": "docvortex.middle", + "schema_version": "2.0", + "is_full_document": True, + "metadata": {"file_suffix": "docx", "producer": {"name": "mineru", "version": "4.2.2"}}, + "pages": [{"page_idx": page_idx, "blocks": blocks}], + } + ) + + +async def _seed_doc(db: DatabaseManager, sha: str = _SHA) -> None: + """只建立文档记录,特意不创建源文件记录以覆盖孤立缓存。""" + await db.execute( + "INSERT OR IGNORE INTO docs (sha256, short_id, size_bytes, first_seen_at, updated_at) VALUES (?, ?, 1, 1, 1)", + (sha, sha[:7]), + ) + + +async def _cache(db: DatabaseManager, root: Path, result: ParseResult, tier: Tier, timestamp: int) -> Path: + """写入真实解析批次协议和对应完成记录,供生产缓存加载器读取。""" + page_range = str(result.pages[0].page_idx + 1) + path = Path(parse_batch_json_path(str(root), _SHA, tier, page_range, timestamp)) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(result.to_json()) + await db.execute( + "INSERT INTO parses (sha256,tier,page_range,status,done_at,created_at,updated_at) VALUES (?,?,?,'done',?,?,?)", + (_SHA, tier, page_range, timestamp, timestamp, timestamp), + ) + return path + + +async def _old_database(root: Path, monkeypatch: pytest.MonkeyPatch) -> DatabaseManager: + """创建 v2 旧库并写入污染索引;文本直读中的字面 data URI 用作误清理反例。""" + db = DatabaseManager(str(root / "library.sqlite")) + with monkeypatch.context() as patch: + patch.setattr(db_module, "SCHEMA_VERSION", 2) + await db.initialize() + await _seed_doc(db) + await _seed_doc(db, _TEXT_SHA) + for sha, tier, text in [(_SHA, "standard", f"needle ![]({_URI})"), (_TEXT_SHA, None, "literal data:image/png;base64")]: + await db.execute( + "INSERT INTO fts_contents (sha256,tier,text,title,author) VALUES (?,?,?,'','')", + (sha, tier, tokenize_for_index(text)), + ) + return db + + +def test_new_parse_and_cached_rebuild_share_image_free_search_text(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """执行真实任务收尾与搜索服务,确认图片不挤占正文预算且重建得到相同文本。""" + result = _result() + before = result.to_json() + + async def parse_result(self: ParseExecutionService, file_row: FileRow, tier: Tier, page_range: str) -> ParseResult: + """仅替换模型推理,保留真实 ParseResult、缓存写出、索引和任务生命周期。""" + return result + + monkeypatch.setattr(ParseExecutionService, "_parse_via_local", parse_result) + + async def scenario() -> None: + """连接真实 SQLite、解析执行器、搜索服务和 CLI 文本格式化。""" + db = DatabaseManager(str(tmp_path / "library.sqlite")) + await db.initialize() + fts = FTSManager(db) + services = create_parse_services(db, fts, ConfigService(db), str(tmp_path), parse_lock_timeout_sec=60) + source = tmp_path / "sample.html" + source.write_text("placeholder") + request = await services.queue.request_parse(str(source), tier="flash") + task = await services.queue.acquire_task() + assert task is not None and await services.execution.process_doc(task) + first = await db.fetchone("SELECT text FROM fts_contents WHERE sha256=?", (request.sha256,)) + assert strip_sep(first["text"]) == render_fts_text(result.middle_json, latex_delimiters=_DELIMITERS) + search = SearchService(db, fts) + for word in ("needle", "recognized", "caption", "middlekeyword", "charttext", "tailkeyword"): + rows, total = await search.search(word) + assert total == 1 + response = SearchResponse(query=word, total=total, results=[SearchResult(**row) for row in rows]) + assert "base64" not in response.model_dump_json() + assert "data:image" not in _render_search_results(response) + for word in ("base64", "png"): + assert await fts.search(word) == [] + await services.queue._rebuild_fts_after_invalidate(request.sha256) + assert await db.fetchone("SELECT text FROM fts_contents WHERE sha256=?", (request.sha256,)) == first + assert result.to_json() == before + assert _URI in result.markdown() + + asyncio.run(scenario()) + + +def test_issue5776_attachment_keeps_images_in_export_but_not_search() -> None: + """用原始 PDF 的真实原生解析验证载荷隔离,不依赖 OCR 模型或 Windows 设备。""" + from mineru.parser import parse + + source = Path(__file__).resolve().parents[1] / "fixtures" / "issue5776" / "source.pdf" + result = parse(source, tier="flash", ocr_mode="txt", image_analysis=False) + exported = result.markdown() + assert "data:image" in exported + indexed = render_fts_text(result.middle_json, latex_delimiters=_DELIMITERS) + assert "测试" in indexed + assert "data:image" not in indexed and "base64" not in indexed + assert result.markdown() == exported + + +def test_search_text_preserves_cross_page_continuation(tmp_path: Path) -> None: + """去除图片不能改变已有跨页正文拼接;缓存重建也必须使用同一整本渲染规则。""" + middle = MiddleJson( + metadata={"file_suffix": "docx", "producer": {"name": "test", "version": "test"}}, + is_full_document=True, + pages=[ + {"page_idx": 0, "blocks": [{"type": "text", "index": 0, "content": [{"type": "text", "content": "跨页"}]}]}, + { + "page_idx": 1, + "blocks": [ + { + "type": "text", + "index": 0, + "continues_prev": True, + "content": [{"type": "text", "content": "关键词"}], + } + ], + }, + ], + ) + expected = render_fts_text(middle, latex_delimiters=_DELIMITERS) + assert expected == "跨页关键词" + + async def scenario() -> None: + """将跨页正文拆成两个真实缓存批次,确认合并后仍能恢复原本的检索文本。""" + db = DatabaseManager(str(tmp_path / "library.sqlite")) + await db.initialize() + await _seed_doc(db) + for index, page in enumerate(middle.pages): + await _cache(db, tmp_path, ParseResult(middle.model_copy(update={"pages": [page]})), "flash", index + 1) + services = create_parse_services(db, FTSManager(db), ConfigService(db), str(tmp_path), parse_lock_timeout_sec=60) + await services.queue._rebuild_fts_after_invalidate(_SHA) + row = await db.fetchone("SELECT text FROM fts_contents WHERE sha256=?", (_SHA,)) + assert strip_sep(row["text"]) == expected + + asyncio.run(scenario()) + + +def test_upgrade_rebuilds_orphan_cache_with_latest_pages_and_keeps_literal_text( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """迁移从最高档位选最新重复页,缺失源文件也可恢复,且不会处理纯文本的字面 URI。""" + + async def scenario() -> None: + """创建旧索引、混合档位和重叠批次,验证事务进度与重复运行。""" + db = await _old_database(tmp_path, monkeypatch) + await _cache(db, tmp_path, _result("lowerword"), "flash", 1) + await _cache(db, tmp_path, _result("obsoleteword"), "standard", 2) + newest = await _cache(db, tmp_path, _result("newestword"), "standard", 3) + await _cache(db, tmp_path, _result("secondpage", 1), "standard", 4) + original = newest.read_bytes() + await db.initialize() + fts = FTSManager(db) + assert await db.fetchall("SELECT sha256 FROM fts_rebuild_queue") == [{"sha256": _SHA}] + assert await fts.search("needle") == [] + await migrate_fts_contents(db, fts, data_dir=str(tmp_path), latex_delimiters=_DELIMITERS) + assert await db.fetchall("SELECT * FROM fts_rebuild_queue") == [] + for word in ("newestword", "secondpage", "caption", "middlekeyword"): + assert await fts.search(word) + for word in ("obsoleteword", "lowerword"): + assert await fts.search(word) == [] + assert [row["sha256"] for row in await fts.search("base64")] == [_TEXT_SHA] + snapshot = await db.fetchall("SELECT * FROM fts_contents ORDER BY sha256") + await db.initialize() + await migrate_fts_contents(db, fts, data_dir=str(tmp_path), latex_delimiters=_DELIMITERS) + assert await db.fetchall("SELECT * FROM fts_contents ORDER BY sha256") == snapshot + assert newest.read_bytes() == original + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("failure", ["missing", "corrupt"]) +def test_upgrade_records_failed_cache_and_retries_without_exposing_old_index( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, failure: str +) -> None: + """缺失或损坏缓存保留待修复标记与原因,恢复后可继续迁移而不丢正文。""" + + async def scenario() -> None: + """保留一页正常缓存,确认迁移不会把部分读取伪报为整份索引恢复。""" + db = await _old_database(tmp_path, monkeypatch) + await _cache(db, tmp_path, _result("firstpage"), "standard", 1) + broken = await _cache(db, tmp_path, _result("repairedword", 1), "standard", 2) + original = broken.read_text() + if failure == "missing": + broken.unlink() + else: + broken.write_text("{broken") + await db.initialize() + fts = FTSManager(db) + old_row = await db.fetchone("SELECT text FROM fts_contents WHERE sha256=?", (_SHA,)) + await migrate_fts_contents(db, fts, data_dir=str(tmp_path), latex_delimiters=_DELIMITERS) + assert (await db.fetchone("SELECT error FROM fts_rebuild_queue WHERE sha256=?", (_SHA,)))["error"] + assert "rebuild deferred" in caplog.text + assert await fts.search("needle") == [] + assert await db.fetchone("SELECT text FROM fts_contents WHERE sha256=?", (_SHA,)) == old_row + broken.write_text(original) + await migrate_fts_contents(db, fts, data_dir=str(tmp_path), latex_delimiters=_DELIMITERS) + assert await fts.search("repairedword") + assert await db.fetchall("SELECT * FROM fts_rebuild_queue") == [] + + asyncio.run(scenario()) + + +def test_index_replacement_and_migration_acknowledgement_are_atomic(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """模拟清除迁移标记时的数据库故障,旧索引和标记必须同时回滚。""" + + async def scenario() -> None: + """使用真实 SQLite 触发器中断事务,随后验证可重试恢复。""" + db = await _old_database(tmp_path, monkeypatch) + await _cache(db, tmp_path, _result(), "standard", 1) + await db.initialize() + old_row = await db.fetchone("SELECT text FROM fts_contents WHERE sha256=?", (_SHA,)) + await db.execute( + "CREATE TRIGGER fail_ack BEFORE DELETE ON fts_rebuild_queue BEGIN SELECT RAISE(ABORT, 'injected'); END" + ) + fts = FTSManager(db) + with pytest.raises(sqlite3.IntegrityError, match="injected"): + await migrate_fts_contents(db, fts, data_dir=str(tmp_path), latex_delimiters=_DELIMITERS) + assert await db.fetchone("SELECT text FROM fts_contents WHERE sha256=?", (_SHA,)) == old_row + assert await db.fetchone("SELECT sha256 FROM fts_rebuild_queue WHERE sha256=?", (_SHA,)) + await db.execute("DROP TRIGGER fail_ack") + await migrate_fts_contents(db, fts, data_dir=str(tmp_path), latex_delimiters=_DELIMITERS) + assert await db.fetchall("SELECT * FROM fts_rebuild_queue") == [] + + asyncio.run(scenario()) + + +def test_doclib_startup_migrates_before_serving_search(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """用真实应用启动与 HTTP 搜索验证升级钩子,而非只测试迁移函数。""" + monkeypatch.setenv("MINERU_HOME", str(tmp_path / "home")) + + async def seed() -> None: + """准备服务即将打开的 v2 数据库和解析缓存。""" + db = await _old_database(tmp_path, monkeypatch) + await _cache(db, tmp_path, _result(), "standard", 1) + + asyncio.run(seed()) + config = Config() + config.doclib.data_dir = str(tmp_path) + config.doclib.sqlite.path = str(tmp_path / "library.sqlite") + app = app_module.create_app(authenticated_doclib_config(config)) + with TestClient(app, headers=DOCLIB_AUTH_HEADERS) as client: + response = client.get("/api/v1/search", params={"query": "middlekeyword"}) + assert response.status_code == 200 + assert response.json()["total"] == 1 + assert "base64" not in response.text + assert client.portal.call(app.state.doclib_state.db.fetchall, "SELECT * FROM fts_rebuild_queue") == [] diff --git a/tests/unittest/test_doclib_telemetry_core.py b/tests/unittest/test_doclib_telemetry_core.py index f5dae90a0c..3db37e28ee 100644 --- a/tests/unittest/test_doclib_telemetry_core.py +++ b/tests/unittest/test_doclib_telemetry_core.py @@ -30,6 +30,9 @@ from mineru.doclib.telemetry import service as telemetry_service_module from mineru.doclib.types import ParseRequest from mineru.doclib.types import TelemetryObservation, TelemetryObservationsRequest +from mineru.doclib.rows import FileRow +from mineru.parser.base import ParseResult +from mineru.types import MiddleJson, PageInfo, Tier def test_bucket_helpers_use_documented_boundaries() -> None: @@ -349,26 +352,16 @@ async def replace(self, **kwargs) -> None: async def upsert_filename(self, *args, **kwargs) -> None: return None - class _Result: - # 写盘后的源数据回填会读取 middle_json 的 metadata 与 extensions,提供最小合法替身。 - middle_json = SimpleNamespace( - metadata=SimpleNamespace(document=None, file_suffix="pdf"), - extensions={}, - ) - - def to_dict(self, *, skip_defaults: bool = True) -> dict: - return {"pages": [{"page_idx": 0, "blocks": []}]} - - def markdown(self, *, add_markers: bool = False) -> str: - return "content" - - def images(self) -> dict[str, bytes]: - """测试替身补齐 ParseResult 图片接口,保持成功路径契约完整。""" - return {} - class _ParseService(ParseExecutionService): - async def _parse_via_local(self, file_row, tier, page_range): - return _Result() + async def _parse_via_local(self, file_row: FileRow, tier: Tier, page_range: str) -> ParseResult: + """仅替换推理,使用真实页面与元数据协议覆盖索引和遥测的成功收尾。""" + return ParseResult( + MiddleJson( + metadata={"file_suffix": "pdf", "producer": {"name": "mineru", "version": "test"}}, + pages=[PageInfo(page_idx=0, blocks=[])], + is_full_document=True, + ) + ) async def _run() -> None: db = DatabaseManager(str(tmp_path / "doclib.db"))