Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions astrbot/core/knowledge_base/kb_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,17 @@ async def upload_document(
parse_result = await parser.parse(file_content, file_name)
except KnowledgeBaseUploadError:
raise
except ModuleNotFoundError as exc:
if exc.name == "markitdown_no_magika":
raise KnowledgeBaseUploadError(
stage="parsing",
user_message=(
"文档解析失败:缺少解析该文件格式所需的依赖"
" markitdown-no-magika,请安装后重试。"
),
details={"file_name": file_name},
) from exc
raise
except Exception as exc:
raise KnowledgeBaseUploadError(
stage="parsing",
Expand Down
6 changes: 5 additions & 1 deletion astrbot/core/knowledge_base/parsers/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@


async def select_parser(ext: str) -> BaseParser:
if ext in {".md", ".txt", ".markdown", ".rst", ".adoc", ".xlsx", ".docx", ".xls"}:
if ext in {".txt", ".md", ".markdown"}:
from .text_parser import TextParser

return TextParser()
if ext in {".rst", ".adoc", ".xlsx", ".docx", ".xls"}:
from .markitdown_parser import MarkitdownParser

return MarkitdownParser()
Expand Down
167 changes: 167 additions & 0 deletions tests/unit/test_kb_select_parser.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
"""
Unit tests for knowledge base parser selection and missing-dependency errors.

Covers:
1. .txt / .md / .markdown are routed to the stdlib TextParser (no third-party
dependency required), so plain-text uploads work without markitdown.
2. .rst / .adoc keep routing to MarkitdownParser (requires markitdown).
3. upload_document surfaces a clear missing-dependency message when
markitdown-no-magika is absent, instead of the generic parse failure.
4. A missing unrelated module (e.g. pypdf) is not mislabeled as a markitdown
problem.
"""

import sys
import types
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from astrbot.core.exceptions import KnowledgeBaseUploadError
from astrbot.core.knowledge_base.models import KnowledgeBase
from astrbot.core.knowledge_base.parsers.text_parser import TextParser
from astrbot.core.knowledge_base.parsers.util import select_parser


@pytest.fixture
def stub_provider_manager_module():
"""Stub provider manager module to avoid circular imports in unit tests."""
original_module = sys.modules.get("astrbot.core.provider.manager")
stub_module = types.ModuleType("astrbot.core.provider.manager")

class ProviderManager: ...

setattr(stub_module, "ProviderManager", ProviderManager)
sys.modules["astrbot.core.provider.manager"] = stub_module

# Drop already-imported modules that transitively need ProviderManager so
# they re-import against the stub.
to_drop = [
name
for name in list(sys.modules)
if name.startswith("astrbot.core.knowledge_base.kb_helper")
or name.startswith("astrbot.core.knowledge_base.kb_mgr")
]
for name in to_drop:
sys.modules.pop(name, None)

try:
yield
finally:
if original_module is not None:
sys.modules["astrbot.core.provider.manager"] = original_module
else:
sys.modules.pop("astrbot.core.provider.manager", None)


def _import_kb_helper():
from astrbot.core.knowledge_base.kb_helper import KBHelper

return KBHelper


@pytest.mark.asyncio
@pytest.mark.parametrize("ext", [".txt", ".md", ".markdown"])
async def test_select_parser_routes_plain_text_to_text_parser(ext):
parser = await select_parser(ext)

assert isinstance(parser, TextParser)


@pytest.mark.asyncio
@pytest.mark.parametrize("ext", [".rst", ".adoc"])
async def test_select_parser_keeps_markup_formats_on_markitdown(ext):
pytest.importorskip("markitdown_no_magika")

from astrbot.core.knowledge_base.parsers.markitdown_parser import MarkitdownParser

parser = await select_parser(ext)

assert isinstance(parser, MarkitdownParser)


@pytest.mark.asyncio
async def test_text_parser_decodes_plain_text_file():
result = await TextParser().parse("你好 world\nsecond line".encode(), "note.txt")
assert result.media == []
assert result.text == "你好 world\nsecond line"

gbk_result = await TextParser().parse("中文内容".encode("gbk"), "note.txt")
assert gbk_result.text == "中文内容"


def _make_upload_helper(tmp_path: Path):
KBHelper = _import_kb_helper()

helper = KBHelper.__new__(KBHelper)
helper.kb = KnowledgeBase(
kb_name="Test KB",
description="",
embedding_provider_id="emb",
)
helper.kb_db = MagicMock()
helper.vec_db = AsyncMock()
helper.vec_db.delete_documents = AsyncMock()
helper.kb_medias_dir = tmp_path / "medias"
helper.chunker = AsyncMock()
return helper


@pytest.mark.asyncio
async def test_upload_document_reports_missing_markitdown_dependency(
tmp_path: Path,
stub_provider_manager_module,
) -> None:
helper = _make_upload_helper(tmp_path)

with (
patch(
"astrbot.core.knowledge_base.kb_helper.select_parser",
new=AsyncMock(
side_effect=ModuleNotFoundError(
"No module named 'markitdown_no_magika'",
name="markitdown_no_magika",
),
),
),
patch.object(helper, "_ensure_vec_db", new=AsyncMock()),
pytest.raises(KnowledgeBaseUploadError) as exc_info,
):
await helper.upload_document(
file_name="guide.docx",
file_content=b"fake",
file_type="docx",
)

assert exc_info.value.stage == "parsing"
assert "markitdown-no-magika" in exc_info.value.user_message


@pytest.mark.asyncio
async def test_upload_document_passes_through_unrelated_missing_module(
tmp_path: Path,
stub_provider_manager_module,
) -> None:
helper = _make_upload_helper(tmp_path)

with (
patch(
"astrbot.core.knowledge_base.kb_helper.select_parser",
new=AsyncMock(
side_effect=ModuleNotFoundError(
"No module named 'pypdf'",
name="pypdf",
),
),
),
patch.object(helper, "_ensure_vec_db", new=AsyncMock()),
pytest.raises(ModuleNotFoundError) as exc_info,
):
await helper.upload_document(
file_name="guide.pdf",
file_content=b"fake",
file_type="pdf",
)

assert exc_info.value.name == "pypdf"
Loading