1f64c87d0e
- 任务化清洗流程：Task/TaskManager + BatchProcessor 三方法
- 数据目录规范化：data/{db,sources,backup}
- CLI 入口移进包，注册 jclean 命令
- 工作流测试驱动（tests/test_cleaner_workflow.py）
- 40 测试通过
100 lines
3.5 KiB
Python
100 lines
3.5 KiB
Python
"""
|
||
Cleaner 模块单元测试
|
||
"""
|
||
import pytest
|
||
|
||
from pl_japanese.cleaner import Classifier, Aligner, PinyinMaker
|
||
|
||
@pytest.fixture
|
||
def classifier():
|
||
"""创建分类器实例"""
|
||
aligner = Aligner()
|
||
pinyin_maker = PinyinMaker()
|
||
return Classifier(aligner, pinyin_maker)
|
||
|
||
class TestRules:
|
||
"""规则测试"""
|
||
|
||
def test_skip_no_hanzi(self, classifier):
|
||
"""规则1：无汉字词跳过"""
|
||
bucket, out, note = classifier.classify("IT", "アイティー")
|
||
assert bucket == 'skip'
|
||
assert "跳过" in note
|
||
|
||
bucket, out, note = classifier.classify("Japan Railways", "ジャパンレールウェイズ")
|
||
assert bucket == 'skip'
|
||
|
||
bucket, out, note = classifier.classify("％", "パーセント")
|
||
assert bucket == 'skip'
|
||
|
||
def test_mixed_letter_kanji(self, classifier):
|
||
"""规则2：含字母+汉字的混合词正常处理"""
|
||
bucket, out, note = classifier.classify("ＪＣ自動車", "ジェーシーじどうしゃ")
|
||
assert bucket in ('auto', 'pinyin', 'special')
|
||
assert "ＪＣ|自|動|車" in out or "JC|自|動|車" in out
|
||
|
||
def test_tilde_handling(self, classifier):
|
||
"""规则3：含~自动处理"""
|
||
# 成功分割
|
||
bucket, out, note = classifier.classify("経営～", "けいえいします")
|
||
assert bucket in ('auto', 'pinyin')
|
||
assert "経|営" in out
|
||
assert "けい|えい" in out
|
||
|
||
# ~作为通配符,吸收非汉字部分(おせち),汉字部分正常对齐
|
||
bucket, out, note = classifier.classify("～料理", "おせちりょうり")
|
||
assert bucket in ('auto', 'pinyin')
|
||
assert "料|理" in out
|
||
assert "りょう|り" in out
|
||
|
||
def test_repeat_mark_expansion(self, classifier):
|
||
"""规则6：々展开"""
|
||
bucket, out, note = classifier.classify("我々", "われわれ")
|
||
assert "我|我" in out or bucket != 'skip'
|
||
|
||
bucket, out, note = classifier.classify("佐々木", "ささき")
|
||
assert "佐|佐|木" in out or bucket != 'skip'
|
||
|
||
class TestAlignment:
|
||
"""对齐测试"""
|
||
|
||
def test_simple_alignment(self, classifier):
|
||
"""简单对齐"""
|
||
bucket, out, note = classifier.classify("中国", "ちゅうごく")
|
||
assert bucket in ('auto', 'pinyin')
|
||
assert "中|国" in out
|
||
assert "ちゅう|ごく" in out
|
||
|
||
def test_mixed_kana(self, classifier):
|
||
"""含假名送り仮名"""
|
||
bucket, out, note = classifier.classify("お父さん", "おとうさん")
|
||
# 熟字训，可能分割失败
|
||
assert bucket in ('auto', 'pinyin', 'split')
|
||
|
||
class TestPinyin:
|
||
"""拼音测试"""
|
||
|
||
def test_polyphone_detection(self, classifier):
|
||
"""多音字检测"""
|
||
bucket, out, note = classifier.classify("行動", "こうどう")
|
||
# '行' 是多音字
|
||
if bucket == 'pinyin':
|
||
assert "多音字" in note or "行" in note
|
||
|
||
class TestTokenSplit:
|
||
"""Token 切分测试"""
|
||
|
||
def test_token_split(self, classifier):
|
||
"""测试 token 切分"""
|
||
tokens = classifier.split_kanji_tokens("お父さん")
|
||
# 预期: ['お', '父', 'さん']
|
||
assert len(tokens) == 3
|
||
|
||
tokens = classifier.split_kanji_tokens("ＪＣ自動車")
|
||
# 预期: ['ＪＣ', '自', '動', '車'] 或类似
|
||
assert '自' in tokens
|
||
assert '動' in tokens
|
||
|
||
if __name__ == '__main__':
|
||
pytest.main([__file__, '-v'])
|