""" Cleaner 模块单元测试 """ import pytest from pl_japanese.cleaner import Classifier, Aligner, PinyinMaker @pytest.fixture def classifier(): """创建分类器实例""" aligner = Aligner() pinyin_maker = PinyinMaker() return Classifier(aligner, pinyin_maker) class TestRules: """规则测试""" def test_skip_no_hanzi(self, classifier): """规则1:无汉字词跳过""" bucket, out, note = classifier.classify("IT", "アイティー") assert bucket == 'skip' assert "跳过" in note bucket, out, note = classifier.classify("Japan Railways", "ジャパンレールウェイズ") assert bucket == 'skip' bucket, out, note = classifier.classify("%", "パーセント") assert bucket == 'skip' def test_mixed_letter_kanji(self, classifier): """规则2:含字母+汉字的混合词正常处理""" bucket, out, note = classifier.classify("JC自動車", "ジェーシーじどうしゃ") assert bucket in ('auto', 'pinyin', 'special') assert "JC|自|動|車" in out or "JC|自|動|車" in out def test_tilde_handling(self, classifier): """规则3:含~自动处理""" # 成功分割 bucket, out, note = classifier.classify("経営~", "けいえいします") assert bucket in ('auto', 'pinyin') assert "経|営" in out assert "けい|えい" in out # ~作为通配符,吸收非汉字部分(おせち),汉字部分正常对齐 bucket, out, note = classifier.classify("~料理", "おせちりょうり") assert bucket in ('auto', 'pinyin') assert "料|理" in out assert "りょう|り" in out def test_repeat_mark_expansion(self, classifier): """规则6:々展开""" bucket, out, note = classifier.classify("我々", "われわれ") assert "我|我" in out or bucket != 'skip' bucket, out, note = classifier.classify("佐々木", "ささき") assert "佐|佐|木" in out or bucket != 'skip' class TestAlignment: """对齐测试""" def test_simple_alignment(self, classifier): """简单对齐""" bucket, out, note = classifier.classify("中国", "ちゅうごく") assert bucket in ('auto', 'pinyin') assert "中|国" in out assert "ちゅう|ごく" in out def test_mixed_kana(self, classifier): """含假名送り仮名""" bucket, out, note = classifier.classify("お父さん", "おとうさん") # 熟字训,可能分割失败 assert bucket in ('auto', 'pinyin', 'split') class TestPinyin: """拼音测试""" def test_polyphone_detection(self, classifier): """多音字检测""" bucket, out, note = classifier.classify("行動", "こうどう") # '行' 是多音字 if bucket == 'pinyin': assert "多音字" in note or "行" in note class TestTokenSplit: """Token 切分测试""" def test_token_split(self, classifier): """测试 token 切分""" tokens = classifier.split_kanji_tokens("お父さん") # 预期: ['お', '父', 'さん'] assert len(tokens) == 3 tokens = classifier.split_kanji_tokens("JC自動車") # 预期: ['JC', '自', '動', '車'] 或类似 assert '自' in tokens assert '動' in tokens if __name__ == '__main__': pytest.main([__file__, '-v'])