""" 词条分类器 根据规则将词条分类到不同桶 """ from typing import Tuple, List from .rules import * from .aligner import Aligner from .pinyin_maker import PinyinMaker import re class Classifier: def __init__(self, aligner: Aligner, pinyin_maker: PinyinMaker): self.aligner = aligner self.pinyin_maker = pinyin_maker def split_kanji_tokens(self, text: str) -> List[str]: """ 将汉字段切分为 tokens: - 每个汉字单独一段 - ~ 与相邻非汉字合并(~お→~お,い~→い~) - 其余连续非汉字(假名/片假名/字母/记号)合并成一段 示例: - ~お伝えください → ['~お', '伝', 'えください'] - つまみ食い~ → ['つまみ', '食', 'い~'] - ~人 → ['~', '人'](~后直接是汉字,独立) """ tokens = [] buf = '' for ch in text: if HANZI.match(ch): # 遇到汉字:先把缓冲区的非汉字段输出 if buf: tokens.append(buf) buf = '' tokens.append(ch) else: # 非汉字(包括~、假名、片假名、字母、符号):连续合并 buf += ch if buf: tokens.append(buf) return tokens def classify(self, kanji_part: str, kana_part: str) -> Tuple[str, str, str]: """ 分类词条 返回: (bucket, 输出字符串, 备注) """ # 规则1:无汉字 → skip(三段格式,拼音留空) if should_skip(kanji_part): return 'skip', f"{kanji_part}:{kana_part}:", "无汉字,跳过" # 清理 ~ 标记 has_tilde_mark = has_tilde(kanji_part) kanji_clean = clean_tilde(kanji_part) # 展开々 kanji_clean = expand_repeat_marks(kanji_clean) # 检测重复词(如 1人1人、時時):前半部分 == 后半部分 # 这种情况应该作为整体重复,而不是逐字符分段 if len(kanji_clean) >= 2 and len(kanji_clean) % 2 == 0: half_k = len(kanji_clean) // 2 if kanji_clean[:half_k] == kanji_clean[half_k:]: # 重复词:切分为两个相同的单元 tokens = [kanji_clean[:half_k], kanji_clean[half_k:]] # 假名也可能是重复的(如 ひとりひとり),尝试对半分割 kana_clean = clean_tilde(kana_part) if len(kana_clean) >= 2 and len(kana_clean) % 2 == 0: half_kn = len(kana_clean) // 2 if kana_clean[:half_kn] == kana_clean[half_kn:]: # 假名也是重复的,直接对齐 kana_tokens = [kana_clean[:half_kn], kana_clean[half_kn:]] pys, multis = self.pinyin_maker.make_pinyin(tokens, kana_tokens) out = f"{'|'.join(tokens)}:{'|'.join(kana_tokens)}:{'|'.join(pys)}" if multis: return 'pinyin', out, f"重复词+多音字({','.join(multis)})" return 'auto', out, "" # 假名不是重复的,走正常流程尝试对齐 if is_verbish(kana_part, has_tilde_mark): return self._handle_verb(kanji_part, kana_part, tokens, has_tilde_mark) return self._handle_normal(kanji_part, kana_part, tokens) # 判断是否动词相关 is_verb = is_verbish(kana_part, has_tilde_mark) # 切分 tokens tokens = self.split_kanji_tokens(kanji_clean) # 处理动词 if is_verb: return self._handle_verb(kanji_part, kana_part, tokens, has_tilde_mark) # 普通词:对齐 return self._handle_normal(kanji_part, kana_part, tokens) def _split_hint(self, tokens, kana_orig): """ 生成"无法拆分"时的提示语。 当假名字符数 < 汉字段数时,每个汉字至少需要1个假名也无法满足, 判定为原始数据假名缺失,给出更明确的提示。 """ # 统计汉字段中的汉字总数(非汉字段不计入,如数字/字母) n_kanji = sum(len(t) for t in tokens if t and HANZI.match(t[0])) # 假名字符数(去除~标记) kana_len = len(clean_tilde(kana_orig)) if n_kanji > 0 and kana_len < n_kanji: # 假名字符数比汉字数还少:每个汉字至少1个假名都凑不齐,必定缺失 return f"假名缺失(汉字{n_kanji}字/假名仅{kana_len}字),请核对原始数据并补全" # 无法精确判断,提醒两种可能 return "请手动分割假名(若段数凑不齐,可能是原始数据缺字)" def _filter_symbols(self, tokens, kana_tokens): """从输出中过滤纯记号段(『』「」等),这些段应零宽,不参与拼音输出。""" out_k, out_kn = [], [] for tk, kn in zip(tokens, kana_tokens): if self.aligner._pure_symbol(tk): continue out_k.append(tk) out_kn.append(kn) return out_k, out_kn def _handle_tilde(self, kanji_orig, kana_orig): """含~处理:~作为任意长通配符对齐,~对应的汉字/假名段不输出。 原理: - 汉字部分保留原顺序,~处替换为占位符,与其他非汉字段一起做非汉字混合段 - 分词器把 ~ 段与相邻假名合并成非汉字段(任意长匹配) - align 找到解后,把含 ~ 的非汉字段从输出中删除 返回 (bucket, out, note) 或 None(无解/多解) """ # 用 kanji_orig(未清理~)做分词,~会被归入非汉字段 kanji_expanded = expand_repeat_marks(kanji_orig) tokens_with_tilde = self.split_kanji_tokens(kanji_expanded) # 对齐 sols = self.aligner.align(tokens_with_tilde, kana_orig) if len(sols) == 0: return None # 过滤:只保留长度匹配的解(aligner可能返回长度不匹配的解) valid_sols = [s for s in sols if len(list(s)) == len(tokens_with_tilde)] if len(valid_sols) == 0: return None # 多解时:选择"汉字段假名总长最大"的解(汉字取完整读音,~段取残余) if len(valid_sols) > 1: def hanzi_kana_len(sol): total = 0 for tk, kn in zip(tokens_with_tilde, sol): if self.aligner.hanzi.search(tk): total += len(kn) return total best = max(valid_sols, key=lambda s: (hanzi_kana_len(s), tuple(-len(x) for x in s))) kana_tokens = list(best) else: kana_tokens = list(next(iter(valid_sols))) # ~ 当作普通非汉字段保留(占一段,拼音留空),只丢弃纯记号段 # 例:~人:アメリカじん: → ~|人:アメリカ|じん:|ren out_kanji = [] out_kana = [] for tk, kn in zip(tokens_with_tilde, kana_tokens): if self.aligner._pure_symbol(tk): continue # ~ 段规范化为全角 ~ if '~' in tk: tk = tk.replace('~', '~') out_kanji.append(tk) out_kana.append(kn) if not out_kanji: return None pys, multis = self.pinyin_maker.make_pinyin(out_kanji, out_kana) out = f"{'|'.join(out_kanji)}:{'|'.join(out_kana)}:{'|'.join(pys)}" if multis: return 'pinyin', out, f"含~+多音字({','.join(multis)}),请确认" return 'auto', out, "" def _is_keigo(self, kanji_clean: str) -> bool: """判断是否敬语(お/ご 前缀的完整寒暄表达)。 规则5:带 お/ご 等构成完整寒暄的整句保留 します/でした, 不转原形(お願いします、お邪魔します 等)。 """ # 去掉 ~/~/| 分隔符后,以 お/ご 开头 → 敬语 cleaned = kanji_clean.replace('~', '').replace('~', '').replace('|', '') return cleaned.startswith(('お', 'ご')) def _convert_suru_masu(self, kanji_clean: str, kana_orig: str): """サ変動詞的ます形(します)→原型(する),敬语除外。 含 ~ 的する動詞走规则3(tilde 通配符),不会触发规则4的 ます→辞書形转换;且 します 是する的不规则ます形,masu_to_dict_form 的 i段→u段映射处理不了。这里针对「~ 段假名以します结尾」单独转换。 返回 None 表示无需转换(不是します结尾,或敬语保留ます形)。 """ if not kana_orig.endswith('します'): return None if self._is_keigo(kanji_clean): return None # します(3字符) → する(2字符) return kana_orig[:-3] + 'する' def _handle_verb(self, kanji_orig, kana_orig, tokens, has_tilde_mark): """处理含~的词和普通动词/复合词""" if has_tilde_mark and RULES['auto_process_tilde']: # 规则3:含~ → ~作为任意长通配符对齐 # 先做サ変動詞 します→する 转换(敬语除外),使 ~ 段吸收「する」而非「します」 kanji_clean_for_keigo = clean_tilde(kanji_orig) if RULES['convert_masu_form']: converted = self._convert_suru_masu(kanji_clean_for_keigo, kana_orig) if converted: kana_orig = converted result = self._handle_tilde(kanji_orig, kana_orig) if result is not None: return result # 兜底:无法自动对齐,给出汉字分段和拼音,假名保持完整待手工分割 pys, _ = self.pinyin_maker.make_pinyin(tokens, kana_tokens=None) out = f"{'|'.join(tokens)}:{kana_orig}:{'|'.join(pys)}" return 'split', out, "含~但假名无法分割,请手动分割假名" # ます形转换(规则4) if RULES['convert_masu_form'] and kana_orig.endswith('ます'): return self._convert_masu_form(kanji_orig, kana_orig, tokens) # 尝试自动对齐完整词(按汉字/非汉字分段) sols = self.aligner.align(tokens, kana_orig) if len(sols) == 1: kana_tokens = list(next(iter(sols))) pys, multis = self.pinyin_maker.make_pinyin(tokens, kana_tokens) out = f"{'|'.join(tokens)}:{'|'.join(kana_tokens)}:{'|'.join(pys)}" if multis: return 'pinyin', out, f"多音字({','.join(multis)}),请确认" return 'auto', out, "" # 无法自动对齐:给出汉字分段和拼音,假名保持完整待手工分割 # 输出格式:北|京|大|学:ペキンだいがく:bei|jing|da|xue # 请手动分割假名 pys, _ = self.pinyin_maker.make_pinyin(tokens, kana_tokens=None) out = f"{'|'.join(tokens)}:{kana_orig}:{'|'.join(pys)}" return 'split', out, self._split_hint(tokens, kana_orig) def _convert_masu_form(self, kanji_orig, kana_orig, tokens): """ます形转辞書形(规则4)- 去掉[原]前缀,按普通词处理""" dict_result, vtype = self.aligner.masu_to_dict_form(kana_orig) # 展开々后的汉字段(用于同步转换) kanji_clean = expand_repeat_marks(clean_tilde(kanji_orig)) if vtype == 'ambiguous': # 歧义:直接按原形分割,能分就分,不能分进待确认 sols = self.aligner.align(tokens, kana_orig) if len(sols) == 1: kana_tokens = list(next(iter(sols))) pys, multis = self.pinyin_maker.make_pinyin(tokens, kana_tokens) out = f"{'|'.join(tokens)}:{'|'.join(kana_tokens)}:{'|'.join(pys)}" if multis: return 'pinyin', out, f"多音字({','.join(multis)}),请确认" return 'auto', out, "" pys, _ = self.pinyin_maker.make_pinyin(tokens, kana_tokens=None) out = f"{'|'.join(tokens)}:{kana_orig}:{'|'.join(pys)}" return 'split', out, self._split_hint(tokens, kana_orig) if vtype in ('ichidan', 'godan'): # 汉字段的送り仮名同步转换 if kanji_clean.endswith('ます'): if vtype == 'ichidan': kanji_dict = kanji_clean[:-2] + 'る' else: stem = kanji_clean[:-2] if stem and stem[-1] in I_TO_U_MAP: kanji_dict = stem[:-1] + I_TO_U_MAP[stem[-1]] else: kanji_dict = kanji_clean else: kanji_dict = kanji_clean tokens_dict = self.split_kanji_tokens(kanji_dict) sols = self.aligner.align(tokens_dict, dict_result) if len(sols) == 1: kana_tokens = list(next(iter(sols))) pys, multis = self.pinyin_maker.make_pinyin(tokens_dict, kana_tokens) out = f"{'|'.join(tokens_dict)}:{'|'.join(kana_tokens)}:{'|'.join(pys)}" if multis: return 'pinyin', out, f"多音字({','.join(multis)})" return 'auto', out, "" else: # 转换后无法分割:回退到原形,给出汉字分段和拼音 pys, _ = self.pinyin_maker.make_pinyin(tokens, kana_tokens=None) out = f"{'|'.join(tokens)}:{kana_orig}:{'|'.join(pys)}" return 'split', out, self._split_hint(tokens, kana_orig) # failed:词典查不到,按原形分割 sols = self.aligner.align(tokens, kana_orig) if len(sols) == 1: kana_tokens = list(next(iter(sols))) pys, multis = self.pinyin_maker.make_pinyin(tokens, kana_tokens) out = f"{'|'.join(tokens)}:{'|'.join(kana_tokens)}:{'|'.join(pys)}" if multis: return 'pinyin', out, f"多音字({','.join(multis)}),请确认" return 'auto', out, "" pys, _ = self.pinyin_maker.make_pinyin(tokens, kana_tokens=None) out = f"{'|'.join(tokens)}:{kana_orig}:{'|'.join(pys)}" return 'split', out, self._split_hint(tokens, kana_orig) def _handle_normal(self, kanji_orig, kana_orig, tokens): """处理普通词""" # 单汉字段:整个假名必然归给这唯一的汉字,直接1对1对齐,无需aligner猜读音 # (如 望:のぞみ、金:キム 这类训读/片假名单字) if len(tokens) == 1 and HANZI.match(tokens[0]): fk, fkn = self._filter_symbols(tokens, [kana_orig]) pys, multis = self.pinyin_maker.make_pinyin(fk, fkn) out = f"{'|'.join(fk)}:{'|'.join(fkn)}:{'|'.join(pys)}" if multis: return 'pinyin', out, f"多音字({','.join(multis)}),请确认" return 'auto', out, "" sols = self.aligner.align(tokens, kana_orig) if len(sols) == 0: # 假名无法分割:给出汉字分段和拼音,假名保持完整 pys, _ = self.pinyin_maker.make_pinyin(tokens, kana_tokens=None) out = f"{'|'.join(tokens)}:{kana_orig}:{'|'.join(pys)}" return 'split', out, self._split_hint(tokens, kana_orig) if len(sols) > 1: # 多解:选择最平衡的分割(各段长度方差最小) # 方差相同时,优先选择首段较长的(更符合音读习惯) def score_solution(sol): lengths = [len(s) for s in sol] avg = sum(lengths) / len(lengths) variance = sum((l - avg) ** 2 for l in lengths) / len(lengths) # 返回 (方差, -首段长度),用于排序(方差小优先,首段长优先) return (variance, -lengths[0] if lengths else 0) kana_tokens = list(min(sols, key=score_solution)) fk, fkn = self._filter_symbols(tokens, kana_tokens) pys, multis = self.pinyin_maker.make_pinyin(fk, fkn) out = f"{'|'.join(fk)}:{'|'.join(fkn)}:{'|'.join(pys)}" if multis: return 'pinyin', out, f"多解+多音字({','.join(multis)})" return 'pinyin', out, "多解,已取第一个" # 唯一解 kana_tokens = list(next(iter(sols))) fk, fkn = self._filter_symbols(tokens, kana_tokens) pys, multis = self.pinyin_maker.make_pinyin(fk, fkn) out = f"{'|'.join(fk)}:{'|'.join(fkn)}:{'|'.join(pys)}" # 覆写tokens/kana为过滤后的,供后续多音字/字母判定使用 tokens = fk kana_tokens = fkn if multis: return 'pinyin', out, f"多音字({','.join(multis)}),请确认" # 含字母/片假名:标记确认 has_latin = LATIN.search(kanji_orig) has_kata = KATA.search(kanji_orig) if has_latin or has_kata: note = [] if has_latin: note.append("含字母") if has_kata: note.append("含片假名") return 'special', out, ','.join(note) + ",请确认" return 'auto', out, ""