1f64c87d0e
- 任务化清洗流程：Task/TaskManager + BatchProcessor 三方法
- 数据目录规范化：data/{db,sources,backup}
- CLI 入口移进包，注册 jclean 命令
- 工作流测试驱动（tests/test_cleaner_workflow.py）
- 40 测试通过
231 lines
8.7 KiB
Python
231 lines
8.7 KiB
Python
"""
|
||
清洗工作流测试（任务驱动 + 人工介入协作）
|
||
|
||
本文件把"处理 → review → 重跑 → 合并"协作流程写成可运行的测试。
|
||
每个测试用隔离的临时任务目录（tmp_path），不碰真实数据（legacy_batch1 / data/db）。
|
||
|
||
设计要点（体现工作流本质）：
|
||
- 工作流需要人工介入：review 桶里的条目由人核对，测试用"模拟人工确认"占位
|
||
- merge 有授权门禁：review 未清零时 merge_all 必须拒绝
|
||
- 条数铁律：输入有效行数 == 各桶输出之和
|
||
- 状态机流转：created → processing → reviewing → ready → merged
|
||
|
||
日常协作（真实数据）请直接用 driver 函数在这里加临时测试驱动，
|
||
或用 `python -m pl_japanese.cleaner.cli` / `jclean` 命令行。
|
||
"""
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from pl_japanese.cleaner import BatchProcessor, TaskManager, Task
|
||
from pl_japanese.cleaner.task import (
|
||
STATUS_CREATED, STATUS_REVIEWING, STATUS_READY, STATUS_MERGED,
|
||
)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 夹具：隔离的任务环境
|
||
# --------------------------------------------------------------------------
|
||
@pytest.fixture
|
||
def isolated_env(tmp_path):
|
||
"""
|
||
构建一个完全隔离的任务环境：
|
||
- tasks_root 在 tmp_path 下
|
||
- 权威库（main/skipped）也在 tmp_path 下（不碰 data/db）
|
||
- 数据源是临时写入的小样本
|
||
返回 (tasks_root, main, skipped, make_source)
|
||
"""
|
||
tasks_root = tmp_path / "tasks"
|
||
main = tmp_path / "db" / "vocabulary.txt"
|
||
skipped = tmp_path / "db" / "skipped.txt"
|
||
main.parent.mkdir(parents=True, exist_ok=True)
|
||
main.write_text("", encoding="utf-8", newline="\n")
|
||
skipped.write_text("", encoding="utf-8", newline="\n")
|
||
|
||
def make_source(lines, name="source.txt"):
|
||
p = tmp_path / name
|
||
p.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n")
|
||
return p
|
||
|
||
return tasks_root, main, skipped, make_source
|
||
|
||
|
||
def _new_task(isolated_env, source_lines, count=100):
|
||
tasks_root, main, skipped, make_source = isolated_env
|
||
src = make_source(source_lines)
|
||
tm = TaskManager(tasks_root=str(tasks_root))
|
||
task = tm.create_task(
|
||
task_id="t1",
|
||
source=str(src),
|
||
name="测试任务",
|
||
start_line=1,
|
||
count=count,
|
||
main=str(main),
|
||
skipped=str(skipped),
|
||
)
|
||
return tm, task
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 任务生命周期
|
||
# --------------------------------------------------------------------------
|
||
def test_task_create_and_load(isolated_env):
|
||
"""创建任务后可从磁盘重新加载，配置一致"""
|
||
tm, task = _new_task(isolated_env, ["日本語:にほんご:"])
|
||
assert task.state.status == STATUS_CREATED
|
||
|
||
reloaded = tm.load_task("t1")
|
||
assert reloaded.task_id == "t1"
|
||
assert reloaded.config.source == task.config.source
|
||
assert reloaded.config.main == task.config.main
|
||
assert reloaded.state.status == STATUS_CREATED
|
||
|
||
|
||
def test_multi_task_isolation(isolated_env):
|
||
"""多任务并存：各自独立目录，互不干扰"""
|
||
tasks_root, main, skipped, make_source = isolated_env
|
||
tm = TaskManager(tasks_root=str(tasks_root))
|
||
s1 = make_source(["学生:がくせい:"], "s1.txt")
|
||
s2 = make_source(["先生:せんせい:"], "s2.txt")
|
||
t1 = tm.create_task("task_a", str(s1), main=str(main), skipped=str(skipped))
|
||
t2 = tm.create_task("task_b", str(s2), main=str(main), skipped=str(skipped))
|
||
|
||
assert Path(t1.task_dir) != Path(t2.task_dir)
|
||
ids = {s["task_id"] for s in tm.list_task_summaries()}
|
||
assert ids == {"task_a", "task_b"}
|
||
|
||
|
||
def test_create_duplicate_rejected(isolated_env):
|
||
"""重复 task_id 创建被拒绝"""
|
||
tm, task = _new_task(isolated_env, ["日本:にほん:"])
|
||
with pytest.raises(FileExistsError):
|
||
tm.create_task("t1", source=task.config.source)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 处理与状态流转
|
||
# --------------------------------------------------------------------------
|
||
def test_process_count_invariant(isolated_env):
|
||
"""条数铁律：有效输入行数 == 各桶输出之和"""
|
||
lines = [
|
||
"中国人:ちゅうごくじん:",
|
||
"日本人:にほんじん:",
|
||
"学生:がくせい:",
|
||
"", # 空行不计
|
||
"先生:せんせい:",
|
||
]
|
||
tm, task = _new_task(isolated_env, lines)
|
||
bp = BatchProcessor(task)
|
||
result = bp.process_batch()
|
||
|
||
assert result["valid_lines"] == 4
|
||
assert result["output_total"] == 4
|
||
assert result["validation"] is True
|
||
|
||
|
||
def test_process_all_auto_goes_ready(isolated_env):
|
||
"""全部自动处理（无 review）→ 任务状态直接 ready"""
|
||
lines = ["日本人:にほんじん:", "中国人:ちゅうごくじん:"]
|
||
tm, task = _new_task(isolated_env, lines)
|
||
bp = BatchProcessor(task)
|
||
result = bp.process_batch()
|
||
|
||
assert result["buckets"]["auto"] == 2
|
||
assert result["status"] == STATUS_READY
|
||
# 从磁盘复核状态已持久化
|
||
assert tm.load_task("t1").state.status == STATUS_READY
|
||
|
||
|
||
def test_process_with_review_goes_reviewing(isolated_env):
|
||
"""产生 review 桶 → 任务状态 reviewing"""
|
||
# 多音字/字母混合等会进 review；用一个含字母的确保进 special
|
||
lines = ["日本人:にほんじん:", "IT:アイティー:"]
|
||
tm, task = _new_task(isolated_env, lines)
|
||
bp = BatchProcessor(task)
|
||
result = bp.process_batch()
|
||
|
||
review_total = sum(
|
||
result["buckets"][b] for b in ("pinyin", "split", "verb", "special")
|
||
)
|
||
# skip 也可能吃掉纯字母词；只要不是全部 auto 即可能有 review
|
||
if review_total > 0:
|
||
assert result["status"] == STATUS_REVIEWING
|
||
else:
|
||
assert result["status"] == STATUS_READY
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 合并授权门禁
|
||
# --------------------------------------------------------------------------
|
||
def test_merge_rejected_when_review_pending(isolated_env):
|
||
"""review 未清零时 merge_all 必须拒绝（授权门禁的前置约束）"""
|
||
tm, task = _new_task(isolated_env, ["日本人:にほんじん:"])
|
||
bp = BatchProcessor(task)
|
||
bp.process_batch()
|
||
|
||
# 人为往 review 桶塞一条，模拟待确认
|
||
bp.review_files["pinyin"].write_text("行|列:こう|れつ:hang|lie\n",
|
||
encoding="utf-8", newline="\n")
|
||
result = bp.merge_all(dry_run=True)
|
||
assert "error" in result
|
||
assert result["total_review"] > 0
|
||
|
||
|
||
def test_merge_final_dedup_and_status(isolated_env):
|
||
"""review 清零后合并到最终库：去重 + 状态变 merged + 单批清空"""
|
||
lines = ["日本人:にほんじん:", "中国人:ちゅうごくじん:"]
|
||
tm, task = _new_task(isolated_env, lines)
|
||
bp = BatchProcessor(task)
|
||
bp.process_batch()
|
||
assert bp.get_status()["ready_to_merge"] is True
|
||
|
||
# 首次合并
|
||
r1 = bp.merge_all(dry_run=False)
|
||
assert r1["merged_auto"] == 2
|
||
assert r1["single_batch_cleared"] is True
|
||
assert tm.load_task("t1").state.status == STATUS_MERGED
|
||
assert r1["main_total"] == 2
|
||
|
||
|
||
def test_merge_idempotent(isolated_env):
|
||
"""幂等：同一批数据再次进入主库不会重复"""
|
||
lines = ["日本人:にほんじん:"]
|
||
tm, task = _new_task(isolated_env, lines)
|
||
bp = BatchProcessor(task)
|
||
bp.process_batch()
|
||
bp.merge_all(dry_run=False)
|
||
|
||
# 再处理同样的数据 + 合并，主库不应增长
|
||
bp.process_batch()
|
||
r2 = bp.merge_all(dry_run=False)
|
||
assert r2["merged_auto"] == 0
|
||
assert r2["duplicated_auto"] == 1
|
||
assert r2["main_total"] == 1
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# review 重跑（改代码后重新分流）
|
||
# --------------------------------------------------------------------------
|
||
def test_reprocess_review_reclassifies(isolated_env):
|
||
"""重跑 review 桶：条目重新分类，原桶清零"""
|
||
tm, task = _new_task(isolated_env, ["日本人:にほんじん:"])
|
||
bp = BatchProcessor(task)
|
||
bp.process_batch()
|
||
|
||
# 手动放一条可自动处理的词进 pinyin 桶，模拟"修正后应归入 auto"
|
||
bp.review_files["pinyin"].write_text("中国人:ちゅうごくじん:\n",
|
||
encoding="utf-8", newline="\n")
|
||
result = bp.reprocess_review("pinyin")
|
||
|
||
assert result["reprocessed"] == 1
|
||
# 原 pinyin 桶已清零
|
||
assert len(bp._read_clean_lines(bp.review_files["pinyin"])) == 0
|
||
|
||
|
||
def test_reprocess_unknown_bucket_raises(isolated_env):
|
||
"""重跑未知桶名报错"""
|
||
tm, task = _new_task(isolated_env, ["日本:にほん:"])
|
||
bp = BatchProcessor(task)
|
||
with pytest.raises(ValueError):
|
||
bp.reprocess_review("nonexistent")
|