ef3df73166
清洗管线重构为严格三层架构：
- tango_analyser.py（底层：单词分析）
- task_processor.py（中层：文件 I/O、桶管理）
- workflow.py（顶层：状态机、任务推进）
- 移除旧的 batch_processor.py
新增配置文件系统：
- config.py：TOML 配置，相对路径相对配置文件目录解析
- 查找优先级 --config > cwd > 项目根 > ~ > 默认值
- count=None 语义为处理到文件末尾
项目目录整理：
- 根脚本归档到 scripts/analysis 与 scripts/legacy
- 文档归档到 docs/{design,history,analysis}
- 临时报告移到 reports/（已 gitignore）
文档质量：
- 新增 .markdownlint.json 与 scripts/mdlint.cmd
- 修复全部 14 个 md 文件的 markdownlint 警告
测试：74 passed（8 cleaner + 31 workflow + 6 tango + 15 validator + 14 config）
164 lines
5.1 KiB
Python
164 lines
5.1 KiB
Python
"""
|
||
配置文件管理测试
|
||
"""
|
||
from pathlib import Path
|
||
import pytest
|
||
|
||
from pl_japanese.cleaner.config import (
|
||
JCleanConfig, load_config, find_config_file, generate_sample_config
|
||
)
|
||
|
||
|
||
def test_default_config():
|
||
"""没有配置文件时使用默认值"""
|
||
config = JCleanConfig()
|
||
assert config.tasks_root == 'tasks'
|
||
assert config.vocabulary == 'data/db/vocabulary.txt'
|
||
assert config.skipped == 'data/db/skipped.txt'
|
||
|
||
|
||
def test_resolve_path_absolute(tmp_path):
|
||
"""绝对路径直接返回"""
|
||
config = JCleanConfig(_config_file=tmp_path / 'jclean.toml')
|
||
# 使用 Windows 兼容的绝对路径
|
||
abs_path = str((tmp_path / 'absolute_file.txt').resolve())
|
||
assert config.resolve_path(abs_path) == abs_path
|
||
|
||
|
||
def test_resolve_path_relative_with_config(tmp_path):
|
||
"""相对路径相对配置文件所在目录"""
|
||
config_file = tmp_path / 'jclean.toml'
|
||
config_file.write_text('[paths]', encoding='utf-8')
|
||
|
||
config = JCleanConfig(_config_file=config_file)
|
||
resolved = config.resolve_path('tasks')
|
||
expected = str((tmp_path / 'tasks').resolve())
|
||
assert resolved == expected
|
||
|
||
|
||
def test_resolve_path_relative_no_config(tmp_path):
|
||
"""没有配置文件时，相对当前工作目录"""
|
||
config = JCleanConfig() # 没有 _config_file
|
||
resolved = config.resolve_path('tasks')
|
||
expected = str((Path.cwd() / 'tasks').resolve())
|
||
assert resolved == expected
|
||
|
||
|
||
def test_find_config_file_current_dir(tmp_path):
|
||
"""在当前目录查找配置文件"""
|
||
config_file = tmp_path / 'jclean.toml'
|
||
config_file.write_text('[paths]', encoding='utf-8')
|
||
|
||
found = find_config_file(start_dir=tmp_path)
|
||
assert found == config_file.resolve()
|
||
|
||
|
||
def test_find_config_file_dotfile(tmp_path):
|
||
"""查找 .jclean.toml"""
|
||
config_file = tmp_path / '.jclean.toml'
|
||
config_file.write_text('[paths]', encoding='utf-8')
|
||
|
||
found = find_config_file(start_dir=tmp_path)
|
||
assert found == config_file.resolve()
|
||
|
||
|
||
def test_find_config_file_custom_path(tmp_path):
|
||
"""命令行指定配置文件路径"""
|
||
config_file = tmp_path / 'custom.toml'
|
||
config_file.write_text('[paths]', encoding='utf-8')
|
||
|
||
found = find_config_file(custom_path=str(config_file))
|
||
assert found == config_file.resolve()
|
||
|
||
|
||
def test_find_config_file_not_found(tmp_path):
|
||
"""找不到配置文件返回 None"""
|
||
found = find_config_file(start_dir=tmp_path)
|
||
assert found is None
|
||
|
||
|
||
def test_load_config_no_file(tmp_path):
|
||
"""没有配置文件时返回默认配置"""
|
||
config = load_config()
|
||
assert isinstance(config, JCleanConfig)
|
||
assert config.tasks_root == 'tasks'
|
||
|
||
|
||
def test_load_config_from_file(tmp_path):
|
||
"""从配置文件加载"""
|
||
config_file = tmp_path / 'jclean.toml'
|
||
config_file.write_text('''
|
||
[paths]
|
||
tasks_root = "my_tasks"
|
||
vocabulary = "my_vocab.txt"
|
||
|
||
[defaults]
|
||
backup_before_merge = false
|
||
''', encoding='utf-8')
|
||
|
||
config = load_config(custom_path=str(config_file))
|
||
assert config.tasks_root == 'my_tasks'
|
||
assert config.vocabulary == 'my_vocab.txt'
|
||
assert config.backup_before_merge == False
|
||
assert config._config_file == config_file.resolve()
|
||
|
||
|
||
def test_generate_sample_config(tmp_path):
|
||
"""生成示例配置文件"""
|
||
output = tmp_path / 'test.toml'
|
||
generate_sample_config(str(output))
|
||
|
||
assert output.exists()
|
||
content = output.read_text(encoding='utf-8')
|
||
assert '[paths]' in content
|
||
assert 'tasks_root' in content
|
||
assert '[defaults]' in content
|
||
assert '[logging]' in content
|
||
|
||
|
||
def test_config_priority_command_line_over_file(tmp_path):
|
||
"""命令行参数优先级高于配置文件"""
|
||
# 这个测试在 CLI 层面验证，这里只测试配置加载
|
||
config_file = tmp_path / 'jclean.toml'
|
||
config_file.write_text('''
|
||
[paths]
|
||
tasks_root = "config_tasks"
|
||
vocabulary = "custom_vocab.txt"
|
||
''', encoding='utf-8')
|
||
|
||
config = load_config(custom_path=str(config_file))
|
||
|
||
# CLI 层面会覆盖这些值
|
||
assert config.tasks_root == 'config_tasks' # 从配置文件读取
|
||
assert config.vocabulary == 'custom_vocab.txt'
|
||
|
||
|
||
def test_config_resolve_path_with_parent_dir(tmp_path):
|
||
"""解析包含 .. 的相对路径"""
|
||
config_dir = tmp_path / 'subdir'
|
||
config_dir.mkdir()
|
||
config_file = config_dir / 'jclean.toml'
|
||
config_file.write_text('[paths]', encoding='utf-8')
|
||
|
||
config = JCleanConfig(_config_file=config_file)
|
||
resolved = config.resolve_path('../data')
|
||
expected = str((tmp_path / 'data').resolve())
|
||
assert resolved == expected
|
||
|
||
|
||
def test_load_config_tolerates_bom(tmp_path):
|
||
"""带 UTF-8 BOM 的配置文件应能正常解析（记事本等编辑器会写入 BOM）"""
|
||
config_file = tmp_path / 'jclean.toml'
|
||
# utf-8-sig 会在文件头写入 BOM (EF BB BF)
|
||
config_file.write_text('''[paths]
|
||
tasks_root = "bom_tasks"
|
||
vocabulary = "bom_vocab.txt"
|
||
''', encoding='utf-8-sig')
|
||
|
||
# 确认文件确实带 BOM
|
||
assert config_file.read_bytes()[:3] == b'\xef\xbb\xbf'
|
||
|
||
config = load_config(custom_path=str(config_file))
|
||
assert config.tasks_root == 'bom_tasks'
|
||
assert config.vocabulary == 'bom_vocab.txt'
|