auto_charset_detect.py 独立命令行脚本

功能：

1. 支持传入文件/文件夹路径
2. 自动检测编码、输出结构化信息
3. 可选：自动转码保存为 UTF-8
4. 支持输出JSON方便程序调用
5. 基于 charset-normalizer，自带容错降级

#!/usr/bin/env python3
# auto_charset_detect.py
import argparse
import json
import os
from pathlib import Path
from typing import Dict, Any, Union
from charset_normalizer import from_bytes, from_path
from charset_normalizer import CharsetMatch


def auto_decode(
    data: Union[bytes, str],
    fallback_encoding: str = "utf-8",
    fallback_errors: str = "replace",
    threshold: float = 0.2
) -> Dict[str, Any]:
    raw_data: bytes
    match: CharsetMatch | None = None

    if isinstance(data, str):
        path = Path(data)
        if not path.exists():
            return {
                "success": False,
                "encoding": None,
                "confidence": None,
                "language": None,
                "text": "",
                "raw_data": b"",
                "error": f"文件不存在: {data}"
            }
        try:
            match = from_path(path).best()
            with open(path, "rb") as f:
                raw_data = f.read()
        except Exception as e:
            return {
                "success": False,
                "encoding": None,
                "confidence": None,
                "language": None,
                "text": "",
                "raw_data": b"",
                "error": f"读取失败: {str(e)}"
            }
    elif isinstance(data, bytes):
        raw_data = data
        match = from_bytes(raw_data).best()
    else:
        raise TypeError("data 仅支持 bytes 二进制 或 文件路径字符串")

    if match and match.chaos <= threshold:
        try:
            text = str(match)
            return {
                "success": True,
                "encoding": match.encoding,
                "confidence": round(match.coherence, 2),
                "language": match.language,
                "text": text,
                "raw_data": raw_data,
                "error": None
            }
        except Exception:
            pass

    # 降级解码
    text = raw_data.decode(fallback_encoding, errors=fallback_errors)
    return {
        "success": False,
        "encoding": match.encoding if match else None,
        "confidence": round(match.coherence, 2) if match else None,
        "language": match.language if match else None,
        "text": text,
        "raw_data": raw_data,
        "error": "置信度不足，使用备用编码解码"
    }


def scan_file(filepath: Path, args) -> Dict[str, Any]:
    res = auto_decode(str(filepath), threshold=args.threshold)
    output = {
        "file": str(filepath),
        "encoding": res["encoding"],
        "confidence": res["confidence"],
        "language": res["language"],
        "success": res["success"],
        "error": res.get("error")
    }

    # 自动转码另存为 utf-8
    if args.convert and res["text"]:
        new_path = filepath.with_suffix(".utf8" + filepath.suffix)
        with open(new_path, "w", encoding="utf-8") as fw:
            fw.write(res["text"])
        output["convert_save_to"] = str(new_path)
    return output


def main():
    parser = argparse.ArgumentParser(description="自动文本编码检测工具 (charset-normalizer)")
    parser.add_argument("target", help="目标文件 / 文件夹路径")
    parser.add_argument("--ext", default=".txt,.srt,.csv,.md,.java,.py,.sql",
                        help="仅扫描这些后缀文件（逗号分隔），仅文件夹模式生效")
    parser.add_argument("--threshold", type=float, default=0.2, help="chaos 置信阈值，默认0.2")
    parser.add_argument("--json", action="store_true", help="输出纯JSON格式，便于脚本调用")
    parser.add_argument("--convert", action="store_true", help="检测后另存为 UTF-8 文件")
    args = parser.parse_args()

    target = Path(args.target)
    results = []

    if target.is_file():
        results.append(scan_file(target, args))
    elif target.is_dir():
        exts = set(args.ext.split(","))
        for p in target.rglob("*"):
            if p.suffix.lower() in exts and p.is_file():
                results.append(scan_file(p, args))
    else:
        print(f"错误：路径不存在 {target}")
        return

    if args.json:
        print(json.dumps(results, ensure_ascii=False, indent=2))
    else:
        for item in results:
            print("-" * 70)
            print(f"文件: {item['file']}")
            print(f"识别编码: {item['encoding']} | 置信度: {item['confidence']} | 语言: {item['language']}")
            print(f"状态: {'✅ 可信识别' if item['success'] else '⚠️ 低置信度'}")
            if item.get("error"):
                print(f"备注: {item['error']}")
            if item.get("convert_save_to"):
                print(f"已转码保存至: {item['convert_save_to']}")


if __name__ == "__main__":
    main()

使用步骤

1. 安装依赖

pip install charset-normalizer

2. 基础用法

检测单个文件

python auto_charset_detect.py test.txt

输出 JSON（适合被其他程序调用、流水线使用）

python auto_charset_detect.py test.txt --json

扫描整个目录（只扫描 txt/srt/csv/md/java/py/sql）

python auto_charset_detect.py ./docs

扫描目录 + 自动转码另存为UTF-8

python auto_charset_detect.py ./docs --convert

指定自定义后缀列表

python auto_charset_detect.py ./data --ext ".txt,.log,.ini"

输出示例（普通文本模式）

----------------------------------------------------------------------
文件: ./test_gbk.txt
识别编码: gbk | 置信度: 96.32 | 语言: Chinese
状态: ✅ 可信识别

输出示例（JSON模式）

[
  {
    "file": "./test_gbk.txt",
    "encoding": "gbk",
    "confidence": 96.32,
    "language": "Chinese",
    "success": true,
    "error": null
  }
]

可直接集成场景

1. CI流水线批量检测源码文件编码
2. 日志预处理、文本文件批量转码
3. 后端服务调用（可剥离 argparse 单独提取 auto_decode 函数）
4. Windows 历史GBK文件批量迁移 UTF-8