Python 使用 multiprocessing 和 find 命令统计大文件行数


0x00 脚本功能

使用多线程multiprocessing,和cmd命令type 文件名称 | find /v /c ""统计文件行数,最终以dict输出到文件。

0x01 完整代码

import json
import os
from multiprocessing.pool import ThreadPool

all = {}
cur = 0
all_count = 0
dir = r"V:\Temp"


def count(path):
    global cur, all_count, all
    site = path.replace(".csv", "")
    result = os.popen(f'type "{os.path.join(dir, path)}" | find /v /c ""').read()
    all.update({site: int(result)})
    cur += 1
    print(f"{cur}/{all_count}")


if __name__ == '__main__':
    paths = [path for path in os.listdir(dir)]
    all_count = len(paths)
    pool = ThreadPool(processes=10)
    pool.map(count, paths)
    pool.close()
    pool.join()

    with open("result.json", "w") as f:
        f.write(json.dumps(all))

0x02 参考文章

https://blog.csdn.net/qq_40579464/article/details/105266838