Python 使用 multiprocessing 和 find 命令统计大文件行数
0x00 脚本功能
使用多线程multiprocessing,和cmd命令type 文件名称 | find /v /c ""统计文件行数,最终以dict输出到文件。
0x01 完整代码
import json
import os
from multiprocessing.pool import ThreadPool
all = {}
cur = 0
all_count = 0
dir = r"V:\Temp"
def count(path):
global cur, all_count, all
site = path.replace(".csv", "")
result = os.popen(f'type "{os.path.join(dir, path)}" | find /v /c ""').read()
all.update({site: int(result)})
cur += 1
print(f"{cur}/{all_count}")
if __name__ == '__main__':
paths = [path for path in os.listdir(dir)]
all_count = len(paths)
pool = ThreadPool(processes=10)
pool.map(count, paths)
pool.close()
pool.join()
with open("result.json", "w") as f:
f.write(json.dumps(all))
0x02 参考文章
https://blog.csdn.net/qq_40579464/article/details/105266838