多进程写文件配合tqdm实现进度条

2023-01-30  本文已影响0人  Donald_32e5

先测试一下变种tqdm

import time
from tqdm import tqdm

bar = tqdm(total=100)
last_num = 0
# 总个数就是100个,下列list中模拟的就是每次读目录的文件个数
for i in [10, 10, 30, 40, 80, 88, 100]:
    flush_num = i-last_num
    # update需要传入此次新完成的个数
    bar.update(flush_num)
    time.sleep(1)
    last_num = i

在多进程中读取文件的个数

def writer_csv(**args):
    """写csv的方法
    多进程写入同一个目录
    """
 
def get_files(total_num):
    this_num = sum(os.listdir(path))
    last_num = 0
    bar = tqdm(total=total_num)
    while this_num <= total_num:
        this_num = sum(os.listdir(path))
        flush_num = this_num - last_num
        bar.update(flush_num)

def start():
    # 提前计算此次需要生成的文件个数
    bar_thrad = Thread(target=get_files, args=(total_num, ))
    bar_thrad.start()

    task = []
    for i in []:
        task.append((i, ))
    with Pool(num) as pool:
        pool.startmap(writer_csv, task)
    
上一篇 下一篇

猜你喜欢

热点阅读