Python的并行--threading和mutiprocess

一、多线程计算试验

    # 计算1!+2!+3!.....

import threading
import time
import sys
import queue
rev = queue.Queue()

#计算num的阶乘，并放入队列
def cal(num):
    time.sleep(2) # 暂停两秒，模拟IO密集任务
    sum = 1
    for i in range(1,num+2):
        sum = sum * i
    sys.stdout.write("thread %s cal %d! = %d\n" % (threading.current_thread().ident, num + 1,sum))
    return rev.put(sum)

def run(thread_num):
    sum = 0
    fs = []
    #阶乘和有几个就开几个线程
    for i in range(thread_num):
        th = threading.Thread(target=cal,args=(i,))
        fs.append(th)
        th.start() 

    for th in fs:
        th.join() #主线程等待各个线程结束，才会继续执行

    rev.put(None)
    while True:
        num = rev.get()
        if num==None: break
        sum +=num
    return sum

if __name__ == "__main__":
      st = time.time()
      print(run(20))
      et = time.time()
      print('total time: %f seconds'% (et-st))

分别计算5，10，20的阶乘和，运行结果都在2s左右，速度比不用多线程快很多，甚至可以开到100个线程。结果如下：

2561327494111820313
sigma(20!) total time: 2.006062 seconds

二、多进程计算试验

# 计算1!+2!+3!.....

from multiprocessing import Process,Queue,Pool
import time
import sys,os
# 进程间的通信要用multiprocessing.Queue()
rev = Queue()

def cal(num):
    time.sleep(2) # 暂停两秒，模拟IO密集任务
    sum = 1
    for i in range(1,num+2):
        sum = sum * i
    sys.stdout.write("process %s cal %d! = %d\n" % (os.getpid(), num + 1,sum))
    return rev.put(sum)

def run(thread_num):
    sum = 0
    fs = []
    # 普通进程方式
    # for i in range(thread_num):
    #     th = Process(target=cal,args=(i,))
    #     fs.append(th)
    #     th.start()
    #
    # for th in fs:
    #     th.join()

    # 进程池方式
    pool = Pool(10)
    for i in range(thread_num):
        pool.apply_async(cal, (i,))
    pool.close()
    pool.join()

    rev.put(None)
    while True:
        num = rev.get()
        #print(num)
        if num == None: break
        sum +=num
    return sum

if __name__ == "__main__" :
      st = time.time()
      print(run(10))
      et = time.time()
      print('sigma(%d!) total time: %f seconds'% (10,et-st))

多进程间的通信必须要用mutiproessing.Queue.

参考

[1] https://www.cnblogs.com/linhaifeng/articles/7428874.html#_label6