about those python async concurrent frameworks - fantix @ ostc 2014

Python 开源异步并发框架的未来About Those

Python Asynchronous Concurrency Frameworks

def about(self):

http://about.me/fantix

Python 3.4胶水语言，内含电池

并行 vs. 并发

__name__ == ‘__main__’

多个窗口并行处理

照相 1

照相 2

照相 3

取号 2

取号 1

办理 1 办理 2 办理 3 办理 4

12

3

照相 1

照相 2

照相 3

取号 2

取号 1

办理 1 办理 2 办理 3 办理 4

17

23

16 5 7 3 6

12

15

最多同时容纳

800人！

并行

并发服务器

?

多进程／多线程• “ 多进程” not in self.topic

• Python GIL

• PyPy STM 还有谁？

{ select(2), epoll(7), kqueue(2),}

事件驱动while True:

张三 = select(“10 万客户端” )

try:

data = 张三 .recv()

张三 .send(data)

except EAGAIN:

pass

sock = socket.socket()sock.setblocking(0)sock.bind((“”, 80))sock.listen(128)

def on_conn(fd, events): conn, address = sock.accept() conn.send(b’Hello’)

io_loop = ioloop.IOLoop.instance()io_loop.add_handler(sock.fileno(), on_conn, io_loop.READ)io_loop.start()

回调函数

from twisted.internet import protocol, reactor

class Echo(protocol.Protocol): def dataReceived(self, data): self.transport.write(data)

class EchoFactory(protocol.Factory): def buildProtocol(self, addr): return Echo()

reactor.listenTCP(1234, EchoFactory())reactor.run() 另外 ...

lol, Deferred

generator@defer.inlineCallbacksdef main(endpoint, username="alice", password="secret"): endpoint = endpoints.clientFromString(reactor, strport) factory = protocol.Factory() factory.protocol = imap4.IMAP4Client try: endpoint.connect(factory) client = yield yield client.login(username, password) yield client.select('INBOX') info = yield client.fetchEnvelope(imap4.MessageSet(1)) print 'First message subject:', info[1]['ENVELOPE'][1] except: print "IMAP4 client interaction failed" failure.Failure().printTraceback()task.react(main, sys.argv[1:])

greenlet@defer.inlineCallbacksdef main(endpoint, username="alice", password=“secret”): endpoint = endpoints.clientFromString(reactor, strport) factory = protocol.Factory() factory.protocol = imap4.IMAP4Client try: client = yield endpoint.connect(factory) yield client.login(username, password) yield client.select('INBOX') info = yield client.fetchEnvelope(imap4.MessageSet(1)) print 'First message subject:', info[1]['ENVELOPE'][1] except: print "IMAP4 client interaction failed" failure.Failure().printTraceback()task.react(main, sys.argv[1:])

def main(endpoint, username="alice", password=“secret”): endpoint = endpoints.clientFromString(reactor, strport) factory = protocol.Factory() factory.protocol = imap4.IMAP4Client try: client = endpoint.connect(factory) client.login(username, password) client.select('INBOX') info = client.fetchEnvelope(imap4.MessageSet(1)) print 'First message subject:', info[1]['ENVELOPE'][1] except: print "IMAP4 client interaction failed" failure.Failure().printTraceback()task.react(main, sys.argv[1:])

Eventlet / Geventfrom gevent import monkey

monkey.patch_all()

def handle(socket, address):

url = socket.recv()

data = urllib.urlopen(url).read()

socket.send(data)

server = StreamServer(('127.0.0.1', 1234), handle)

server.serve_forever()

https://glyph.twistedmatrix.com/2014/02/unyielding.html

__feature__callbac

kgenerator

greenlet

Python 3

Twisted ✅ ✅ 57%

Tornado ✅ ✅ ✅

Gevent ✅ ✅ 80%

activity in day * 30issues mails commits

Twisted 122 40 19 (258)

Tornado 31 36 17

Gevent 12 5 26

2014.03.18

互操作性－今天Twisted

Tornado Eventlet/

Gevent

Torna

doRe

actor

Twist

edIOLoo

p

geventreact

or

Stackless

互操作性－今天redis postgresq

l mysql

Twisted txredisapi txpostgres txmysql

Tornado tornado-redis momoko amysql

Gevent <monkey> psycogreen greenify

from __future__

tulip / asyncioby Guido van Rossum

Python 3.4胶水语言，内含电池

像 Twisted

callbackimport asyncio

def print_and_repeat(loop): print('Hello World') loop.call_later(2, print_and_repeat, loop)

if __name__ == '__main__': loop = asyncio.get_event_loop() print_and_repeat(loop) loop.run_forever()

protocol && transportclass MyServerUdpEchoProtocol:

def connection_made(self, transport): print('start', transport) self.transport = transport

def datagram_received(self, data, addr): print('Data received:', data, addr) self.transport.sendto(data, addr)

def error_received(self, exc): print('Error received:', exc)

def connection_lost(self, exc): print('stop', exc)

FutureIn [1]: from asyncio.futures import Future

In [2]: import asyncio

In [3]: loop = asyncio.get_event_loop()

In [4]: f = Future()

In [5]: loop.call_later(2, f.set_result, 'OK')Out[5]: TimerHandle(74709.18277206, <bound method Future.set_result of Future<PENDING>>, ('OK',))

In [6]: loop.run_until_complete(f)Out[6]: 'OK'

Task && Coroutineimport asynciofrom asyncio.tasks import Task

@asyncio.coroutinedef greet_every_two_seconds(): while True: print('Hello World') yield from asyncio.sleep(2)

if __name__ == '__main__': loop = asyncio.get_event_loop() t = Task(greet_every_two_seconds()) loop.run_until_complete(t)

PEP 3156

asyncio

Event LoopRI

apps libs

Third-partyEvent Loop

adapter

原厂发动机

asyncio

Event LoopRI

Twisted gevent

adapter adapter

“ 组装”发动机

asyncio

TwistedEvent Loop

Twisted

gevent

adapter

互操作性－明天Twisted

Tornado Eventlet/

Gevent

Stackless

Torna

doRe

actor

Twist

edIOLoo

p

geventreact

orasyncio

txredisapi

txmysql

momokoamysql psycogreen

互操作性－明天框架 适配 asyncio

Twisted 讨论中

Tornado 实验中

Gevent https://github.com/decentfox/gevent3

谢谢！http://about.me/fantix

http://github.com/fantix

http://weibo.com/fantix

http://twitter.com/fantix

http://www.linkedin.com/in/fantix