mysql user conference 2009: python and mysql
DESCRIPTION
TRANSCRIPT
Python and MySQLTed LeungSun Microsystems
Python-DB In the beginning there was the DB-API PEP-249
MySQLdb MySQLdb written in Python _mysql written in C and is MySQL specific
TextTextText
import MySQLdb
conn = MySQLdb.connect( host = 'localhost', user = 'twl', passwd='pass', db='flickrmgr')
Text
c = conn.cursor()c.execute("""SELECT * from curator_photo""")
# read one rowpprint(c.fetchone())
# read all the rowspprint(c.fetchall())
((3L, 3422686825L, "OPG Toymaker's Doll 2009", 2L, 'http://farm4.static.flickr.com/3377/3422686825_fd57ea30e7.jpg', datetime.datetime(2009, 4, 8, 1, 21, 44)), (4L, 3422685683L, "OPG Toymaker's Doll 2009", 2L, 'http://farm4.static.flickr.com/3608/3422685683_4d16829c19.jpg', datetime.datetime(2009, 4, 8, 1, 20, 54)),
dc = conn.cursor(MySQLdb.cursors.DictCursor)dc.execute("""SELECT * from curator_photo""")
pprint(dc.fetchmany(5))
({'flickr_id': 2147483647L, 'id': 2L, 'pub_date': datetime.datetime(2009, 4, 8, 1, 21, 44), 'title': "OPG Toymaker's Doll 2009", 'url': 'http://farm4.static.flickr.com/3377/3422686825_fd57ea30e7.jpg', 'user_id': 2L}, {'flickr_id': 3422686825L, 'id': 3L, 'pub_date': datetime.datetime(2009, 4, 8, 1, 21, 44), 'title': "OPG Toymaker's Doll 2009", 'url': 'http://farm4.static.flickr.com/3377/3422686825_fd57ea30e7.jpg', 'user_id': 2L})
c.execute( """SELECT * from curator_photo WHERE id < %s""", (50,))
pprint(c.fetchall())
c.execute( """INSERT INTO curator_user (flickr_id, name) VALUES (%s, %s)""", ("000", "No User"))
conn.commit()
Django Leading web application framework Lots of people coming to Python via Django today Model-View-Controller Command line management scripts It’s own Rails-like ORM Based on the Active Record pattern
Active Record Pattern Database table is wrapped in a class Each class instance is a row in the table Relationships expressd as foreign key constraints
DATABASE_ENGINE = 'mysql' DATABASE_NAME = 'flickrmgr'DATABASE_USER = 'twl' DATABASE_PASSWORD = 'pass'
from django.db import models
class User(models.Model): flickr_id = models.TextField() name = models.TextField()
def __unicode__(self): return self.name
class Group(models.Model): flickr_id = models.TextField() name = models.TextField() throttle_count = models.IntegerField() throttle_mode = models.TextField() throttle_remaining = models.IntegerField()
def __unicode__(self): return self.name
class Photo(models.Model): flickr_id = models.IntegerField() title = models.TextField() user = models.ForeignKey(User) groups = models.ManyToManyField(Group) url = models.URLField() pub_date = models.DateTimeField()
def __unicode__(self): return self.title
from curator.models import Photofrom datetime import datetime
Photo.objects.all()Photo.objects.filter(title__contains='PyCon')
Photo.objects.filter(title__contains='PyCon').exclude( pub_date__lte=datetime(2009,4,1))
Photo.objects.filter(title__contains='PyCon').exclude( pub_date__lte=datetime(2009,4,1))[2:4]
from django.contrib import admin
class PhotoAdmin(admin.ModelAdmin): pass
class GroupAdmin(admin.ModelAdmin): pass
class UserAdmin(admin.ModelAdmin): pass
admin.site.register(Group, GroupAdmin)admin.site.register(Photo, PhotoAdmin)admin.site.register(User, UserAdmin)
def load_from_flickr(request): api = API()
# 51035696189@N01 twl, created = User.objects.get_or_create( flickr_id = '51035696189@N01', name = 'Ted Leung' ) if created: twl.save()
photos = api.list_user_info( 'http://www.flickr.com/photos/twleung')
for photo in photos: flickr_id, title, pub_date, url, pools = photo new_photo = Photo() new_photo.flickr_id = flickr_id new_photo.title = title new_photo.user = twl new_photo.pub_date = datetime.fromtimestamp(int(pub_date)) new_photo.url = url new_photo.save()
# do pools for pool in pools: new_pool, created=Group.objects.get_or_create( flickr_id = pool[0], name = pool[1], throttle_count = pool[2], throttle_mode = pool[3], throttle_remaining = pool[4] ) new_photo.groups.add(new_pool) if created: new_pool.save()
output = '''<html> <head> <title>Bulk loading from Flickr</title> </head> <body> </body></html>''' return HttpResponse(output)
from django.conf.urls.defaults import *
from curator.models import User, Group, Photofrom curator.views import load_from_flickr
photo_info_dict = { 'queryset': Photo.objects.all(), 'date_field': 'pub_date',}
urlpatterns = patterns( '', (r'^$', 'django.views.generic.date_based.archive_index', photo_info_dict), (r'^load/$', load_from_flickr),)
Transactions Commit on save or delete Commit at HTTP request/response boundaries commit_manually decorator
Connection Pooling Important as systems scale up No framework wide solution yet
Migration South Integrates with manage.py Can automatically migrate models
../bin/django startmigration curator --initial
Creating migrations directory at '/Users/twl/work/mysql-2009/django/flickrmgr/curator/migrations'...
Creating __init__.py in '/Users/twl/work/mysql-2009/django/flickrmgr/curator/migrations'... + Added model 'curator.Photo' + Added model 'curator.Group' + Added model 'curator.User' + Added field 'curator.Photo.groups'Created 0001_initial.py.
class Photo(models.Model): flickr_id = models.IntegerField() title = models.TextField() user = models.ForeignKey(User) groups = models.ManyToManyField(Group) url = models.URLField() pub_date = models.DateTimeField() visible = models.BooleanField()
../bin/django startmigration curator add_visible --auto + Added field 'curator.photo.visible'Created 0002_add_visible.py.
from south.db import dbfrom django.db import modelsfrom curator.models import *
class Migration: def forwards(self, orm): # Adding field 'Photo.visible' db.add_column('curator_photo', 'visible', models.BooleanField()) def backwards(self, orm): # Deleting field 'Photo.visible' db.delete_column('curator_photo', 'visible')
SQLObject ORM using Active Record pattern Tight coupling to Python classes Used in TurboGears 1
connection_string='mysql://twl:pass@localhost/sqlobject'
connection=connectionForURI(connection_string)
sqlhub.processConnection = connection
try: User.createTable(ifNotExists=True) Photo.createTable(ifNotExists=True) FlickrGroup.createTable(ifNotExists=True)except dberrors.OperationalError, oe: print oe
class User(SQLObject): flickr_id = StringCol() name = StringCol()
class Photo(SQLObject): flickr_id = StringCol() title = StringCol() user = ForeignKey('User') groups = RelatedJoin('FlickrGroup') url = StringCol() pub_date = DateTimeCol()
class FlickrGroup(SQLObject): flickr_id = StringCol() name = StringCol() throttle_count = IntCol() throttle_mode = StringCol() throttle_remaining = IntCol() photos = RelatedJoin('Photo')
pycon_photos = list(Photo.select(Photo.q.title.contains('PyCon')))
pprint(pycon_photos)
pycon_apr_photos = list(Photo.select(AND(Photo.q.title.contains('PyCon'), Photo.q.pub_date > datetime(2009,4,1) )))
pprint(pycon_apr_photos)
api = API()
twl = User( flickr_id = '51035696189@N01', name = 'Ted Leung')
photos = api.list_user_info( 'http://www.flickr.com/photos/twleung')
for photo in photos: print photo flickr_id, title, pub_date, url, pools = photo new_photo = Photo( flickr_id = flickr_id, title = title, user = twl, pub_date = datetime.fromtimestamp(int(pub_date)), url = url )
# do pools for pool in pools: new_pool = list(FlickrGroup.select( FlickrGroup.q.flickr_id == pool[0])) if len(new_pool) > 0: new_pool = new_pool[0] else: new_pool = None if not new_pool: new_pool = FlickrGroup( flickr_id = pool[0], name = pool[1], throttle_count = pool[2], throttle_mode = pool[3], throttle_remaining = pool[4] )
new_photo.addFlickrGroup(new_pool)
Transactionstxn = connection.transaction()
p = Photo.get(1, txn)p.title = ‘updated photo’
txn.commit()
Connection Pooling Connection pooling is built in and on by default
sqlmetaclass Photo(SQLObject): class sqlmeta: lazyUpdate = True cacheValues = False
flickr_id = StringCol() title = StringCol() user = ForeignKey('User') groups = RelatedJoin('FlickrGroup') url = StringCol() pub_date = DateTimeCol()
Eventsfrom sqlobject.events import listenfrom sqlobject.events RowUpdateSignal, RowCreatedSignal
def update_listener(instance, kwargs): kwargs['pub_date'] = datetime.datetime.now()
def created_listener(kwargs, post_funcs): print “created photo %s” % (kwargs[‘title’])
listen(update_listener, Photo, RowUpdateSignal)listen(created_listener, Photo, RowCreatedSignal)
SQLAlchemy Python’s answer to Hibernate Low level SQL manipulations High Level ORM Used in TurboGears 2
Low Level SQL
from sqlalchemy import create_enginefrom sqlalchemy import Table, Column, Integer, String, Text, DateTime, MetaData, ForeignKeyfrom sqlalchemy import select
engine = create_engine( 'mysql://twl:pass@localhost/sqlobject',echo=True) metadata = MetaData()
users_table = Table('user', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('name', Text) )
groups_table = Table('group', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('name', Text), Column('throttle_count', Integer), Column('throttle_mode', Text), Column('throttle_remaining', Integer) ) photos_table = Table('photo', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('title', Text), Column('user_id', Integer, ForeignKey(‘user.id’)), Column('url', Text), Column('pub_date', DateTime) )
metadata.create_all(engine)
conn = engine.connect()
s = select([users_table])result = conn.execute(s)
for row in result: print row
s = select([photos_table], photos_table.c.title.contains('PyCon'))
for row in conn.execute(s): print row
ins = users.insert().values(name='Thomas Hawk', flickr_id='000')
result = conn.execute(ins)
update = users.update().where( users.c.name=='Thomas Hawk' ).values( flickr_id='51035555243@N01'))
result = conn.execute(update)
Transactions Regular manual control Autocommit - per session 2PC
Connection Pooling Built-in framework Done at engine configuration time
Other Low Level SQL features Unions and other Set operations Scalar Selects Correlated Subqueries Ordering, Grouping, Offsetting Correlated Updates
ORM Mapping Declarative
class User(object): def __repr__(self): return "<User('%s','%s','%s')>" % ( self.id, self.flickr_id, self.name)
class Group(object): def __repr__(self): return "<Group('%s','%s','%s','%s')>" % ( self.id, self.flickr_id, self.name, self.throttle_remaining )
class Photo(object): def __repr__(self): return "<Photo('%s','%s','%s','%s')>" % ( self.id, self.flickr_id, self.title, self.url )
metadata = MetaData()
flickr_group_photo = Table( 'flickr_group_photo', metadata, Column('flickr_group_id', Integer, ForeignKey('flickr_group.id')), Column('photo_id', Integer, ForeignKey('photo.id')))
from sqlalchemy.orm import mapper, sessionmaker
engine = create_engine( 'mysql://twl:pass@localhost/sqlobject')
Session = sessionmaker(bind = engine)session = Session()
mapper(User, users_table, properties = { 'photos' : relation(Photo, backref='user') })
mapper(Group, groups_table, properties = { 'photos' : relation(Photo, secondary=flickr_group_photo, backref='groups') })
mapper(Photo, photos_table)
for u in session.query(User): print u
for g in session.query(Group): print g
for p in session.query(Photo): print p
Declarative ORM
from sqlalchemy.ext.declarative import declarative_basefrom sqlalchemy.orm import sessionmakerfrom sqlalchemy import Column, ForeignKeyfrom sqlalchemy import Integer, String, Text, DateTime
Base = declarative_base()
class User(Base): __tablename__ = 'user'
id = Column(Integer, primary_key=True) flickr_id = Column(Text) name = Column(Text)
def __repr__(self): return "<User('%s','%s','%s')>" % ( self.id, self.flickr_id, self.name)
class Group(Base): __tablename__ = 'flickr_group' id = Column(Integer, primary_key=True) flickr_id = Column(Text) name = Column(Text) throttle_count = Column(Integer) throttle_mode = Column(Text) throttle_remaining = Column(Integer)
def __repr__(self): return "<Group('%s','%s','%s','%s')>" % ( self.id, self.flickr_id, self.name, self.throttle_remaining )
class Photo(Base): __tablename__ = 'photo' id = Column(Integer, primary_key=True) flickr_id = Column(Text) title = Column(Text) user_id = Column(Integer,ForeignKey('user.id')) user = relation(User, backref=backref('photos', order_by=id)) groups = relation('Group', secondary=flickr_group_photo, backref=backref('photos')) url = Column(Text) pub_date = Column(DateTime)
def __repr__(self): return "<Photo('%s','%s','%s','%s')>" % ( self.id, self.flickr_id, self.title, self.url )
Session = sessionmaker(bind = engine)session = Session()
for u in session.query(User): print u
for g in session.query(Group): print g
for p in session.query(Photo): print p
And more Advanced mapping Multiple table mapping
Elixir a simpler way of doing declarative ORM for
SQLAlchemy
from elixir import *
class User(Entity): using_options(tablename = 'user')
flickr_id = Field(Text) name = Field(Text) photos = OneToMany('Photo')
def __repr__(self): return "<User('%s','%s','%s')>" % ( self.id, self.flickr_id, self.name)
class Group(Entity): using_options(tablename = 'flickr_group') flickr_id = Field(Text) name = Field(Text) throttle_count = Field(Integer) throttle_mode = Field(Text) throttle_remaining = Field(Integer)
def __repr__(self): return "<Group('%s','%s','%s','%s')>" % ( self.id, self.flickr_id, self.name, self.throttle_remaining )
class Photo(Entity): using_options(tablename = 'photo')
flickr_id = Field(Text) title = Field(Text) url = Column(Text) pub_date = Column(DateTime) groups = ManyToMany('Group') user = ManyToOne('User')
def __repr__(self): return "<Photo('%s','%s','%s','%s')>" % ( self.id, self.flickr_id, self.title, self.url )
engine = create_engine( 'mysql://twl:pass@localhost/sqlobject', echo=True) Session = sessionmaker(bind = engine)session = Session()
metadata = MetaData()
metadata.bind='mysql://twl:pass@localhost/sqlobject'metadata.bind.echo = True
setup_all()
for u in session.query(User): print u
for g in session.query(Group): print g
for p in session.query(Photo): print print
SqlSoupfrom sqlalchemy.ext.sqlsoup import SqlSoup
soup = SqlSoup('mysql://twl:pass@localhost/sqlobject')
soup.photo.all()
[MappedPhoto(id=71L,flickr_id='3286255071',title='',user_id=1L,url='http://farm4.static.flickr.com/3249/3286255071_ff168f220b.jpg',pub_date=datetime.datetime(2009, 2, 16, 20, 47, 56)), MappedPhoto(id=72L,flickr_id='3287070244',title='',user_id=1L,url='http://farm4.static.flickr.com/3298/3287070244_87a2a1b3ed.jpg',pub_date=datetime.datetime(2009, 2, 16, 20, 47, 22)), MappedPhoto(id=46L,flickr_id='3395647634',title='Andy Dustman',user_id=1L,url='http://farm4.static.flickr.com/3554/3395647634_cc0c9f5a0a.jpg',pub_date=datetime.datetime(2009, 3, 29, 9, 7, 34)), MappedPhoto(id=73L,flickr_id='3277628077',title='Bainbridge Island Chinese New Year 2009',user_id=1L,url='http://farm4.static.flickr.com/3334/3277628077_78025002f5.jpg',pub_date=datetime.datetime(2009, 2, 13, 23, 31, 52)),
soup.photo.order_by(soup.photo.title).all()
Migrations sqlalchemy-migrate Version control a database A repository of upgrade scripts
from sqlalchemy import *from migrate import *
metadata = MetaData(migrate_engine)
photos_table = Table('photo', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('title', Text), Column('user_id', Integer), Column('url', Text), Column('pub_date', DateTime), )
visible_col = Column('visible', Integer)
def upgrade(): visible_col.create(photos_table) assert visible_col is photos_table.c.visible
def downgrade(): visible_col.drop()
from sqlalchemy import *from migrate import *
metadata = MetaData(migrate_engine)
photos_table = Table('photo', metadata, Column('id', Integer, primary_key=True), Column('flickr_id', Text), Column('title', Text), Column('user_id', Integer), Column('url', Text), Column('pub_date', DateTime), Column('visible', Integer) )
col = photos_table.c.visible
def upgrade(): col.alter(type=Boolean, nullable=False)
def downgrade(): col.alter(type=Integer, nullable=True)
Migration operations Create/Drop a table Create/Drop/Alter a column Create/Drop index Create/Drop primary/foreign key constraints
Summary Django for Django SQLAlchemy for the rest
