1 from scrapy import log 2 import pymysql 3 import pymysql.cursors 4 import codecs 5 from twisted.enterprise import adbapi 6 7 class MysqlTwistedPipeline(object): 8 9 @classmethod10 def from_settings(cls, settings):11 dbargs = dict(12 host=settings['MYSQL_HOST'],13 db=settings['MYSQL_DBNAME'],14 user=settings['MYSQL_USER'],15 passwd=settings['MYSQL_PASSWD'],16 port=settings['MYSQL_PORT'],17 charset='utf8',18 cursorclass=pymysql.cursors.DictCursor,19 use_unicode=True,20 )21 dbpool = adbapi.ConnectionPool('pymysql', **dbargs)22 return cls(dbpool)23 24 25 def __init__(self,dbpool):26 self.dbpool=dbpool27 28 #pipeline默认调用29 def process_item(self, item, spider):30 d=self.dbpool.runInteraction(self._conditional_insert, item, spider)#调用插入的方法31 log.msg("-------------------连接好了-------------------")32 d.addErrback(self._handle_error,item,spider)#调用异常处理方法33 d.addBoth(lambda _: item)34 return d35 36 def _conditional_insert(self, conn, item, spider):37 log.msg("-------------------打印-------------------")38 39 conn.execute("insert into MovieInfo (poster, name, director, actor, tag_char, score, critics_num,) values(%s, %s, %s,%s,%s,%s,%s,)",40 (item['cover'], item['title'], item['region'], item['years'], item['types'], item['score'], item['comment_num'],))41 log.msg("-------------------一轮循环完毕-------------------")42 def _handle_error(self, failue, item, spider):43 print(failue) 
