Scrapy 小技巧 - MySQL 存储

def process_item(self, item, spider):
 if isinstance(item, WhoscoredNewItem):
 table_name = item.pop('table_name')
 col_str = ''
 row_str = ''
 for key in item.keys():
 col_str = col_str + " " + key + ","
 row_str = "{}'{}',".format(row_str, item[key] if "'" not in item[key] else item[key].replace("'", "\\'"))
 sql = "insert INTO {} ({}) VALUES ({}) ON DUPLICATE KEY UPDATE ".format(table_name, col_str[1:-1], row_str[:-1])
 for (key, value) in six.iteritems(item):
 sql += "{} = '{}', ".format(key, value if "'" not in value else value.replace("'", "\\'"))
 sql = sql[:-2]
 self.cursor.execute(sql) #执行SQL
 self.cnx.commit()# 写入操作