管道模块

# -*- coding: utf-8 -*-

# Define your item pipelines here

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymysql

class MyspiderPipeline(object):

def __int__(self):

print('my pipelines is ready....__init__ invoking....')

def open_spider(self,spider):

'''

        建立数据库连接

        :paramspider:

        :return:

'''

        self.my_conn = pymysql.connect(

host ="localhost",

port =3306,

database ="jobs",

user ='root',

password ='123',

charset ='utf8'

        )

#获取游标对象

        self.my_cursor =self.my_conn.cursor()

def process_item(self,item,spider):

print('----------------------process_item is invoking----------------------')

# print item['name']

# print item['salary']

# print item['company']

# print item['day']

# print item['experience']

# print item['area']

# print item['number']

# print item['nature']

# print item['education']

# print item['description']

        insert_sql ="insert into wuxi(name, salary, company, day,experience,area,number,nature,education,description) values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"

        self.my_cursor.execute(insert_sql,[item['name'],item['salary'],item['company'],item['day'],item['experience'],item['area'],item['number'],item['nature'],item['education'],item['description']])

def close_spider(self, spider):

#提交MySQL语句

        self.my_conn.commit()

#关闭游标对象与数据库连接

        self.my_cursor.close()

self.my_conn.close()

items

# -*- coding: utf-8 -*-

# Define here the models for your scraped items

#

# See documentation in:

# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class MyspiderItem(scrapy.Item):

# define the fields for your item here like:

# name = scrapy.Field()

    name = scrapy.Field()

salary = scrapy.Field()

company = scrapy.Field()

day = scrapy.Field()

experience = scrapy.Field()

area = scrapy.Field()

number = scrapy.Field()

nature = scrapy.Field()

education = scrapy.Field()

description = scrapy.Field()