python Selenium 简易爬虫

安装Selenium

终端命令pip install selenium

访问页面、在输入框中填写内容并模拟键盘操作、获取页面内容

from selenium import webdriver  #导入selenium
from selenium.webdriver.common.keys import Keys

browser = webdriver.Chrome()  #打开谷歌浏览器
browser.get('http://www.baidu.com/')  #访问url

#获取浏览器中 name 为 wd 的标签
elem = browser.find_element_by_name('wd')
#搜索 python
elem.send_keys('python')
elem.send_keys(Keys.RETURN)
#打印页面
print browser.page_source

改善获取的内容样式

from selenium import webdriver  #导入selenium
from selenium.webdriver.common.keys import Keys

browser = webdriver.Chrome()  #打开谷歌浏览器
browser.get('https://www.qiushibaike.com/')  #访问url

main_content = browser.find_element_by_id('content-left')
contents = main_content.find_elements_by_class_name('content')

i = 1
for content in contents:
    print(str(i) + '.' + content.text + '\n')
    i += 1

browser.quit()

用类写

from selenium import webdriver  #导入selenium

class Qiubai:
    def __init__(self):
        self.dr = webdriver.Chrome()
        self.dr.get('https://www.qiushibaike.com/')

    def print_content(self):
        main_content = self.dr.find_element_by_id('content-left')
        contents = main_content.find_elements_by_class_name('content')

        i = 1
        for content in contents:
            print(str(i) + '.' + content.text + '\n')
            i += 1

        self.quit()

    def quit(self):
        self.dr.quit()

Qiubai().print_content()