工作生活

fetch google arts and culture bi

2019-07-05  本文已影响0人  狼无雨雪
"""
really used in fetching url from https://artsandculture.google.com/entity/m0bwbv?categoryid=art-movement
"""
from selenium import webdriver
import time
import os
from bs4 import BeautifulSoup
# os.environ["PATH"] += os.pathsep + 'D:\google-art-downloader-master'

browser = webdriver.Chrome()
browser.get('https://artsandculture.google.com/entity/m0bwbv?categoryid=art-movement')
asserts_all=set()
while browser.find_element_by_xpath('//*[@id="exp_tab_popular"]/div/div/div[2]').get_attribute("aria-hidden") != "true":
    pageSource = browser.page_source

    soup = BeautifulSoup(pageSource,'lxml')
    asserts = soup.find_all('a',{'class':"e0WtYb kdYEFe ZEnmnd PJLMUc"})
    for assert_value in asserts:
        asserts_all.add(assert_value.get("href"))
    print(len(asserts_all))
    browser.find_element_by_xpath('//*[@id="exp_tab_popular"]/div/div/div[2]').click()
    time.sleep(2)
with open("asserts.txt",'w',encoding="utf8") as write_file:
    for line in asserts_all:
        write_file.write(line+"\n")

browser.close()
上一篇下一篇

猜你喜欢

热点阅读