fetch google arts and culture bi
2019-07-05 本文已影响0人
狼无雨雪
"""
really used in fetching url from https://artsandculture.google.com/entity/m0bwbv?categoryid=art-movement
"""
from selenium import webdriver
import time
import os
from bs4 import BeautifulSoup
# os.environ["PATH"] += os.pathsep + 'D:\google-art-downloader-master'
browser = webdriver.Chrome()
browser.get('https://artsandculture.google.com/entity/m0bwbv?categoryid=art-movement')
asserts_all=set()
while browser.find_element_by_xpath('//*[@id="exp_tab_popular"]/div/div/div[2]').get_attribute("aria-hidden") != "true":
pageSource = browser.page_source
soup = BeautifulSoup(pageSource,'lxml')
asserts = soup.find_all('a',{'class':"e0WtYb kdYEFe ZEnmnd PJLMUc"})
for assert_value in asserts:
asserts_all.add(assert_value.get("href"))
print(len(asserts_all))
browser.find_element_by_xpath('//*[@id="exp_tab_popular"]/div/div/div[2]').click()
time.sleep(2)
with open("asserts.txt",'w',encoding="utf8") as write_file:
for line in asserts_all:
write_file.write(line+"\n")
browser.close()