zhihu_spider/crawler.py

917 lines
41 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

#################
###ZouJiu-202306
#################
from selenium import webdriver
from selenium.webdriver.edge.service import Service
from selenium.webdriver import EdgeOptions
import os
from selenium.webdriver.common.by import By
import time
import pickle
import json
from selenium.webdriver.support.wait import WebDriverWait
import requests
from copy import deepcopy
import argparse
from datetime import datetime
# from selenium.webdriver.common import keys
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.common.actions.wheel_input import ScrollOrigin
# import numpy as np
import shutil
# from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.print_page_options import PrintOptions
# from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support import expected_conditions as EC
import base64
from zipfile import ZipFile
abspath = os.path.abspath(__file__)
filename = abspath.split(os.sep)[-1]
abspath = abspath.replace(filename, "")
import sys
sys.path.append(abspath)
# wkhtmltopdf_path = os.path.join(abspath, r'wkhtmltopdf\bin\wkhtmltopdf.exe')
# sys.path.append(wkhtmltopdf_path)
from thinkdeal import *
def save_cookie(driverkkk, path):
#https://stackoverflow.com/questions/45417335/python-use-cookie-to-login-with-selenium
with open(path, 'wb') as filehandler:
pickle.dump(driverkkk.get_cookies(), filehandler)
def load_cookie(driverkkk, path):
#https://stackoverflow.com/questions/45417335/python-use-cookie-to-login-with-selenium
with open(path, 'rb') as cookiesfile:
cookies = pickle.load(cookiesfile)
for cookie in cookies:
driverkkk.add_cookie(cookie)
def crawlsleep(times):
time.sleep(times)
def now():
return time.time()
def nowtime():
nowtm = datetime.fromtimestamp(time.time()).isoformat().replace(":", "_")
return nowtm
def edgeopen(driverpath):
service=Service(executable_path=driverpath)
edge_options = EdgeOptions()
#https://stackoverflow.com/questions/53039551/selenium-webdriver-modifying-navigator-webdriver-flag-to-prevent-selenium-detec
edge_options.add_experimental_option('excludeSwitches', ['enable-automation'])
edge_options.add_experimental_option('useAutomationExtension', False)
edge_options.add_argument('lang=zh-CN,zh,zh-TW,en-US,en')
edge_options.add_argument("disable-blink-features=AutomationControlled")#就是这一行告诉chrome去掉了webdriver痕迹
# #https://stackoverflow.com/questions/56897041/how-to-save-opened-page-as-pdf-in-selenium-python
# settings = {
# "recentDestinations": [{
# "id": "Save as PDF",
# "origin": "local",
# "account": "",
# }],
# "selectedDestinationId": "Save as PDF",
# "version": 2
# }
# #https://www.apiref.com/java11-zh/java.desktop/javax/print/attribute/standard/MediaSize.ISO.html
# settings = {
# "recentDestinations": [{
# "id": "Save as PDF",
# "origin": "local",
# "account": ""
# }],
# "selectedDestinationId": "Save as PDF",
# "version": 2,
# "isHeaderFooterEnabled": False,
# "mediaSize": {
# "height_microns": 297000,
# "name": "ISO_A4",
# "width_microns": 210000,
# "custom_display_name": "A4"
# },
# "customMargins": {"margin": 0},
# "marginsType": 3,
# # "scaling": 130,
# # "scalingType": 3,
# # "scalingTypePdf": 3,
# "isCssBackgroundEnabled": True
# }
# prefs = {
# 'printing.print_preview_sticky_settings.appState': json.dumps(settings),
# 'savefile.default_directory': pdfpath,
# }
# edge_options.add_experimental_option('prefs', prefs)
# edge_options.add_argument('--kiosk-printing')
# https://www.selenium.dev/documentation/webdriver/drivers/options/#pageloadstrategy
# https://stackoverflow.com/questions/44503576/selenium-python-how-to-stop-page-loading-when-certain-element-gets-loaded
# edge_options.add_argument(page_load_strategy, 'normal')
# if strategy:
edge_options.page_load_strategy = 'normal'
# cap = DesiredCapabilities.EDGE
# cap['pageLoadStrategy'] = 'none'
driver = webdriver.Edge(options=edge_options, service = service)
driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
driver.execute_cdp_cmd('Network.setUserAgentOverride', {"userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.53 Safari/537.36'})
driver.set_script_timeout(20)
return driver
def login(driver):
driver.find_elements(By.CLASS_NAME, "SignFlow-tab")[1].click()
# driver.find_elements(By.CLASS_NAME, "username-input")[0].send_keys("")
# driver.find_elements(By.CLASS_NAME, "username-input")[1].send_keys("")
# driver.find_element(By.CLASS_NAME, "SignFlow-submitButton").click()
time.sleep(130)
# WebDriverWait(driver, timeout=60).until(lambda d:d.find_element(By.CLASS_NAME, "TopstoryTabs-link"))
return driver
def crawl_article_links(driver:webdriver, username:str):
#crawl articles links
articles = r'https://www.zhihu.com/people/zoujiu1/posts'
articles_one = r'https://www.zhihu.com/people/zoujiu1/posts?page='
article_detail = r'https://zhuanlan.zhihu.com/p/'
driver.get(articles.replace("zoujiu1", username))
pages = driver.find_elements(By.CLASS_NAME, 'PaginationButton')[-2]
assert isinstance(int(pages.text), int)
maxpages = int(pages.text)
all_article_detail = {}
#how many pages of articles
for p in range(1, maxpages + 1):
driver.get(articles_one + str(p))
items = driver.find_elements(By.CLASS_NAME, "ArticleItem")
#crawl article one by one
for a in range(len(items)):
introduce = items[a].get_attribute("data-zop")
itemId = json.loads(introduce)
links = items[a].find_elements(By.TAG_NAME, 'a')[0].get_attribute('href')
# id = itemId['itemId']
title = str(itemId['title'])
all_article_detail[str(title)] = links #article_detail + str(id)
crawlsleep(sleeptime)
with open(os.path.join(articledir, 'article.txt'), 'w', encoding='utf-%d'%(6+2)) as obj:
for key, val in all_article_detail.items():
obj.write(val + " " + key + '\n')
def crawl_answers_links(driver:webdriver, username:str):
#crawl answers links
answer = r'https://www.zhihu.com/people/zoujiu1/answers'
answer_one = r'https://www.zhihu.com/people/zoujiu1/answers?page='
driver.get(answer.replace("zoujiu1", username))
WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.CLASS_NAME, "Pagination"))
pages = driver.find_elements(By.CLASS_NAME, 'PaginationButton')[-2]
assert isinstance(int(pages.text), int)
maxpages = int(pages.text)
all_answer_detail = []
#how many pages of answers
for p in range(1, maxpages + 1):
driver.get(answer_one + str(p))
WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.CLASS_NAME, "Pagination"))
items = driver.find_elements(By.CLASS_NAME, "AnswerItem")
#crawl answer one by one
for i in range(len(items)):
introduce = items[i].get_attribute("data-zop")
itemId = json.loads(introduce)
id = itemId['itemId']
title = str(itemId['title'])
links = items[i].find_elements(By.TAG_NAME, 'a')[0].get_attribute('href')
all_answer_detail.append([links, str(title)])
crawlsleep(sleeptime)
with open(os.path.join(answerdir, 'answers.txt'), 'w', encoding='utf-8') as obj:
for links, title in all_answer_detail:
obj.write(links + " " + title + '\n')
def crawl_think_links(driver:webdriver, username:str):
#crawl think links
think = r'https://www.zhihu.com/people/zoujiu1/pins'
think_one = r'https://www.zhihu.com/people/zoujiu1/pins?page='
driver.get(think.replace("zoujiu1", username))
WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.CLASS_NAME, "Pagination"))
pages = driver.find_elements(By.CLASS_NAME, 'PaginationButton')[-2]
assert isinstance(int(pages.text), int)
maxpages = int(pages.text)
# all_think_detail = []
#how many pages of think
allbegin = now()
numberpage = 1e-6
for p in range(1, maxpages + 1):
begin = now()
driver.get(think_one + str(p))
WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.CLASS_NAME, "Pagination"))
items = driver.find_elements(By.CLASS_NAME, "PinItem")
#crawl answer one by one
for i in range(len(items)):
RichContent = items[i].find_element(By.CLASS_NAME, 'RichContent-inner')
clockitem = items[i].find_element(By.CLASS_NAME, 'ContentItem-time')
try:
WebDriverWait(items[i], timeout=10).until(lambda d: len(d.text) > 2)
except:
driver.get(think_one + str(p))
WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.CLASS_NAME, "Pagination"))
items = driver.find_elements(By.CLASS_NAME, "PinItem")
RichContent = items[i].find_element(By.CLASS_NAME, 'RichContent-inner')
clockitem = items[i].find_element(By.CLASS_NAME, 'ContentItem-time')
WebDriverWait(items[i], timeout=10).until(lambda d: len(d.text) > 2)
# clockspan = clockitem.find_element(By.TAG_NAME, 'span')
clock = clockitem.text
clock = clock[3 + 1:].replace(" ", "_").replace(":", "_")
dirthink = os.path.join(thinkdir, clock)
if os.path.exists(dirthink):
continue
os.makedirs(dirthink, exist_ok=True)
try:
RichContent.find_element(By.CLASS_NAME, 'Button').click()
WebDriverWait(items[i], timeout=10).until(lambda d: d.find_element(By.CLASS_NAME, "RichContent-inner"))
RichContent = items[i].find_element(By.CLASS_NAME, 'RichContent-inner')
except:
pass
content = RichContent.find_element(By.CLASS_NAME, 'RichText')
links_col = content.find_elements(By.TAG_NAME, 'a')
links = []
for itext in links_col:
try:
links.append(itext.get_attribute("href"))
except:
continue
text = content.text.strip()
if len(text)!=0:
with open(os.path.join(dirthink, clock+".txt"), 'w', encoding='utf-8') as obj:
obj.write(text.replace("<br>", '\n').replace("<br data-first-child=\"\">", '\n')+"\n")
for itext in links:
obj.write(itext + "\n")
# all_think_detail.append([text])
try:
items[i].find_elements(By.CLASS_NAME, 'Image-PreviewVague')[0].click()
except:
continue
cnt = 0
while True:
WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.CLASS_NAME, "ImageGallery-Inner"))
img = driver.find_element(By.CLASS_NAME, 'ImageGallery-Inner').find_element(By.TAG_NAME, 'img')
imglink = img.get_attribute('data-original')
if imglink==None:
imglink = img.get_attribute("src")
try:
response = requests.get(imglink, timeout=30)
except:
try:
response = requests.get(imglink, timeout=30)
except:
continue
if response.status_code==200:
with open(os.path.join(dirthink, clock + "_" + str(cnt) + '.jpg'), 'wb') as obj:
obj.write(response.content)
cnt += 1
crawlsleep(sleeptime)
try:
disable = driver.find_element(By.CLASS_NAME, 'ImageGallery-arrow-right')
if 'disabled' in disable.get_attribute('class'):
driver.find_element(By.CLASS_NAME, 'ImageGallery-close').click()
break
else:
disable.click()
except:
break
crawlsleep(sleeptime)
end = now()
print("爬取一篇想法耗时:", clock, round(end - begin, 3))
logfp.write("爬取一篇想法耗时:" +clock + " "+ str(round(end - begin, 3)) + "\n")
numberpage += 1
# crawlsleep(600)
allend = now()
print("平均爬取一篇想法耗时:", round((allend - allbegin) / numberpage, 3))
logfp.write("平均爬取一篇想法耗时:" + str(round((allend - allbegin) / numberpage, 3)) + "\n")
dealthink(thinkdir)
def cleartxt(kkk):
while ' 'in kkk:
kkk = kkk.replace(" ", "")
while "\n" in kkk:
kkk = kkk.replace("\n", "")
return kkk
def recursion(nod, article, number, driver, dircrea):
if isinstance(nod, dict):
if 'nodeName' in nod.keys() and nod['nodeName']=='#text':
kkk = cleartxt(nod['textContent'])
if len(kkk) > 0:
article += nod['textContent']
return article, number
elif isinstance(nod, webdriver.remote.webelement.WebElement):
tag_name = nod.tag_name
if tag_name=="br":
article += "<br>\n"
elif tag_name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
article += "\n" + '#' * int(tag_name[-1]) + ' '
p_childNodes = driver.execute_script("return arguments[0].childNodes;", nod)
for pnode in p_childNodes:
article, number = recursion(pnode, article, number, driver, dircrea)
article += '\n'
elif tag_name=="span":
datatex = nod.get_attribute("data-tex")
classc = nod.get_attribute("class")
if datatex and classc and 'ztext-math' in classc:
if article[-3-1:]=='<br>' or article[-1:]=='\n':
article += "\n$" + nod.get_attribute("data-tex") + "$"
else:
article += "$" + nod.get_attribute("data-tex") + "$"
else:
imgchunk = nod.find_elements(By.TAG_NAME, 'img')
achunk = nod.find_elements(By.TAG_NAME, 'a')
if len(imgchunk)==0 and len(achunk)==0:
article += nod.text
else:
p_childNodes = driver.execute_script("return arguments[0].childNodes;", nod)
for pnode in p_childNodes:
article, number = recursion(pnode, article, number, driver, dircrea)
# else:
# formula_span = nod.find_elements(By.CLASS_NAME, "ztext-math")
# for jf in range(len(formula_span)):
# ele = formula_span[jf]
# article += "$" + ele.get_attribute("data-tex") + "$"
elif tag_name=="a":
linksite = nod.get_attribute("href")
if linksite:
linksite = linksite.replace("//link.zhihu.com/?target=https%3A", "").replace("//link.zhihu.com/?target=http%3A", "")
if article[-3-1:]=='<br>' or article[-1:]=='\n':
article += "\n\n["+nod.text+"]"+"("+linksite + ")"
else:
article += "["+nod.text+"]"+"("+linksite + ")"
elif tag_name=="b" or tag_name=="strong":
txt = nod.text
while len(txt) > 0 and txt[-1] == " ":
txt = txt[:-1]
article += " **" + txt + "** "
elif tag_name=="em":
article += nod.text
# elif tag_name=='td':
# article += nod.text
elif tag_name in ['table', 'tbody', 'tr', 'td', 'u']:
p_childNodes = driver.execute_script("return arguments[0].childNodes;", nod)
for pnode in p_childNodes:
article, number = recursion(pnode, article, number, driver, dircrea)
elif tag_name=='p':
p_childNodes = driver.execute_script("return arguments[0].childNodes;", nod)
for pnode in p_childNodes:
article, number = recursion(pnode, article, number, driver, dircrea)
article += "\n"
elif tag_name=="div":
# atags = nod.find_elements(By.TAG_NAME, 'a')
prenode = nod.find_elements(By.TAG_NAME, 'code')
if len(prenode) > 0:
for i in prenode:
article += "<br>\n```\n" + i.text + "\n```\n<br>"
else:
p_childNodes = driver.execute_script("return arguments[0].childNodes;", nod)
for pnode in p_childNodes:
article, number = recursion(pnode, article, number, driver, dircrea)
elif tag_name=="figure":
imgchunk = nod.find_elements(By.TAG_NAME, 'img')
for i in range(len(imgchunk)):
imglink = imgchunk[i].get_attribute("data-original")
if imglink==None:
imglink = imgchunk[i].get_attribute("src")
try:
response = requests.get(imglink, timeout=30)
except:
try:
response = requests.get(imglink, timeout=30)
except:
continue
if response.status_code==200:
article += ''' <img src="%d.jpg" width="100%%"/> '''%number
# article += '''<img src="%d.jpg"/>'''%number
with open(os.path.join(dircrea, str(number) + '.jpg'), 'wb') as obj:
obj.write(response.content)
number += 1
crawlsleep(sleeptime)
return article, number
def crawl_article_detail(driver:webdriver):
website_col = {}
for i in os.listdir(articledir):
try:
kk = int(i)
shutil.rmtree(os.path.join(articledir, i))
except:
pass
with open(os.path.join(articledir, 'article.txt'), 'r', encoding='utf-8') as obj:
for i in obj.readlines():
i = i.strip()
ind = i.index(" ")
website = i[:ind]
title = i[ind+1:].replace(" ", "_").replace("\n", "")
website_col[website] = title
allbegin = now()
numberpage = 1e-6
for website, title in website_col.items():
begin = now()
nam = title.replace(":", "_").replace("?", ";"). \
replace("/","_").replace("\\","_").replace("\"", "_").\
replace("*","_").replace("|", "_").replace(" ", "_").replace("", "").replace("", "").\
replace("<", "小于").replace(">", "大于").replace("(", "").\
replace(")", "")
temp_name = nam #str(np.random.randint(999999999)) + str(np.random.randint(999999999))
if len(temp_name) > 200:
temp_name = temp_name[:100]
# nam_pinyin = pinyin.get(nam, format='numerical')
# if '租房' not in title:
# continue
direxit = False
fileexit = False
dirname = ''
filesize = 0
kkk = -9
for i in os.listdir(articledir):
if nam in i and os.path.isdir(os.path.join(articledir, i)):
direxit = True
dirname = i
fileexit = os.path.exists(os.path.join(articledir, dirname, nam + ".pdf"))
if fileexit:
filesize = os.path.getsize(os.path.join(articledir, dirname, nam + ".pdf"))
# break
if direxit and fileexit and filesize > 0:
if '_IP_' in dirname:
filnam = dirname[16+1:].split("_IP_")[0]
else:
filnam = dirname[16+1:][:-1]
if filnam == nam:
kkk = 9
break
if kkk > 0:
continue
dircrea = os.path.join(articledir, temp_name)
os.makedirs(dircrea, exist_ok = True)
#get article text
driver.get(website)
WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.CLASS_NAME, "Post-Topics"))
#https://stackoverflow.com/questions/61877719/how-to-get-current-scroll-height-in-selenium-python
scrollHeight = driver.execute_script('''return document.documentElement.scrollHeight''')
footer = driver.find_element(By.TAG_NAME, "html")
scroll_origin = ScrollOrigin.from_element(footer, 0, -60)
ActionChains(driver).scroll_from_origin(scroll_origin, 0, -100000).perform()
for i in range(18):
try:
ActionChains(driver).scroll_from_origin(scroll_origin, 0, scrollHeight//18).perform()
except:
try:
ActionChains(driver).scroll_from_origin(scroll_origin, 0, -scrollHeight//18).perform()
except:
pass
crawlsleep(0.8)
#remove noneed element
try:
driver.execute_script('''document.getElementsByClassName("Post-Sub")[0].remove();''')
except:
pass
try:
driver.execute_script('''document.getElementsByClassName("ColumnPageHeader-Wrapper")[0].remove();''')
except:
pass
try:
driver.execute_script('''document.getElementsByClassName("RichContent-actions")[0].remove();''')
except:
pass
richtext = driver.find_element(By.CLASS_NAME, "Post-RichText")
titletext = driver.find_element(By.CLASS_NAME, "Post-Title")
article_childNodes = driver.execute_script("return arguments[0].childNodes;", richtext)
article = ""
number = 0
for nod in article_childNodes:
article, number = recursion(nod, article, number, driver, dircrea)
article = article.replace("修改\n", "").replace("开启赞赏\n", "开启赞赏, ").replace("添加评论\n", "").replace("分享\n", "").\
replace("收藏\n", "").replace("设置\n", "")
tle = titletext.text
article += "<br>\n\n["+driver.current_url+"](" + driver.current_url + ")<br>\n"
if len(article) > 0:
try:
f=open(os.path.join(dircrea, nam + "_formula_" + ".md"), 'w', encoding='utf-8')
f.close()
except:
nam = nam[:len(nam)//2]
with open(os.path.join(dircrea, nam + "_formula_" + ".md"), 'w', encoding='utf-8') as obj:
obj.write("# " + tle+"\n\n")
if len(article) > 0:
obj.write(article + "\n\n\n")
# article to pdf
clocktxt = driver.find_element(By.CLASS_NAME, "Post-NormalMain").find_element(By.CLASS_NAME, "ContentItem-time")
crawlsleep(1)
url = driver.current_url
driver.execute_script("const para = document.createElement(\"h2\"); \
const br = document.createElement(\"br\"); \
const node = document.createTextNode(\"%s\");\
para.appendChild(node);\
const currentDiv = document.getElementsByClassName(\"Post-Header\")[0];\
currentDiv.appendChild(br); \
currentDiv.appendChild(para);"%url \
)
clock = clocktxt.text[3+1:].replace(" ", "_").replace(":", "_")
pagetopdf(driver, dircrea, temp_name, nam, articledir, url, Created=clock)
crawlsleep(sleeptime)
#https://stackoverflow.com/questions/23359083/how-to-convert-webpage-into-pdf-by-using-python
#https://github.com/JazzCore/python-pdfkit
# if article_to_jpg_pdf_markdown:
# config = pdfkit.configuration(wkhtmltopdf = wkhtmltopdf_path)
# pdfkit.from_url(website, os.path.join(dircrea, nam_pinyin+".pdf"), configuration = config)
end = now()
print("爬取一篇article耗时", title, round(end - begin, 3))
logfp.write("爬取一篇article耗时" + title + " " + str(round(end - begin, 3)) + "\n")
numberpage += 1
# crawlsleep(600)
allend = now()
print("平均爬取一篇article耗时", round((allend - allbegin) / numberpage, 3))
logfp.write("平均爬取一篇article耗时" + str(round((allend - allbegin) / numberpage, 3)) + "\n")
def pagetopdf(driver, dircrea, temp_name, nam, destdir, url, Created=""):
fileexit = os.path.exists(os.path.join(dircrea, temp_name + ".pdf"))
if fileexit:
try:
os.remove(os.path.join(dircrea, temp_name + ".pdf"))
except:
pass
printop = PrintOptions()
# printop.shrink_to_fit = True
# printop.margin_left = 0
# printop.margin_right = 0
# printop.margin_top = 0
# printop.margin_bottom = 0
printop.page_height = 29.7
printop.page_width = 21
printop.background = True
# printop.scale = 1.0
pdf = driver.print_page(print_options=printop)
with open(os.path.join(dircrea, nam + ".pdf"), 'wb') as obj:
obj.write(base64.b64decode(pdf))
# driver.execute_script('window.print();')
clock = Created #clocktxt.text[3+1:].replace(" ", "_").replace(":", "_")
with open(os.path.join(dircrea, clock+".txt"), 'w', encoding='utf-8') as obj:
obj.write(clock+"\n")
obj.write(url)
clocktmp = clock.split(".")[0].replace("T", "_")
clock = clocktmp.split("")[0]
address = ""
try:
address += clocktmp.split("")[1]
except:
pass
try:
os.rename(dircrea, os.path.join(destdir, clock + "_" + nam + "_" + address))
except:
crawlsleep(3+addtime)
try:
os.rename(dircrea, os.path.join(destdir, clock + "_" + nam + "_" + address))
except:
pass
def crawl_answer_detail(driver:webdriver):
website_col = {}
for i in os.listdir(answerdir):
try:
kk = int(i)
shutil.rmtree(os.path.join(answerdir, i))
except:
pass
with open(os.path.join(answerdir, 'answers.txt'), 'r', encoding='utf-8') as obj:
for i in obj.readlines():
i = i.strip()
ind = i.index(" ")
website = i[:ind]
title = i[ind+1:].replace(" ", "_").replace("\n", "")
website_col[website] = title
allbegin = now()
numberpage = 1e-6
for website, title in website_col.items():
begin = now()
nam = title.replace(":", "_").replace("?", ";"). \
replace("/","_").replace("\\","_").replace("\"", "_").\
replace("*","_").replace("|", "_").replace(" ", "_").replace("", "").replace("", "").\
replace("<", "小于").replace(">", "大于")
if len(nam) > 200:
nam = nam[:100]
temp_name = nam #str(np.random.randint(999999999)) + str(np.random.randint(999999999))
# nam_pinyin = pinyin.get(nam, format='numerical')
# if '不定积分该用什么方' not in title:
# continue
direxit = False
fileexit = False
dirname = ''
filesize = 0
for i in os.listdir(answerdir):
if nam in i and os.path.isdir(os.path.join(answerdir, i)):
direxit = True
dirname = i
fileexit = os.path.exists(os.path.join(answerdir, dirname, nam + ".pdf"))
if fileexit:
filesize = os.path.getsize(os.path.join(answerdir, dirname, nam + ".pdf"))
break
dircrea = os.path.join(answerdir, temp_name)
if direxit and fileexit and filesize > 0:
if '_IP_' in dirname:
filnam = dirname[16+1:].split("_IP_")[0]
else:
filnam = dirname[16+1:][:-1]
if filnam == nam:
continue
os.makedirs(dircrea, exist_ok = True)
#get article text
driver.get(website)
WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.CLASS_NAME, "AnswerItem-editButtonText"))
#https://stackoverflow.com/questions/61877719/how-to-get-current-scroll-height-in-selenium-python
scrollHeight = driver.execute_script('''return document.getElementsByClassName("QuestionAnswer-content")[0].scrollHeight''')
footer = driver.find_element(By.TAG_NAME, "html")
scroll_origin = ScrollOrigin.from_element(footer, 0, 0)
ActionChains(driver).scroll_from_origin(scroll_origin, 0, -100000).perform()
for i in range(18):
try:
ActionChains(driver).scroll_from_origin(scroll_origin, 0, scrollHeight//18).perform()
except:
try:
ActionChains(driver).scroll_from_origin(scroll_origin, 0, -scrollHeight//18).perform()
except:
pass
crawlsleep(0.8)
ActionChains(driver).scroll_from_origin(scroll_origin, 0, -100000).perform()
article = ""
number = 0
try:
QuestionRichText = driver.find_element(By.CLASS_NAME, "QuestionRichText")
button = QuestionRichText.find_element(By.CLASS_NAME, "QuestionRichText-more")
WebDriverWait(driver, timeout=20).until(EC.element_to_be_clickable((By.CLASS_NAME, "QuestionRichText-more")))
crawlsleep(max(2, sleeptime))
button.click()
question_RichText = QuestionRichText.find_element(By.CLASS_NAME, "RichText")
question_childNodes = driver.execute_script("return arguments[0].childNodes;", question_RichText)
article += "# question <br>\n"
for nod in question_childNodes:
article, number = recursion(nod, article, number, driver, dircrea)
except:
pass
article += "<br>\n\n\n# answer <br>\n"
#remove noneed element
try:
driver.execute_script('''document.getElementsByClassName("MoreAnswers")[0].remove();''')
except:
pass
try:
driver.execute_script('''document.getElementsByClassName("ViewAll")[0].remove();''')
except:
pass
try:
driver.execute_script('''document.getElementsByClassName("ViewAll")[0].remove();''')
except:
pass
try:
driver.execute_script('''document.getElementsByClassName("AppHeader")[0].remove();''')
except:
pass
try:
driver.execute_script('''document.getElementsByClassName("Reward")[0].remove();''')
except:
pass
try:
driver.execute_script('''document.getElementsByClassName("Question-sideColumn")[0].remove();''')
except:
pass
Created = "not found"
Modified = "not found"
QuestionAnswer = driver.find_element(By.CLASS_NAME, "QuestionAnswer-content")
richtext = QuestionAnswer.find_element(By.CLASS_NAME, "CopyrightRichText-richText")
Createdtime = QuestionAnswer.find_element(By.CLASS_NAME, "ContentItem-time")
Created = Createdtime.text[4:].replace(" ", "_").replace(":", "_").replace(".", "_")
metatext = QuestionAnswer.find_elements(By.TAG_NAME, "meta")
for i in range(len(metatext)):
# if metatext[i].get_attribute("itemprop")=="dateCreated":
# Created = metatext[i].get_attribute("content").replace(" ", "_").replace(":", "_").replace(".", "_")
if metatext[i].get_attribute("itemprop")=="dateModified":
Modified = metatext[i].get_attribute("content").replace(" ", "_").replace(":", "_").replace(".", "_")
answer_childNodes = driver.execute_script("return arguments[0].childNodes;", richtext)
for nod in answer_childNodes:
article, number = recursion(nod, article, number, driver, dircrea)
article = article.replace("修改\n", "").replace("开启赞赏\n", "开启赞赏, ").replace("添加评论\n", "").replace("分享\n", "").\
replace("收藏\n", "").replace("设置\n", "")
url = driver.current_url
article += "<br>\n\n["+url+"](" + url + ")<br>\n"
driver.execute_script("const para = document.createElement(\"h2\"); \
const br = document.createElement(\"br\"); \
const node = document.createTextNode(\"%s\");\
para.appendChild(node);\
const currentDiv = document.getElementsByClassName(\"QuestionHeader-title\")[0];\
currentDiv.appendChild(br); \
currentDiv.appendChild(para);"%url \
)
if len(article) > 0:
try:
f=open(os.path.join(dircrea, nam + "_formula_" + ".md"), 'w', encoding='utf-8')
f.close()
except:
nam = nam[:len(nam)//2]
with open(os.path.join(dircrea, nam + "_formula_" + ".md"), 'w', encoding='utf-8') as obj:
obj.write("# "+ title+"\n\n")
if len(article) > 0:
obj.write(article + "\n\n\n")
obj.write("Created: " + Created + "\n")
obj.write("Modified: " + Modified + "\n")
# article to pdf
pagetopdf(driver, dircrea, temp_name, nam, answerdir, url, Created=Created)
crawlsleep(sleeptime)
end = now()
print("爬取一篇回答耗时:", title, round(end - begin, 3))
logfp.write("爬取一篇回答耗时:" +title+" "+ str(round(end - begin, 3)) + "\n")
numberpage += 1
# crawlsleep(600)
allend = now()
print("平均爬取一篇回答耗时:", round((allend - allbegin) / numberpage, 3))
logfp.write("平均爬取一篇回答耗时:" + str(round((allend - allbegin) / numberpage, 3)) + "\n")
def login_loadsavecookie():
website = r"https://www.zhihu.com/signin"
#login and save cookies of zhihu
driver = edgeopen(driverpath)
driver.get(website)
try:
load_cookie(driver, cookie_path)
driver.get(r"https://www.zhihu.com/")
except:
driver = login(driver)
save_cookie(driver, cookie_path)
driver.quit()
exit(0)
try:
driver.find_element(By.ID, 'Popover15-toggle').click()
driver.find_element(By.CLASS_NAME, 'Menu-item').click()
except:
crawlsleep(6)
driver.get(r"https://www.zhihu.com/")
crawlsleep(3)
driver.find_element(By.ID, 'Popover15-toggle').click()
driver.find_element(By.CLASS_NAME, 'Menu-item').click()
url = driver.current_url
username = url.split("/")[-1]
return driver, username
def zhihu():
# #crawl articles links
if not os.path.exists(driverpath):
response = requests.get("https://msedgedriver.azureedge.net/114.0.1823.67/edgedriver_win64.zip")
if response.status_code==200:
with open(os.path.join(abspath, 'msedgedriver/edgedriver.zip'), 'wb') as obj:
obj.write(response.content)
with ZipFile(os.path.join(abspath, 'msedgedriver/edgedriver.zip'), "r") as obj:
obj.extractall(os.path.join(abspath, 'msedgedriver'))
nth = os.path.join(abspath, 'msedgedriver')
for r, d, f in os.walk(nth):
kk = 6
for i in f:
if 'driver' in i and '.exe' in i:
try:
shutil.move(os.path.join(r, i), os.path.join(nth, i))
except:
pass
os.rename(os.path.join(nth, i), os.path.join(nth, "msedgedriver.exe"))
kk = -6
break
if kk < 0:
break
driver, username = login_loadsavecookie()
# #crawl think links
if crawl_think:
crawl_think_links(driver, username)
logfp.write(nowtime() + ', 想法爬取已经好了的\n')
# #crawl articles links
if crawl_article:
if not os.path.exists(os.path.join(articledir, 'article.txt')):
crawl_article_links(driver, username)
logfp.write(nowtime() + ', article weblink爬取已经好了的\n')
else:
if crawl_links_scratch:
os.rename(os.path.join(articledir, 'article.txt'), os.path.join(articledir, 'article_%s.txt'%nowtime()))
crawl_article_links(driver, username)
logfp.write(nowtime() + ', article weblink爬取已经好了的\n')
else:
pass
crawl_article_detail(driver)
logfp.write(nowtime() + ', article爬取已经好了的\n')
# #crawl answers links
if crawl_answer:
if not os.path.exists(os.path.join(answerdir, 'answers.txt')):
crawl_answers_links(driver, username)
logfp.write(nowtime() + ', 回答 weblink爬取已经好了的\n')
else:
if crawl_links_scratch:
os.rename(os.path.join(answerdir, 'answers.txt'), os.path.join(answerdir, 'answers_%s.txt'%nowtime()))
crawl_answers_links(driver, username)
logfp.write(nowtime() + ', 回答 weblink爬取已经好了的\n')
else:
pass
crawl_answer_detail(driver)
logfp.write(nowtime() + ', 回答爬取已经好了的\n')
driver.quit()
if __name__ == "__main__":
#version four.one_zero.zero
driverpath = os.path.join(abspath, 'msedgedriver\msedgedriver.exe')
savepath = deepcopy(abspath)
cookiedir = os.path.join(savepath, 'cookie')
thinkdir = os.path.join(savepath, 'think')
answerdir = os.path.join(savepath, 'answer')
articledir = os.path.join(savepath, 'article')
logdir = os.path.join(savepath, 'log')
logfile = os.path.join(logdir, nowtime() + '_log.txt')
os.makedirs(cookiedir, exist_ok=True)
os.makedirs(thinkdir, exist_ok=True)
os.makedirs(answerdir, exist_ok=True)
os.makedirs(articledir, exist_ok=True)
os.makedirs(logdir, exist_ok=True)
logfp = open(logfile, 'w', encoding='utf-8')
cookie_path =os.path.join(cookiedir, 'cookie_zhihu.pkl')
parser = argparse.ArgumentParser(description=r'crawler zhihu.com, 爬取知乎的想法, 回答, 文章, 包括数学公式')
parser.add_argument('--sleep_time', type=float, default = 6, \
help=r'crawler sleep time during crawling, 爬取时的睡眠时间, 避免给知乎服务器带来太大压力, \
可以日间调试好,然后深夜运行爬取人少, 给其他小伙伴更好的用户体验, 避免知乎顺着网线过来找人, 默认: 6s')
parser.add_argument('--computer_time_sleep', type=float, default=0, \
help=r'computer running sleep time 默认:0, 电脑运行速度的sleep时间, 默认:0')
parser.add_argument('--think', action="store_true", help=r'crawl think, 是否爬取知乎的想法, 已经爬取过的想法不会重复爬取, 所以可以多次爬取断了也没关系')
parser.add_argument('--answer', action="store_true", help=r'crawl answer, 是否爬取知乎的回答, 保存到pdf、markdown以及相关图片等已经爬取过的不会重复爬取\
断了再次爬取的话,可以配置到--links_scratch事先保存好website')
parser.add_argument('--article', action="store_true", help=r'crawl article, 是否爬取知乎的文章, 保存到pdf、markdown以及相关图片等已经爬取过的不会重复爬取\
断了再次爬取的话,可以配置到--links_scratch事先保存好website')
parser.add_argument('--links_scratch', action="store_true", \
help=r'crawl links scratch for answer or article, 是否使用已经保存好的website和title, 否则再次爬取website')
args = parser.parse_args()
sleeptime = args.sleep_time
crawl_think = args.think
crawl_answer = args.answer
crawl_article = args.article
crawl_links_scratch = args.links_scratch
addtime = args.computer_time_sleep
# crawl_think = False
# crawl_article = False
# crawl_answer = True
# crawl_links_scratch = False
# python.exe c:/Users/10696/Desktop/access/zhihu/crawler.py --think
# python.exe c:/Users/10696/Desktop/access/zhihu/crawler.py --article
# python.exe c:/Users/10696/Desktop/access/zhihu/crawler.py --answer
# python.exe c:/Users/10696/Desktop/access/zhihu/crawler.py --think --answer --article
zhihu()
try:
crawl_links_scratch = False
zhihu()
except:
time.sleep(600)
try:
zhihu()
except:
time.sleep(600)
zhihu()
logfp.close()