186 lines
7.3 KiB
Python
186 lines
7.3 KiB
Python
from selenium import webdriver
|
||
from bs4 import BeautifulSoup
|
||
from markdownify import markdownify as md
|
||
import time
|
||
import os
|
||
from selenium.webdriver.edge.service import Service
|
||
from selenium.webdriver import EdgeOptions
|
||
import requests
|
||
import json
|
||
from selenium.webdriver.common.by import By
|
||
|
||
def crawlsleep(times):
|
||
time.sleep(times)
|
||
|
||
def parser_beautiful(innerHTML, article, number, dircrea, bk=False):
|
||
if not innerHTML:
|
||
return article, number
|
||
if bk:
|
||
article += "**"
|
||
if isinstance(innerHTML, str):
|
||
article += innerHTML.text
|
||
return article, number
|
||
|
||
for chi in innerHTML.children:
|
||
# article, number = parser_beautiful(chi, article, number, dircrea, bk)
|
||
tag_name = chi.name
|
||
if isinstance(chi, str):
|
||
article += chi.text
|
||
continue
|
||
else:
|
||
cll = [c for c in chi.children]
|
||
if tag_name in ['table', 'tbody', 'tr', 'td', 'u', 'em']:
|
||
article, number = parser_beautiful(chi, article, number, dircrea, bk)
|
||
elif tag_name=="br":
|
||
article += "\n"
|
||
elif tag_name=="p":
|
||
article, number = parser_beautiful(chi, article, number, dircrea, bk)
|
||
article += "\n\n"
|
||
# elif tag_name=="br":
|
||
# article += "<br>\n"
|
||
elif tag_name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
|
||
article += '#' * int(tag_name[-1]) + ' '
|
||
article, number = parser_beautiful(chi, article, number, dircrea, bk)
|
||
article += '\n\n'
|
||
elif tag_name=="span":
|
||
datatex = None
|
||
classc = None
|
||
if 'data-tex' in chi.attrs.keys():
|
||
datatex = chi.attrs["data-tex"]
|
||
if 'class' in chi.attrs.keys():
|
||
classc = chi.attrs["class"]
|
||
if datatex and classc and 'ztext-math' in classc:
|
||
content = chi.attrs["data-tex"]
|
||
while len(content) > 0 and ' '==content[0]:
|
||
content = content[1:]
|
||
while len(content) > 0 and ' '==content[-1]:
|
||
content = content[:-1]
|
||
if len(content) > 0:
|
||
if article[-3-1:]=='<br>' or article[-1:]=='\n':
|
||
article += "\n$" + content + "$"
|
||
else:
|
||
article += "$" + content + "$"
|
||
else:
|
||
article, number = parser_beautiful(chi, article, number, dircrea, bk)
|
||
# article += nod.text
|
||
elif tag_name=="a":
|
||
linksite = None
|
||
if 'href' in chi.attrs.keys():
|
||
linksite = chi.attrs['href']
|
||
if linksite:
|
||
linksite = linksite.replace("//link.zhihu.com/?target=https%3A", "").replace("//link.zhihu.com/?target=http%3A", "")
|
||
if len(article) > 0 and article[-1]=='\n':
|
||
article += "["+chi.text+"]"+"("+linksite + ")"
|
||
else:
|
||
article += "\n\n["+chi.text+"]"+"("+linksite + ")"
|
||
elif tag_name=='b' or tag_name=='strong':
|
||
if len(cll) > 1:
|
||
article, number = parser_beautiful(chi, article, number, dircrea, True)
|
||
else:
|
||
txt = chi.text
|
||
while len(txt) > 0 and txt[-1] == " ":
|
||
txt = txt[:-1]
|
||
article += " **" + txt + "** "
|
||
elif tag_name=="figure":
|
||
noscript = chi.find_all('noscript')
|
||
if len(noscript) > 0:
|
||
chi.noscript.extract()
|
||
imgchunk = chi.find_all('img')
|
||
for i in range(len(imgchunk)):
|
||
imglink = None
|
||
if 'data-original' in imgchunk[i].attrs.keys():
|
||
imglink = imgchunk[i].attrs["data-original"]
|
||
|
||
if 'data-actualsrc' in imgchunk[i].attrs.keys():
|
||
imglink = imgchunk[i].attrs['data-actualsrc']
|
||
|
||
if imglink==None:
|
||
imglink = imgchunk[i].attrs["src"]
|
||
try:
|
||
response = requests.get(imglink, timeout=30)
|
||
except:
|
||
try:
|
||
response = requests.get(imglink, timeout=30)
|
||
except:
|
||
continue
|
||
if response.status_code==200:
|
||
with open(os.path.join(dircrea, str(number) + '.jpg'), 'wb') as obj:
|
||
obj.write(response.content)
|
||
|
||
files = {'image': open(os.path.join(dircrea, str(number) + '.jpg'), 'rb')}
|
||
response1 = requests.post('https://www.testingcloud.club/sapi/api/image_upload',
|
||
files=files,timeout=30)
|
||
if response1.status_code==200:
|
||
continue
|
||
# jsons = json.loads(response1.text)
|
||
# print(jsons)
|
||
# article += ''' <img src="https://www.testingcloud.club/sapi/api/image_download/%s" width="100%%"/> \n\n'''%jsons['url']
|
||
|
||
requests.put(imglink, timeout=30)
|
||
number += 1
|
||
crawlsleep(1)
|
||
elif tag_name=="div":
|
||
prenode = chi.find_all('code')
|
||
if len(prenode) > 0:
|
||
for i in prenode:
|
||
article += "\n\n```\n" + i.text + "\n```\n\n"
|
||
else:
|
||
article, number = parser_beautiful(chi, article, number, dircrea, bk)
|
||
article += "\n\n"
|
||
if bk:
|
||
article += "**"
|
||
return article, number
|
||
|
||
|
||
# Xpanx.com 专业网络爬虫程序定制，加微信 LiteMango（付费）
|
||
# 设置webdriver路径，替换为你的webdriver路径
|
||
abspath = os.path.abspath(__file__)
|
||
|
||
driverpath = os.path.join(abspath, 'msedgedriver\msedgedriver.exe')
|
||
service = Service(executable_path=driverpath)
|
||
edge_options = EdgeOptions()
|
||
|
||
#https://stackoverflow.com/questions/53039551/selenium-webdriver-modifying-navigator-webdriver-flag-to-prevent-selenium-detec
|
||
edge_options.add_experimental_option('excludeSwitches', ['enable-automation'])
|
||
edge_options.add_experimental_option('useAutomationExtension', False)
|
||
edge_options.add_argument('lang=zh-CN,zh,zh-TW,en-US,en')
|
||
edge_options.add_argument("disable-blink-features=AutomationControlled")#就是这一行告诉chrome去掉了webdriver痕迹
|
||
|
||
# 初始化webdriver
|
||
driver = webdriver.Edge(options=edge_options, service = service)
|
||
|
||
# 微信公众号文章的 URL
|
||
url = 'https://mp.weixin.qq.com/s?__biz=MzI4Njg5MDA5NA==&mid=2247483973&idx=2&sn=483265ffa9087ca956ec2d637119a5f8&chksm=ebd74344dca0ca5298b894fbb706c26ee942a423e858e27679f06df4b83899e1a97cc9d5eb97&scene=21###wechat_redirect'
|
||
|
||
# 打开页面
|
||
driver.get(url)
|
||
|
||
# 等待一定时间让页面加载完成
|
||
time.sleep(5)
|
||
|
||
# 获取页面的源代码
|
||
html = driver.page_source
|
||
|
||
# 解析 HTML
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
|
||
# 提取文章标题
|
||
title = soup.find('h1', {'class': 'rich_media_title'}).get_text(strip=True)
|
||
|
||
# 提取文章内容
|
||
content = soup.find('div', {'id': 'js_content'})
|
||
richtext = driver.find_element(By.TAG_NAME, "section")
|
||
article = ''
|
||
print(content)
|
||
inner = driver.execute_script("return arguments[0].innerHTML;", richtext)
|
||
innerHTML = BeautifulSoup(inner, "html.parser")
|
||
res,num = parser_beautiful(innerHTML,article,0,'d://',False)
|
||
# 将 HTML 转换为 Markdown
|
||
#markdown = md(str(content))
|
||
|
||
# 将 Markdown 写入文件
|
||
with open(f'{title}.md', 'w', encoding='utf-8') as f:
|
||
f.write(res)
|
||
|
||
# 关闭webdriver
|
||
driver.quit() |