21 lines
797 B
Plaintext
21 lines
797 B
Plaintext
|
|
|||
|
|
def start3(self):
|
|||
|
|
num = 0
|
|||
|
|
articles = [None]
|
|||
|
|
while len(articles) > 0:
|
|||
|
|
num += 1
|
|||
|
|
url = u'https://xuesong.blog.csdn.net/category_8454447_' + str(num) + '.html'
|
|||
|
|
print(url)
|
|||
|
|
# url = u'https://' + self.username + u'.blog.csdn.net/article/list/' + str(num)
|
|||
|
|
scraper = cloudscraper.create_scraper() # returns a CloudScraper instance
|
|||
|
|
response = scraper.get(url,headers=self.headers)
|
|||
|
|
|
|||
|
|
# response = self.s.get(url=url, headers=self.headers)
|
|||
|
|
html = response.text
|
|||
|
|
soup = BeautifulSoup(html, "html.parser")
|
|||
|
|
articles = soup.find_all('div', attrs={"class":"column_article_title"})
|
|||
|
|
for article in articles:
|
|||
|
|
article_title = article.a.text.strip().replace(' ','：')
|
|||
|
|
article_href = article.a['href']
|
|||
|
|
self.TaskQueue.append((article_title, article_href))
|
|||
|
|
|