Python爬虫实战(十)：爬取Linux公社资源站的所有电子资源

      #coding=utf-8
      import re
      import requests
      from tenacity import retry, stop_after_attempt
      @retry(stop=stop_after_attempt(3))
      def get_html(url):
      '''获取页面源代码'''
       headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'}
       page = requests.get(url,headers = headers)
       html = page.text
      return html
      def get_real_url(url,reg):
      '''获取真实的页面地址'''
       mainurl = 'http://linux.linuxidc.com/'
       html = get_html(url)
       items = re.findall(reg,html)
      for item in items:
       realurl = '{}{}'.format(mainurl,item)
      yield realurl
      def get_year_url():
      '''获取**年的页面地址'''
       url = 'http://linux.linuxidc.com/index.php'
       reg = re.compile(r'href="(.*?)">\d+年资料')
       year_urls = get_real_url(url,reg)
      for year_url in year_urls:
      yield year_url
      def get_month_url():
      '''获取**月的页面地址'''
       reg = re.compile(r'href="(.*?)">\d+月')
       year_urls = get_year_url()
      for year_url in year_urls:
       month_urls =  get_real_url(year_url,reg)
      for month_url in month_urls:
      yield month_url
      def get_day_url():
      '''获取**日的页面地址'''
       reg = re.compile(r'href="(.*?)">\d+日')
       month_urls = get_month_url()
      for month_url in month_urls:
       day_urls =  get_real_url(month_url,reg)
      for day_url in day_urls:
      yield day_url
      def get_books_urls(urls):
      '''获取资料名称及下载页'''
      for url in urls:
       reg = re.compile(r'href="(index.*?)">(.*?)</a></div></td><td width="100">')
       html = get_html(url)
       items = re.findall(reg,html)
      for item in items:
      yield item
      def get_other_url(url):
      '''获取其他链接的资料'''
       reg = re.compile(r'href="(index.*?)">.*?</a></div></td><td width="100">')
       all_urls = get_real_url(url,reg)
      for all_url in all_urls:
      yield all_url
      def print_book_url(book,book_url):
      '''打印可下载的书籍及链接'''
       url = 'http://linux.linuxidc.com/'
       book = book.lower()
       item = book_url
      if len(item) == 2:
       bookname = item[1].lower()
      if book in bookname:
      print ('\n'+ item[1])
      print ("资料下载链接:")
       dlurl = '{}{}'.format(url,item[0])
       reg = re.compile(r'href="(linuxconf/download.php.*?)">.*?</a></div></td><td width="100">')
       download_urls = get_real_url(dlurl,reg)
      for download_url in download_urls:
      print (download_url)
      def get_download_url(book):
       url_2011 = 'http://linux.linuxidc.com/index.php?folder=MjAxMcTq18rBzw=='
       all_urls = get_other_url(url_2011)
       books_urls = get_books_urls(all_urls)
      for book_url in books_urls:
       print_book_url (book,book_url)
       day_urls = get_day_url()
       books_urls = get_books_urls(day_urls)
      for book_url in books_urls:
       print_book_url (book,book_url)
      if __name__=='__main__':
       book = input ("请输入资料名称:")
       get_download_url(book)
  
 

说明：待添加。

文章来源: blog.csdn.net，作者：悦来客栈的老板，版权归原作者所有，如需转载，请联系作者。

原文链接：blog.csdn.net/qq523176585/article/details/78443714