Python爬虫实战(十)：爬取Linux公社资源站的所有电子资源

  

   

    

     

    
    

     

      #coding=utf-8
     
    
   

    

     

    
    

     

      import re
     
    
   

    

     

    
    

     

      import requests
     
    
   

    

     

    
    

     

      from tenacity import retry, stop_after_attempt
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      @retry(stop=stop_after_attempt(3))
     
    
   

    

     

    
    

     

      def get_html(url):
     
    
   

    

     

    
    

     
 '''获取页面源代码'''
     
    
   

    

     

    
    

     

       headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'}
     
    
   

    

     

    
    

     

       page = requests.get(url,headers = headers)
     
    
   

    

     

    
    

     

       html = page.text
     
    
   

    

     

    
    

     
 return html
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def get_real_url(url,reg):
     
    
   

    

     

    
    

     
 '''获取真实的页面地址'''
     
    
   

    

     

    
    

     

       mainurl = 'http://linux.linuxidc.com/'
     
    
   

    

     

    
    

     

       html = get_html(url)
     
    
   

    

     

    
    

     

       items = re.findall(reg,html)
     
    
   

    

     

    
    

     
 for item in items:
     
    
   

    

     

    
    

     

       realurl = '{}{}'.format(mainurl,item)
     
    
   

    

     

    
    

     
 yield realurl
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def get_year_url():
     
    
   

    

     

    
    

     
 '''获取**年的页面地址'''
     
    
   

    

     

    
    

     

       url = 'http://linux.linuxidc.com/index.php'
     
    
   

    

     

    
    

     

       reg = re.compile(r'href="(.*?)">\d+年资料')
     
    
   

    

     

    
    

     

       year_urls = get_real_url(url,reg)
     
    
   

    

     

    
    

     
 for year_url in year_urls:
     
    
   

    

     

    
    

     
 yield year_url
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def get_month_url():
     
    
   

    

     

    
    

     
 '''获取**月的页面地址'''
     
    
   

    

     

    
    

     

       reg = re.compile(r'href="(.*?)">\d+月')
     
    
   

    

     

    
    

     

       year_urls = get_year_url()
     
    
   

    

     

    
    

     
 for year_url in year_urls:
     
    
   

    

     

    
    

     

       month_urls =  get_real_url(year_url,reg)
     
    
   

    

     

    
    

     
 for month_url in month_urls:
     
    
   

    

     

    
    

     
 yield month_url
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def get_day_url():
     
    
   

    

     

    
    

     
 '''获取**日的页面地址'''
     
    
   

    

     

    
    

     

       reg = re.compile(r'href="(.*?)">\d+日')
     
    
   

    

     

    
    

     

       month_urls = get_month_url()
     
    
   

    

     

    
    

     
 for month_url in month_urls:
     
    
   

    

     

    
    

     

       day_urls =  get_real_url(month_url,reg)
     
    
   

    

     

    
    

     
 for day_url in day_urls:
     
    
   

    

     

    
    

     
 yield day_url
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def get_books_urls(urls):
     
    
   

    

     

    
    

     
 '''获取资料名称及下载页'''
     
    
   

    

     

    
    

     
 for url in urls:
     
    
   

    

     

    
    

     

       reg = re.compile(r'href="(index.*?)">(.*?)</a></div></td><td width="100">')
     
    
   

    

     

    
    

     

       html = get_html(url)
     
    
   

    

     

    
    

     

       items = re.findall(reg,html)
     
    
   

    

     

    
    

     
 for item in items:
     
    
   

    

     

    
    

     
 yield item
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def get_other_url(url):
     
    
   

    

     

    
    

     
 '''获取其他链接的资料'''
     
    
   

    

     

    
    

     

       reg = re.compile(r'href="(index.*?)">.*?</a></div></td><td width="100">')
     
    
   

    

     

    
    

     

       all_urls = get_real_url(url,reg)
     
    
   

    

     

    
    

     
 for all_url in all_urls:
     
    
   

    

     

    
    

     
 yield all_url
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def print_book_url(book,book_url):
     
    
   

    

     

    
    

     
 '''打印可下载的书籍及链接'''
     
    
   

    

     

    
    

     

       url = 'http://linux.linuxidc.com/'
     
    
   

    

     

    
    

     

       book = book.lower()
     
    
   

    

     

    
    

     

       item = book_url
     
    
   

    

     

    
    

     
 if len(item) == 2:
     
    
   

    

     

    
    

     

       bookname = item[1].lower()
     
    
   

    

     

    
    

     
 if book in bookname:
     
    
   

    

     

    
    

     
 print ('\n'+ item[1])
     
    
   

    

     

    
    

     
 print ("资料下载链接:")
     
    
   

    

     

    
    

     

       dlurl = '{}{}'.format(url,item[0])
     
    
   

    

     

    
    

     

       reg = re.compile(r'href="(linuxconf/download.php.*?)">.*?</a></div></td><td width="100">')
     
    
   

    

     

    
    

     

       download_urls = get_real_url(dlurl,reg)
     
    
   

    

     

    
    

     
 for download_url in download_urls:
     
    
   

    

     

    
    

     
 print (download_url)
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def get_download_url(book):
     
    
   

    

     

    
    

     

       url_2011 = 'http://linux.linuxidc.com/index.php?folder=MjAxMcTq18rBzw=='
     
    
   

    

     

    
    

     

       all_urls = get_other_url(url_2011)
     
    
   

    

     

    
    

     

       books_urls = get_books_urls(all_urls)
     
    
   

    

     

    
    

     
 for book_url in books_urls:
     
    
   

    

     

    
    

     

       print_book_url (book,book_url)
     
    
   

    

     

    
    

     

       day_urls = get_day_url()
     
    
   

    

     

    
    

     

       books_urls = get_books_urls(day_urls)
     
    
   

    

     

    
    

     
 for book_url in books_urls:
     
    
   

    

     

    
    

     

       print_book_url (book,book_url)
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      if __name__=='__main__':
     
    
   

    

     

    
    

     

       book = input ("请输入资料名称:")
     
    
   

    

     

    
    

     

       get_download_url(book)
     
    
  
 

说明：待添加。

文章来源: blog.csdn.net，作者：悦来客栈的老板，版权归原作者所有，如需转载，请联系作者。

原文链接：blog.csdn.net/qq523176585/article/details/78443714