Xpath实战一：解析爬取糗事百科段子

  

   

    

     

    
    

     

      #coding=utf-8
     
    
   

    

     

    
    

     

      import requests
     
    
   

    

     

    
    

     

      from lxml import etree
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def getHtml(url):  
     
    
   

    

     

    
    

     

       page = requests.get(url)  
     
    
   

    

     

    
    

     

       html = page.text  
     
    
   

    

     

    
    

     
 return html
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def getImg(html):
     
    
   

    

     

    
    

     

       texts = []
     
    
   

    

     

    
    

     

       html = etree.HTML(html)
     
    
   

    

     

    
    

     

       authors = html.xpath('//h2/text()')   #所有作者信息都在h2标签的文本里
     
    
   

    

     

    
    

     

       contents = html.xpath('//div[@class = "content"]//span[1]')  #span[1] 只选择第一个span标签的内容，有些第二个span标签内容是“查看全文”
     
    
   

    

     

    
    

     
 for content in contents:
     
    
   

    

     

    
    

     

       texts.append(content.xpath('string(.)'))  #过滤多个类似<br>标签，只选择文本
     
    
   

    

     

    
    

     

       votes = html.xpath('//*[@class = "stats-vote"]//i[@class = "number"]/text()')   #好笑数
     
    
   

    

     

    
    

     

       comments = html.xpath('//*[@class = "qiushi_comments"]//*[@class = "number"]/text()')  #回复数
     
    
   

    

     

    
    

     
 for x in zip(authors,texts,votes,comments):
     
    
   

    

     

    
    

     

       Enter = input()
     
    
   

    

     

    
    

     
 if Enter == 'Q' or Enter == 'q':
     
    
   

    

     

    
    

     
 return -1
     
    
   

    

     

    
    

     

       print("\n".join((s.strip() for s in x)))
     
    
   

    

     

    
    

     
 return 1
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      if __name__=='__main__':
     
    
   

    

     

    
    

     

       url = "https://www.qiushibaike.com/hot/"
     
    
   

    

     

    
    

     

       html,i = getHtml(url),2
     
    
   

    

     

    
    

     

       print("开始读取数据，请按回车键...按Q键退出\n")
     
    
   

    

     

    
    

     
 while (getImg(html) == 1):
     
    
   

    

     

    
    

     

       print("\n第{}页\n".format(i))
     
    
   

    

     

    
    

     

       url = ("https://www.qiushibaike.com/hot/page/%s/" % str(i))
     
    
   

    

     

    
    

     

       html,i = getHtml(url),i+1
     
    
  
 

文章来源: blog.csdn.net，作者：悦来客栈的老板，版权归原作者所有，如需转载，请联系作者。

原文链接：blog.csdn.net/qq523176585/article/details/78523853