Python爬虫实战(九)：爬取动态网页

  

   

    

     

    
    

     

      #coding=utf-8
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      import re
     
    
   

    

     

    
    

     

      import json
     
    
   

    

     

    
    

     

      import requests
     
    
   

    

     

    
    

     

      from prettytable import PrettyTable
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def getHtml(url):
     
    
   

    

     

    
    

     

       data = {
     
    
   

    

     

    
    

     
 'page':1,
     
    
   

    

     

    
    

     
 'num':40,
     
    
   

    

     

    
    

     
 'sort':'symbol',
     
    
   

    

     

    
    

     
 'asc':1,
     
    
   

    

     

    
    

     
 'node':'cyb',
     
    
   

    

     

    
    

     
 'symbol':'',
     
    
   

    

     

    
    

     
 '_s_r_a':'page'}
     
    
   

    

     

    
    

     

       headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'}
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 try: 
     
    
   

    

     

    
    

     

       page = requests.post(url,data = data,headers = headers)  
     
    
   

    

     

    
    

     

       page.encoding = 'gbk'  
     
    
   

    

     

    
    

     

       html = page.text
     
    
   

    

     

    
    

     
 return html
     
    
   

    

     

    
    

     
 except:   
     
    
   

    

     

    
    

     
 return ""
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def getdata(html):
     
    
   

    

     

    
    

     

       data = html.replace(':','":')
     
    
   

    

     

    
    

     

       data = data.replace(',',',"')
     
    
   

    

     

    
    

     

       data = data.replace('{','{"')
     
    
   

    

     

    
    

     

       data = data.replace('"{','{')
     
    
   

    

     

    
    

     

       data = re.sub('\d+":\d+":\d+','',data)
     
    
   

    

     

    
    

     

       data = json.loads(data)
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

       row = PrettyTable()
     
    
   

    

     

    
    

     

       row.field_names = ["代码", "名称", "最新价", "涨跌额","涨跌幅","买入","卖出","昨收","今开","最高"
     
    
   

    

     

    
    

     

       ,"最低","成交量/手","成交额/万"]
     
    
   

    

     

    
    

     
 for item in data:
     
    
   

    

     

    
    

     

       row.add_row((item['symbol'],item['name'],item['trade'],item['pricechange'],item['changepercent']
     
    
   

    

     

    
    

     

       ,item['buy'],item['sell'],item['settlement'],item['open'],item['high']
     
    
   

    

     

    
    

     

       ,item['low'],item['volume'],item['amount']))
     
    
   

    

     

    
    

     

       print(row)
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      if __name__=='__main__':
     
    
   

    

     

    
    

     

       url = 'http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?'
     
    
   

    

     

    
    

     

       html = getHtml(url)
     
    
   

    

     

    
    

     

       getdata(html)
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      #coding=utf-8
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      import re
     
    
   

    

     

    
    

     

      import json
     
    
   

    

     

    
    

     

      import requests
     
    
   

    

     

    
    

     

      from prettytable import PrettyTable
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      def getHtml(url):
     
    
   

    

     

    
    

     

           data = {
     
    
   

    

     

    
    

     

              'page.pageNo':2,
     
    
   

    

     

    
    

     

              'tempPageSize':40,
     
    
   

    

     

    
    

     

              }
     
    
   

    

     

    
    

     

           headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'} 
     
    
   

    

     

    
    

     

           page = requests.post(url,headers = headers,data = data)  
     
    
   

    

     

    
    

     

           html = page.text
     
    
   

    

     

    
    

     

           print (html)
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      if __name__=='__main__':
     
    
   

    

     

    
    

     

          url = 'http://datacenter.mep.gov.cn:8099/ths-report/report!list.action?xmlname=1465594312346'
     
    
   

    

     

    
    

     

          getHtml(url)
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
  
 

文章来源: blog.csdn.net，作者：悦来客栈的老板，版权归原作者所有，如需转载，请联系作者。

原文链接：blog.csdn.net/qq523176585/article/details/78388934