爬虫案例

举报
建帅小伙儿 发表于 2022/09/25 03:21:12 2022/09/25
【摘要】 一、壁纸网站 # coding=utf-8""" 作者：gaojs 功能： 新增功能： 日期：2022/3/25 19:35"""import os.path import requestsimport parsel def get_address(): """ 获取url地址 :r...

一、壁纸网站


   
    
    
  1. 
     
    
      
    
     
    
     
    
      
    
       # coding=utf-8
      
    
     
    2. 
    
  2. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    3. 
    
  3. 
     
    
      
    
     
    
     
    
      
    
        作者：gaojs
      
    
     
    4. 
    
  4. 
     
    
      
    
     
    
     
    
      
    
        功能：
      
    
     
    5. 
    
  5. 
     
    
      
    
     
    
     
    
      
    
        新增功能：
      
    
     
    6. 
    
  6. 
     
    
      
    
     
    
     
    
      
    
        日期：2022/3/25 19:35
      
    
     
    7. 
    
  7. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    8. 
    
  8. 
     
    
      
    
     
    
     
    
      
    
       import os.path
      
    
     
    9. 
    
  9. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    10. 
    
  10. 
     
    
      
    
     
    
     
    
      
    
       import requests
      
    
     
    11. 
    
  11. 
     
    
      
    
     
    
     
    
      
    
       import parsel
      
    
     
    12. 
    
  12. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    13. 
    
  13. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    14. 
    
  14. 
     
    
      
    
     
    
     
    
      
    
       def get_address():
      
    
     
    15. 
    
  15. 
     
    
      
    
     
    
     
    
      
        """
      
    
     
    16. 
    
  16. 
     
    
      
    
     
    
     
    
      
    
        获取url地址
      
    
     
    17. 
    
  17. 
     
    
      
    
     
    
     
    
      
    
        :return:
      
    
     
    18. 
    
  18. 
     
    
      
    
     
    
     
    
      
    
        """
      
    
     
    19. 
    
  19. 
     
    
      
    
     
    
     
    
      
    
           dirname = 'photo/'
      
    
     
    20. 
    
  20. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    21. 
    
  21. 
     
    
      
    
     
    
     
    
      
        if not os.path.exists(dirname):
      
    
     
    22. 
    
  22. 
     
    
      
    
     
    
     
    
      
    
               os.mkdir(dirname)
      
    
     
    23. 
    
  23. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    24. 
    
  24. 
     
    
      
    
     
    
     
    
      
        for page in range(2, 11):
      
    
     
    25. 
    
  25. 
     
    
      
    
     
    
     
    
      
            print(f'=====================正在爬取第{page}页内容========================')
      
    
     
    26. 
    
  26. 
     
    
      
    
     
    
     
    
      
    
               url = f'http://www.netbian.com/1920x1080/index_{page}.htm'
      
    
     
    27. 
    
  27. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    28. 
    
  28. 
     
    
      
    
     
    
     
    
      
            # url = 'http://www.netbian.com/1920x1080/'
      
    
     
    29. 
    
  29. 
     
    
      
    
     
    
     
    
      
    
               headers = {
      
    
     
    30. 
    
  30. 
     
    
      
    
     
    
     
    
      
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46'
      
    
     
    31. 
    
  31. 
     
    
      
    
     
    
     
    
      
    
               }
      
    
     
    32. 
    
  32. 
     
    
      
    
     
    
     
    
      
    
               res = requests.get(url, headers=headers)
      
    
     
    33. 
    
  33. 
     
    
      
    
     
    
     
    
      
    
               res.encoding = res.apparent_encoding
      
    
     
    34. 
    
  34. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    35. 
    
  35. 
     
    
      
    
     
    
     
    
      
    
               selector = parsel.Selector(res.text)
      
    
     
    36. 
    
  36. 
     
    
      
    
     
    
     
    
      
    
               href = selector.css('.list li a::attr(href)').getall()
      
    
     
    37. 
    
  37. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    38. 
    
  38. 
     
    
      
    
     
    
     
    
      
            # <img src="http://img.netbian.com/file/2022/0326/small003835uYAUe1648226315.jpg" alt="绿色草地 美女刘亦菲2022年4月日历桌面壁纸护眼">
      
    
     
    39. 
    
  39. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    40. 
    
  40. 
     
    
      
    
     
    
     
    
      
    
               url_lis = selector.css('.list li')
      
    
     
    41. 
    
  41. 
     
    
      
    
     
    
     
    
      
            for lis in url_lis:
      
    
     
    42. 
    
  42. 
     
    
      
    
     
    
     
    
      
    
                   title = lis.css('b::text').get()
      
    
     
    43. 
    
  43. 
     
    
      
    
     
    
     
    
      
                # 取出广告页面
      
    
     
    44. 
    
  44. 
     
    
      
    
     
    
     
    
      
                if title:
      
    
     
    45. 
    
  45. 
     
    
      
    
     
    
     
    
      
    
                       list_url = 'http://www.netbian.com' + lis.css('a::attr(href)').get()
      
    
     
    46. 
    
  46. 
     
    
      
    
     
    
     
    
      
                    # print(list_url)
      
    
     
    47. 
    
  47. 
     
    
      
    
     
    
     
    
      
    
                       res1 = requests.get(list_url, headers=headers)
      
    
     
    48. 
    
  48. 
     
    
      
    
     
    
     
    
      
                    # print(res1.text)
      
    
     
    49. 
    
  49. 
     
    
      
    
     
    
     
    
      
    
                       selector1 = parsel.Selector(res1.text)
      
    
     
    50. 
    
  50. 
     
    
      
    
     
    
     
    
      
    
                       img_url = selector1.css('.pic img::attr(src)').get()
      
    
     
    51. 
    
  51. 
     
    
      
    
     
    
     
    
      
                    # print(img_url)
      
    
     
    52. 
    
  52. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    53. 
    
  53. 
     
    
      
    
     
    
     
    
      
                    # 保存图片
      
    
     
    54. 
    
  54. 
     
    
      
    
     
    
     
    
      
    
                       img_content = requests.get(url=img_url).content
      
    
     
    55. 
    
  55. 
     
    
      
    
     
    
     
    
      
                    with open('photo/' + title + '.jpg', 'wb') as f:
      
    
     
    56. 
    
  56. 
     
    
      
    
     
    
     
    
      
    
                           f.write(img_content)
      
    
     
    57. 
    
  57. 
     
    
      
    
     
    
     
    
      
                        print(title, img_url)
      
    
     
    58. 
    
  58. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    59. 
    
  59. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    60. 
    
  60. 
     
    
      
    
     
    
     
    
      
    
       get_address()
      
    
     
    61.

二、彼岸壁纸


   
    
    
  1. 
     
    
      
    
     
    
     
    
      
    
       # coding=utf-8
      
    
     
    2. 
    
  2. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    3. 
    
  3. 
     
    
      
    
     
    
     
    
      
    
        作者：gaojs
      
    
     
    4. 
    
  4. 
     
    
      
    
     
    
     
    
      
    
        功能：
      
    
     
    5. 
    
  5. 
     
    
      
    
     
    
     
    
      
    
        新增功能：
      
    
     
    6. 
    
  6. 
     
    
      
    
     
    
     
    
      
    
        日期：2022/4/2 14:59
      
    
     
    7. 
    
  7. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    8. 
    
  8. 
     
    
      
    
     
    
     
    
      
    
       import os.path
      
    
     
    9. 
    
  9. 
     
    
      
    
     
    
     
    
      
    
       import re
      
    
     
    10. 
    
  10. 
     
    
      
    
     
    
     
    
      
    
       import requests
      
    
     
    11. 
    
  11. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    12. 
    
  12. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    13. 
    
  13. 
     
    
      
    
     
    
     
    
      
    
       if not os.path.exists('photo/'):
      
    
     
    14. 
    
  14. 
     
    
      
    
     
    
     
    
      
    
           os.mkdir('photo/')
      
    
     
    15. 
    
  15. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    16. 
    
  16. 
     
    
      
    
     
    
     
    
      
    
       url = 'http://www.netbian.com'
      
    
     
    17. 
    
  17. 
     
    
      
    
     
    
     
    
      
    
       # http://www.netbian.com/index_2.htm
      
    
     
    18. 
    
  18. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    19. 
    
  19. 
     
    
      
    
     
    
     
    
      
    
       # http://www.netbian.com/desk/26344-1920x1080.htm
      
    
     
    20. 
    
  20. 
     
    
      
    
     
    
     
    
      
    
       # http://www.netbian.com/desk/26345-1920x1080.htm
      
    
     
    21. 
    
  21. 
     
    
      
    
     
    
     
    
      
    
       headers = {
      
    
     
    22. 
    
  22. 
     
    
      
    
     
    
     
    
      
        'Host': 'www.netbian.com',
      
    
     
    23. 
    
  23. 
     
    
      
    
     
    
     
    
      
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36',
      
    
     
    24. 
    
  24. 
     
    
      
    
     
    
     
    
      
        'Upgrade-Insecure-Requests': '1',
      
    
     
    25. 
    
  25. 
     
    
      
    
     
    
     
    
      
        'Cookie': '__yjs_duid=1_4535c561a20964f1ade88776981a0f411648389371877; Hm_lvt_0f461eb489c245a31c209d36e41fcc0f=1648389374,1648986956; Hm_lpvt_0f461eb489c245a31c209d36e41fcc0f=1648986956'
      
    
     
    26. 
    
  26. 
     
    
      
    
     
    
     
    
      
    
       }
      
    
     
    27. 
    
  27. 
     
    
      
    
     
    
     
    
      
    
       rsp = requests.get(url, headers=headers)
      
    
     
    28. 
    
  28. 
     
    
      
    
     
    
     
    
      
    
       rsp.encoding = rsp.apparent_encoding
      
    
     
    29. 
    
  29. 
     
    
      
    
     
    
     
    
      
    
       # print(rsp.text)
      
    
     
    30. 
    
  30. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    31. 
    
  31. 
     
    
      
    
     
    
     
    
      
    
       # <img src="http://img.netbian.com/file/2022/0402/small004425v1bwe1648831465.jpg" alt="lol英雄联盟九尾妖狐 命运之子 阿狸壁纸"/>
      
    
     
    32. 
    
  32. 
     
    
      
    
     
    
     
    
      
    
       # <a href="(.*?)"title="(.*?)" target="_blank"><img src=".*?" alt=".*?" />
      
    
     
    33. 
    
  33. 
     
    
      
    
     
    
     
    
      
    
       url_list = re.findall('<a href="(.*?)"title="(.*?)" target="_blank"><img src=".*?" alt=".*?" />', rsp.text)
      
    
     
    34. 
    
  34. 
     
    
      
    
     
    
     
    
      
    
       # print(url_list)
      
    
     
    35. 
    
  35. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    36. 
    
  36. 
     
    
      
    
     
    
     
    
      
    
       for index in url_list:
      
    
     
    37. 
    
  37. 
     
    
      
    
     
    
     
    
      
    
           url_lis = index[0]
      
    
     
    38. 
    
  38. 
     
    
      
    
     
    
     
    
      
    
           title = index[1]
      
    
     
    39. 
    
  39. 
     
    
      
    
     
    
     
    
      
    
           new_url = url + url_lis
      
    
     
    40. 
    
  40. 
     
    
      
    
     
    
     
    
      
        # print(new_url)
      
    
     
    41. 
    
  41. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    42. 
    
  42. 
     
    
      
    
     
    
     
    
      
    
           rsp1 = requests.get(new_url)
      
    
     
    43. 
    
  43. 
     
    
      
    
     
    
     
    
      
    
           rsp1.encoding = rsp1.apparent_encoding
      
    
     
    44. 
    
  44. 
     
    
      
    
     
    
     
    
      
    
           img_list = re.findall('<a href=".*?" target="_blank"><img src="(.*?)" alt="(.*?)" title=".*?"></a>', rsp1.text)
      
    
     
    45. 
    
  45. 
     
    
      
    
     
    
     
    
      
        # print(img_list)
      
    
     
    46. 
    
  46. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    47. 
    
  47. 
     
    
      
    
     
    
     
    
      
        for img in img_list:
      
    
     
    48. 
    
  48. 
     
    
      
    
     
    
     
    
      
    
               img_url = img[0]
      
    
     
    49. 
    
  49. 
     
    
      
    
     
    
     
    
      
    
               img_title = img[1]
      
    
     
    50. 
    
  50. 
     
    
      
    
     
    
     
    
      
    
               content_data = requests.get(img_url).content
      
    
     
    51. 
    
  51. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    52. 
    
  52. 
     
    
      
    
     
    
     
    
      
            with open('photo/' + img_title + '.jpg', 'wb') as f:
      
    
     
    53. 
    
  53. 
     
    
      
    
     
    
     
    
      
    
                   f.write(content_data)
      
    
     
    54. 
    
  54. 
     
    
      
    
     
    
     
    
      
                print(f'***************正在爬取{title}中****************')
      
    
     
    55.

三、某手视频


   
    
    
  1. 
     
    
      
    
     
    
     
    
      
    
       # coding=utf-8
      
    
     
    2. 
    
  2. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    3. 
    
  3. 
     
    
      
    
     
    
     
    
      
    
        作者：gaojs
      
    
     
    4. 
    
  4. 
     
    
      
    
     
    
     
    
      
    
        功能：
      
    
     
    5. 
    
  5. 
     
    
      
    
     
    
     
    
      
    
        新增功能：
      
    
     
    6. 
    
  6. 
     
    
      
    
     
    
     
    
      
    
        日期：2022/4/15 20:13
      
    
     
    7. 
    
  7. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    8. 
    
  8. 
     
    
      
    
     
    
     
    
      
    
       import json
      
    
     
    9. 
    
  9. 
     
    
      
    
     
    
     
    
      
    
       import os.path
      
    
     
    10. 
    
  10. 
     
    
      
    
     
    
     
    
      
    
       import pprint
      
    
     
    11. 
    
  11. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    12. 
    
  12. 
     
    
      
    
     
    
     
    
      
    
       import requests
      
    
     
    13. 
    
  13. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    14. 
    
  14. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    15. 
    
  15. 
     
    
      
    
     
    
     
    
      
    
       def get_page(pcursor):
      
    
     
    16. 
    
  16. 
     
    
      
    
     
    
     
    
      
    
           path = 'video/'
      
    
     
    17. 
    
  17. 
     
    
      
    
     
    
     
    
      
        if not os.path.exists(path):
      
    
     
    18. 
    
  18. 
     
    
      
    
     
    
     
    
      
    
               os.mkdir(path)
      
    
     
    19. 
    
  19. 
     
    
      
    
     
    
     
    
      
        # 爬取对象'https://www.kuaishou.com/profile/3xhv7zhkfr3rqag'
      
    
     
    20. 
    
  20. 
     
    
      
    
     
    
     
    
      
        """
      
    
     
    21. 
    
  21. 
     
    
      
    
     
    
     
    
      
    
        ctrl+r 批量替换
      
    
     
    22. 
    
  22. 
     
    
      
    
     
    
     
    
      
    
        https://www.kuaishou.com/short-video/3xw5fmcf9jdap29?authorId=3xhv7zhkfr3rqag&streamSource=profile&area=profilexxnull
      
    
     
    23. 
    
  23. 
     
    
      
    
     
    
     
    
      
    
        https://www.kuaishou.com/short-video/3xf98wc5q2cuxtq?authorId=3xhv7zhkfr3rqag&streamSource=profile&area=profilexxnull
      
    
     
    24. 
    
  24. 
     
    
      
    
     
    
     
    
      
    
        """
      
    
     
    25. 
    
  25. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    26. 
    
  26. 
     
    
      
    
     
    
     
    
      
    
           url = 'https://www.kuaishou.com/graphql'
      
    
     
    27. 
    
  27. 
     
    
      
    
     
    
     
    
      
    
           headers = {
      
    
     
    28. 
    
  28. 
     
    
      
    
     
    
     
    
      
            'content-type': 'application/json',
      
    
     
    29. 
    
  29. 
     
    
      
    
     
    
     
    
      
            'Cookie': 'kpf=PC_WEB; kpn=KUAISHOU_VISION; clientid=3; did=web_72314bf978cb158dd7034b2370d2ae70',
      
    
     
    30. 
    
  30. 
     
    
      
    
     
    
     
    
      
            'Host': 'www.kuaishou.com',
      
    
     
    31. 
    
  31. 
     
    
      
    
     
    
     
    
      
            'Origin': 'https://www.kuaishou.com',
      
    
     
    32. 
    
  32. 
     
    
      
    
     
    
     
    
      
            'Referer': 'https://www.kuaishou.com/short-video/3x6v3xmcjsd5cki?authorId=3xhv7zhkfr3rqag&streamSource=profile&area=profilexxnull',
      
    
     
    33. 
    
  33. 
     
    
      
    
     
    
     
    
      
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36',
      
    
     
    34. 
    
  34. 
     
    
      
    
     
    
     
    
      
    
           }
      
    
     
    35. 
    
  35. 
     
    
      
    
     
    
     
    
      
    
           data = {
      
    
     
    36. 
    
  36. 
     
    
      
    
     
    
     
    
      
            "operationName": "visionProfilePhotoList",
      
    
     
    37. 
    
  37. 
     
    
      
    
     
    
     
    
      
            "query": "query visionProfilePhotoList($pcursor: String, $userId: String, $page: String, $webPageArea: String) {\n visionProfilePhotoList(pcursor: $pcursor, userId: $userId, page: $page, webPageArea: $webPageArea) {\n result\n llsid\n webPageArea\n feeds {\n type\n author {\n id\n name\n following\n headerUrl\n headerUrls {\n cdn\n url\n __typename\n }\n __typename\n }\n tags {\n type\n name\n __typename\n }\n photo {\n id\n duration\n caption\n likeCount\n realLikeCount\n coverUrl\n coverUrls {\n cdn\n url\n __typename\n }\n photoUrls {\n cdn\n url\n __typename\n }\n photoUrl\n liked\n timestamp\n expTag\n animatedCoverUrl\n stereoType\n videoRatio\n profileUserTopPhoto\n __typename\n }\n canAddComment\n currentPcursor\n llsid\n status\n __typename\n }\n hostName\n pcursor\n __typename\n }\n}\n",
      
    
     
    38. 
    
  38. 
     
    
      
    
     
    
     
    
      
            "variables": {"userId": "3xhv7zhkfr3rqag", "pcursor": pcursor, "page": "detail", "webPageArea": "profilexxnull"}
      
    
     
    39. 
    
  39. 
     
    
      
    
     
    
     
    
      
    
           }
      
    
     
    40. 
    
  40. 
     
    
      
    
     
    
     
    
      
    
           rsp = requests.post(url=url, json=data, headers=headers)
      
    
     
    41. 
    
  41. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    42. 
    
  42. 
     
    
      
    
     
    
     
    
      
        # 第一种方式转成json
      
    
     
    43. 
    
  43. 
     
    
      
    
     
    
     
    
      
        # json_data = json.loads(rsp.text)
      
    
     
    44. 
    
  44. 
     
    
      
    
     
    
     
    
      
        # 或者
      
    
     
    45. 
    
  45. 
     
    
      
    
     
    
     
    
      
    
           json_data = rsp.json()
      
    
     
    46. 
    
  46. 
     
    
      
    
     
    
     
    
      
        # print(json_data, type(json_data))
      
    
     
    47. 
    
  47. 
     
    
      
    
     
    
     
    
      
    
           url_list = json_data['data']['visionProfilePhotoList']['feeds']
      
    
     
    48. 
    
  48. 
     
    
      
    
     
    
     
    
      
    
           pcursor = json_data['data']['visionProfilePhotoList']['pcursor']
      
    
     
    49. 
    
  49. 
     
    
      
    
     
    
     
    
      
        # print(url_list)
      
    
     
    50. 
    
  50. 
     
    
      
    
     
    
     
    
      
        # pprint.pprint(url_list)
      
    
     
    51. 
    
  51. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    52. 
    
  52. 
     
    
      
    
     
    
     
    
      
        for key in url_list:
      
    
     
    53. 
    
  53. 
     
    
      
    
     
    
     
    
      
            # 视屏标题
      
    
     
    54. 
    
  54. 
     
    
      
    
     
    
     
    
      
    
               title = key['photo']['caption']
      
    
     
    55. 
    
  55. 
     
    
      
    
     
    
     
    
      
            # print(title)
      
    
     
    56. 
    
  56. 
     
    
      
    
     
    
     
    
      
            # 视频url
      
    
     
    57. 
    
  57. 
     
    
      
    
     
    
     
    
      
    
               new_url = key['photo']['photoUrl']
      
    
     
    58. 
    
  58. 
     
    
      
    
     
    
     
    
      
            # print(title, new_url)
      
    
     
    59. 
    
  59. 
     
    
      
    
     
    
     
    
      
            # 发送请求
      
    
     
    60. 
    
  60. 
     
    
      
    
     
    
     
    
      
    
               content_data = requests.get(url=new_url).content
      
    
     
    61. 
    
  61. 
     
    
      
    
     
    
     
    
      
            # 保存目录
      
    
     
    62. 
    
  62. 
     
    
      
    
     
    
     
    
      
            with open(f'video/{title}.mp4', mode='wb') as f:
      
    
     
    63. 
    
  63. 
     
    
      
    
     
    
     
    
      
    
                   f.write(content_data)
      
    
     
    64. 
    
  64. 
     
    
      
    
     
    
     
    
      
                print(f'=======================正在下载标题为 {title} 的快手短视频==========================')
      
    
     
    65. 
    
  65. 
     
    
      
    
     
    
     
    
      
        if pcursor != "no_more":
      
    
     
    66. 
    
  66. 
     
    
      
    
     
    
     
    
      
    
               get_page(pcursor)
      
    
     
    67. 
    
  67. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    68. 
    
  68. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    69. 
    
  69. 
     
    
      
    
     
    
     
    
      
    
       get_page("")
      
    
     
    70.

四、拉钩数据


   
    
    
  1. 
     
    
      
    
     
    
     
    
      
    
       # coding=utf-8
      
    
     
    2. 
    
  2. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    3. 
    
  3. 
     
    
      
    
     
    
     
    
      
    
        作者：gaojs
      
    
     
    4. 
    
  4. 
     
    
      
    
     
    
     
    
      
    
        功能：
      
    
     
    5. 
    
  5. 
     
    
      
    
     
    
     
    
      
    
        新增功能：
      
    
     
    6. 
    
  6. 
     
    
      
    
     
    
     
    
      
    
        日期：2022/4/3 17:58
      
    
     
    7. 
    
  7. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    8. 
    
  8. 
     
    
      
    
     
    
     
    
      
    
       import csv
      
    
     
    9. 
    
  9. 
     
    
      
    
     
    
     
    
      
    
       import json
      
    
     
    10. 
    
  10. 
     
    
      
    
     
    
     
    
      
    
       import os.path
      
    
     
    11. 
    
  11. 
     
    
      
    
     
    
     
    
      
    
       import pprint
      
    
     
    12. 
    
  12. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    13. 
    
  13. 
     
    
      
    
     
    
     
    
      
    
       import requests
      
    
     
    14. 
    
  14. 
     
    
      
    
     
    
     
    
      
    
       import re
      
    
     
    15. 
    
  15. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    16. 
    
  16. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    17. 
    
  17. 
     
    
      
    
     
    
     
    
      
    
       if not os.path.exists('info/'):
      
    
     
    18. 
    
  18. 
     
    
      
    
     
    
     
    
      
    
           os.makedirs('info/')
      
    
     
    19. 
    
  19. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    20. 
    
  20. 
     
    
      
    
     
    
     
    
      
    
       f = open('info/招聘.csv', encoding='utf-8', mode='a', newline='')
      
    
     
    21. 
    
  21. 
     
    
      
    
     
    
     
    
      
    
       csv_writer = csv.DictWriter(f, fieldnames=[
      
    
     
    22. 
    
  22. 
     
    
      
    
     
    
     
    
      
        '职位名字',
      
    
     
    23. 
    
  23. 
     
    
      
    
     
    
     
    
      
        '公司名字',
      
    
     
    24. 
    
  24. 
     
    
      
    
     
    
     
    
      
        '工作城市',
      
    
     
    25. 
    
  25. 
     
    
      
    
     
    
     
    
      
        '学历要求',
      
    
     
    26. 
    
  26. 
     
    
      
    
     
    
     
    
      
        '经验要求',
      
    
     
    27. 
    
  27. 
     
    
      
    
     
    
     
    
      
        '薪资要求',
      
    
     
    28. 
    
  28. 
     
    
      
    
     
    
     
    
      
        '公司地址',
      
    
     
    29. 
    
  29. 
     
    
      
    
     
    
     
    
      
        '详情页'
      
    
     
    30. 
    
  30. 
     
    
      
    
     
    
     
    
      
    
       ])
      
    
     
    31. 
    
  31. 
     
    
      
    
     
    
     
    
      
    
       # 写入表头
      
    
     
    32. 
    
  32. 
     
    
      
    
     
    
     
    
      
    
       csv_writer.writeheader()
      
    
     
    33. 
    
  33. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    34. 
    
  34. 
     
    
      
    
     
    
     
    
      
    
       for page in range(1, 11):
      
    
     
    35. 
    
  35. 
     
    
      
    
     
    
     
    
      
    
           url = f'https://www.lagou.com/wn/jobs?pn={page}&fromSearch=true&kd=python'
      
    
     
    36. 
    
  36. 
     
    
      
    
     
    
     
    
      
    
           headers = {
      
    
     
    37. 
    
  37. 
     
    
      
    
     
    
     
    
      
            'Host': 'www.lagou.com',
      
    
     
    38. 
    
  38. 
     
    
      
    
     
    
     
    
      
            'Referer': 'https://www.lagou.com/utrack/trackMid.html?f=https%3A%2F%2Fwww.lagou.com%2Fwn%2Fjobs%3Fpn%3D2%26fromSearch%3Dtrue%26kd%3Dpython&t=1648984113&_ti=1',
      
    
     
    39. 
    
  39. 
     
    
      
    
     
    
     
    
      
            'Cookie': 'user_trace_token=20211122110451-60eec88a-fbaf-47fd-9a53-188f3632144b; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1637550277; _ga=GA1.2.1219095688.1637550277; LGUID=20211122110452-94ffa347-2c46-4c2d-8429-b83e30e86693; RECOMMEND_TIP=true; __lg_stoken__=9ec31e7a3301bab4f215bd5f80c8af0ab0dc2b8ce81af654fe848cf33ad7c4f33d0748020b30281d56a28a756342ce5d42e6c218bcfd56dbf764c51686741cbaf14de987ef24; JSESSIONID=ABAAABAABEIABCIA45B6C458598FF70789BDFD5A4574786; WEBTJ-ID=20220403173842-17feeca7ea0402-090b1b6ee61841-a3e3164-3686400-17feeca7ea15f1; sensorsdata2015session=%7B%7D; X_HTTP_TOKEN=1ca92d1d8ffe4ecb3114898461b10fa2c7054519c6; X_MIDDLE_TOKEN=3e27b9a5a69f9fa78d5d2fe99174c9c5; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%229659966%22%2C%22%24device_id%22%3A%2217d459f2858540-02719bae0efae1-4343363-2073600-17d459f2859704%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24os%22%3A%22Windows%22%2C%22%24browser%22%3A%22Chrome%22%2C%22%24browser_version%22%3A%2298.0.4758.102%22%7D%2C%22first_id%22%3A%2217d459f2858540-02719bae0efae1-4343363-2073600-17d459f2859704%22%7D',
      
    
     
    40. 
    
  40. 
     
    
      
    
     
    
     
    
      
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36'
      
    
     
    41. 
    
  41. 
     
    
      
    
     
    
     
    
      
    
           }
      
    
     
    42. 
    
  42. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    43. 
    
  43. 
     
    
      
    
     
    
     
    
      
    
           rsp = requests.get(url=url, headers=headers)
      
    
     
    44. 
    
  44. 
     
    
      
    
     
    
     
    
      
        print(rsp.status_code)
      
    
     
    45. 
    
  45. 
     
    
      
    
     
    
     
    
      
        # print(rsp.text)
      
    
     
    46. 
    
  46. 
     
    
      
    
     
    
     
    
      
        # <script id="__NEXT_DATA__" type="application/json">(.*?)</script>
      
    
     
    47. 
    
  47. 
     
    
      
    
     
    
     
    
      
    
           html_data = re.findall('<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', rsp.text)[0]
      
    
     
    48. 
    
  48. 
     
    
      
    
     
    
     
    
      
        # print(html_data)
      
    
     
    49. 
    
  49. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    50. 
    
  50. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    51. 
    
  51. 
     
    
      
    
     
    
     
    
      
    
           json_data = json.loads(html_data)
      
    
     
    52. 
    
  52. 
     
    
      
    
     
    
     
    
      
        # print(json_data)
      
    
     
    53. 
    
  53. 
     
    
      
    
     
    
     
    
      
        # pprint.pprint(json_data)
      
    
     
    54. 
    
  54. 
     
    
      
    
     
    
     
    
      
    
           result = json_data['props']['pageProps']['initData']['content']['positionResult']['result']
      
    
     
    55. 
    
  55. 
     
    
      
    
     
    
     
    
      
        # print(result)
      
    
     
    56. 
    
  56. 
     
    
      
    
     
    
     
    
      
        # 格式输出
      
    
     
    57. 
    
  57. 
     
    
      
    
     
    
     
    
      
        # pprint.pprint(result)
      
    
     
    58. 
    
  58. 
     
    
      
    
     
    
     
    
      
        for index in result:
      
    
     
    59. 
    
  59. 
     
    
      
    
     
    
     
    
      
            # pprint.pprint(index)
      
    
     
    60. 
    
  60. 
     
    
      
    
     
    
     
    
      
            # 岗位职责
      
    
     
    61. 
    
  61. 
     
    
      
    
     
    
     
    
      
    
               job_index = index['positionDetail'].replace('<br />', '').replace('<br>', '')
      
    
     
    62. 
    
  62. 
     
    
      
    
     
    
     
    
      
    
               href = f'https://www.lagou.com/wn/jobs{index["positionId"]}.html'
      
    
     
    63. 
    
  63. 
     
    
      
    
     
    
     
    
      
    
               dict1 = {
      
    
     
    64. 
    
  64. 
     
    
      
    
     
    
     
    
      
                '职位名字': index['positionName'],
      
    
     
    65. 
    
  65. 
     
    
      
    
     
    
     
    
      
                '公司名字': index['companyFullName'],
      
    
     
    66. 
    
  66. 
     
    
      
    
     
    
     
    
      
                '工作城市': index['city'],
      
    
     
    67. 
    
  67. 
     
    
      
    
     
    
     
    
      
                '学历要求': index['education'],
      
    
     
    68. 
    
  68. 
     
    
      
    
     
    
     
    
      
                '经验要求': index['workYear'],
      
    
     
    69. 
    
  69. 
     
    
      
    
     
    
     
    
      
                '薪资要求': index['workYear'],
      
    
     
    70. 
    
  70. 
     
    
      
    
     
    
     
    
      
                '公司地址': index['positionAddress'],
      
    
     
    71. 
    
  71. 
     
    
      
    
     
    
     
    
      
                '详情页': href
      
    
     
    72. 
    
  72. 
     
    
      
    
     
    
     
    
      
    
               }
      
    
     
    73. 
    
  73. 
     
    
      
    
     
    
     
    
      
    
               csv_writer.writerow(dict1)
      
    
     
    74. 
    
  74. 
     
    
      
    
     
    
     
    
      
    
               title = index['positionName'] + index['companyFullName']
      
    
     
    75. 
    
  75. 
     
    
      
    
     
    
     
    
      
    
               new_title = re.sub(r'[\/?:"<>|]', '', title)
      
    
     
    76. 
    
  76. 
     
    
      
    
     
    
     
    
      
            with open('info/' + new_title + '.txt', 'w', encoding='utf-8') as f:
      
    
     
    77. 
    
  77. 
     
    
      
    
     
    
     
    
      
    
                   f.write(job_index)
      
    
     
    78. 
    
  78. 
     
    
      
    
     
    
     
    
      
            print(dict1)
      
    
     
    79.

五、王者荣耀英雄皮肤高清壁纸


   
    
    
  1. 
     
    
      
    
     
    
     
    
      
    
       # coding=utf-8
      
    
     
    2. 
    
  2. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    3. 
    
  3. 
     
    
      
    
     
    
     
    
      
    
        作者：gaojs
      
    
     
    4. 
    
  4. 
     
    
      
    
     
    
     
    
      
    
        功能：
      
    
     
    5. 
    
  5. 
     
    
      
    
     
    
     
    
      
    
        新增功能：
      
    
     
    6. 
    
  6. 
     
    
      
    
     
    
     
    
      
    
        日期：2022/4/2 13:05
      
    
     
    7. 
    
  7. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    8. 
    
  8. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    9. 
    
  9. 
     
    
      
    
     
    
     
    
      
    
       import requests
      
    
     
    10. 
    
  10. 
     
    
      
    
     
    
     
    
      
    
       import os
      
    
     
    11. 
    
  11. 
     
    
      
    
     
    
     
    
      
    
       import re
      
    
     
    12. 
    
  12. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    13. 
    
  13. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    14. 
    
  14. 
     
    
      
    
     
    
     
    
      
    
       url = 'https://pvp.qq.com/web201605/js/herolist.json'
      
    
     
    15. 
    
  15. 
     
    
      
    
     
    
     
    
      
    
       headers = {
      
    
     
    16. 
    
  16. 
     
    
      
    
     
    
     
    
      
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.55'
      
    
     
    17. 
    
  17. 
     
    
      
    
     
    
     
    
      
    
       }
      
    
     
    18. 
    
  18. 
     
    
      
    
     
    
     
    
      
    
       rsp = requests.get(url, headers=headers)
      
    
     
    19. 
    
  19. 
     
    
      
    
     
    
     
    
      
    
       # print(rsp.text)
      
    
     
    20. 
    
  20. 
     
    
      
    
     
    
     
    
      
    
       print(rsp.status_code)
      
    
     
    21. 
    
  21. 
     
    
      
    
     
    
     
    
      
    
       # print(rsp.json())
      
    
     
    22. 
    
  22. 
     
    
      
    
     
    
     
    
      
    
       for index in rsp.json():
      
    
     
    23. 
    
  23. 
     
    
      
    
     
    
     
    
      
        # 获取英雄名字和id
      
    
     
    24. 
    
  24. 
     
    
      
    
     
    
     
    
      
    
           hero_name = index['cname']
      
    
     
    25. 
    
  25. 
     
    
      
    
     
    
     
    
      
    
           hero_id = index['ename']
      
    
     
    26. 
    
  26. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    27. 
    
  27. 
     
    
      
    
     
    
     
    
      
        # filename = f'{hero_name}\\'
      
    
     
    28. 
    
  28. 
     
    
      
    
     
    
     
    
      
        # if not os.path.exists(filename):
      
    
     
    29. 
    
  29. 
     
    
      
    
     
    
     
    
      
        # os.mkdir(filename)
      
    
     
    30. 
    
  30. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    31. 
    
  31. 
     
    
      
    
     
    
     
    
      
    
           index_url = f'https://pvp.qq.com/web201605/herodetail/{hero_id}.shtml'
      
    
     
    32. 
    
  32. 
     
    
      
    
     
    
     
    
      
        # print(hero_name, hero_id, index_url)
      
    
     
    33. 
    
  33. 
     
    
      
    
     
    
     
    
      
    
           rsp1 = requests.get(url=index_url, headers=headers)
      
    
     
    34. 
    
  34. 
     
    
      
    
     
    
     
    
      
        # rsp1.encoding = 'gbk'
      
    
     
    35. 
    
  35. 
     
    
      
    
     
    
     
    
      
    
           rsp1.encoding = rsp1.apparent_encoding#自动识别编码
      
    
     
    36. 
    
  36. 
     
    
      
    
     
    
     
    
      
        # print(rsp1.text)
      
    
     
    37. 
    
  37. 
     
    
      
    
     
    
     
    
      
    
           temp = '<ul class="pic-pf-list pic-pf-list3" data-imgname="(.*?)">'
      
    
     
    38. 
    
  38. 
     
    
      
    
     
    
     
    
      
    
           title_list = re.findall('<ul class="pic-pf-list pic-pf-list3" data-imgname="(.*?)">', rsp1.text)[0]
      
    
     
    39. 
    
  39. 
     
    
      
    
     
    
     
    
      
    
           title_list = re.sub('&\d+', '', title_list).split('|')
      
    
     
    40. 
    
  40. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    41. 
    
  41. 
     
    
      
    
     
    
     
    
      
        for num in range(1, len(title_list) + 1):
      
    
     
    42. 
    
  42. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    43. 
    
  43. 
     
    
      
    
     
    
     
    
      
    
               img_url = f'https://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/{hero_id}/{hero_id}-bigskin-{num}.jpg'
      
    
     
    44. 
    
  44. 
     
    
      
    
     
    
     
    
      
    
               img_title = title_list[num - 1]
      
    
     
    45. 
    
  45. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    46. 
    
  46. 
     
    
      
    
     
    
     
    
      
    
               img_data = requests.get(url=img_url, headers=headers).content
      
    
     
    47. 
    
  47. 
     
    
      
    
     
    
     
    
      
            with open('photo/' + img_title + '.jpg', 'wb') as f:
      
    
     
    48. 
    
  48. 
     
    
      
    
     
    
     
    
      
                print(f'=====================正在爬取{hero_name}的皮肤========================')
      
    
     
    49. 
    
  49. 
     
    
      
    
     
    
     
    
      
    
                   f.write(img_data)
      
    
     
    50. 
    
  50. 
     
    
      
    
     
    
     
    
      
            # print(img_title, img_url)
      
    
     
    51.

六、美图网站


   
    
    
  1. 
     
    
      
    
     
    
     
    
      
    
       # coding=utf-8
      
    
     
    2. 
    
  2. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    3. 
    
  3. 
     
    
      
    
     
    
     
    
      
    
        作者：gaojs
      
    
     
    4. 
    
  4. 
     
    
      
    
     
    
     
    
      
    
        功能：
      
    
     
    5. 
    
  5. 
     
    
      
    
     
    
     
    
      
    
        新增功能：
      
    
     
    6. 
    
  6. 
     
    
      
    
     
    
     
    
      
    
        日期：2022/3/26 12:17
      
    
     
    7. 
    
  7. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    8. 
    
  8. 
     
    
      
    
     
    
     
    
      
    
       import os.path
      
    
     
    9. 
    
  9. 
     
    
      
    
     
    
     
    
      
    
       from time import sleep
      
    
     
    10. 
    
  10. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    11. 
    
  11. 
     
    
      
    
     
    
     
    
      
    
       import requests
      
    
     
    12. 
    
  12. 
     
    
      
    
     
    
     
    
      
    
       import re
      
    
     
    13. 
    
  13. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    14. 
    
  14. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    15. 
    
  15. 
     
    
      
    
     
    
     
    
      
    
       dirname = 'photo/'
      
    
     
    16. 
    
  16. 
     
    
      
    
     
    
     
    
      
    
       if not os.path.exists(dirname):
      
    
     
    17. 
    
  17. 
     
    
      
    
     
    
     
    
      
    
           os.mkdir(dirname)
      
    
     
    18. 
    
  18. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    19. 
    
  19. 
     
    
      
    
     
    
     
    
      
    
       url = 'https://www.vmgirls.com/17081.html'
      
    
     
    20. 
    
  20. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    21. 
    
  21. 
     
    
      
    
     
    
     
    
      
    
       headers = {
      
    
     
    22. 
    
  22. 
     
    
      
    
     
    
     
    
      
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46'
      
    
     
    23. 
    
  23. 
     
    
      
    
     
    
     
    
      
    
       }
      
    
     
    24. 
    
  24. 
     
    
      
    
     
    
     
    
      
    
       res = requests.get(url, headers=headers)
      
    
     
    25. 
    
  25. 
     
    
      
    
     
    
     
    
      
    
       # print(res.text)
      
    
     
    26. 
    
  26. 
     
    
      
    
     
    
     
    
      
    
       print(res.status_code)
      
    
     
    27. 
    
  27. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    28. 
    
  28. 
     
    
      
    
     
    
     
    
      
    
       # <a href="(.*?)" alt=".*?" title=".*?">
      
    
     
    29. 
    
  29. 
     
    
      
    
     
    
     
    
      
    
       # 只匹配括号内的内容
      
    
     
    30. 
    
  30. 
     
    
      
    
     
    
     
    
      
    
       url_list = re.findall('<a href="(.*?)" alt=".*?" title=".*?">', res.text)
      
    
     
    31. 
    
  31. 
     
    
      
    
     
    
     
    
      
    
       print(url_list)
      
    
     
    32. 
    
  32. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    33. 
    
  33. 
     
    
      
    
     
    
     
    
      
    
       for urls in url_list:
      
    
     
    34. 
    
  34. 
     
    
      
    
     
    
     
    
      
    
           name = urls.split('/')[-1]
      
    
     
    35. 
    
  35. 
     
    
      
    
     
    
     
    
      
    
           new_url = 'https:' + urls
      
    
     
    36. 
    
  36. 
     
    
      
    
     
    
     
    
      
        # print(new_url)
      
    
     
    37. 
    
  37. 
     
    
      
    
     
    
     
    
      
    
           res_content = requests.get(url=new_url, headers=headers).content
      
    
     
    38. 
    
  38. 
     
    
      
    
     
    
     
    
      
    
           sleep(2)
      
    
     
    39. 
    
  39. 
     
    
      
    
     
    
     
    
      
        # 保存文件
      
    
     
    40. 
    
  40. 
     
    
      
    
     
    
     
    
      
        with open('photo/' + name + '.jpeg', mode='wb') as f:
      
    
     
    41. 
    
  41. 
     
    
      
    
     
    
     
    
      
    
               f.write(res_content)
      
    
     
    42.

七、表情包


   
    
    
  1. 
     
    
      
    
     
    
     
    
      
    
       # coding=utf-8
      
    
     
    2. 
    
  2. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    3. 
    
  3. 
     
    
      
    
     
    
     
    
      
    
        作者：gaojs
      
    
     
    4. 
    
  4. 
     
    
      
    
     
    
     
    
      
    
        功能：
      
    
     
    5. 
    
  5. 
     
    
      
    
     
    
     
    
      
    
        新增功能：
      
    
     
    6. 
    
  6. 
     
    
      
    
     
    
     
    
      
    
        日期：2022/3/25 17:35
      
    
     
    7. 
    
  7. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    8. 
    
  8. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    9. 
    
  9. 
     
    
      
    
     
    
     
    
      
    
       import requests
      
    
     
    10. 
    
  10. 
     
    
      
    
     
    
     
    
      
    
       import re
      
    
     
    11. 
    
  11. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    12. 
    
  12. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    13. 
    
  13. 
     
    
      
    
     
    
     
    
      
    
       def download_photo(name, url):
      
    
     
    14. 
    
  14. 
     
    
      
    
     
    
     
    
      
    
           res = requests.get(url)
      
    
     
    15. 
    
  15. 
     
    
      
    
     
    
     
    
      
        print(res.status_code)
      
    
     
    16. 
    
  16. 
     
    
      
    
     
    
     
    
      
    
           suffix = url.split('.')[-1]
      
    
     
    17. 
    
  17. 
     
    
      
    
     
    
     
    
      
        with open('photo/' + name + '.' + suffix, 'wb') as f:
      
    
     
    18. 
    
  18. 
     
    
      
    
     
    
     
    
      
    
               f.write(res.content)
      
    
     
    19. 
    
  19. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    20. 
    
  20. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    21. 
    
  21. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    22. 
    
  22. 
     
    
      
    
     
    
     
    
      
    
       http://tva1.sinaimg.cn/large/6a2a7a61ly1gy5fd1pb7ij20iz0iz41l.jpg
      
    
     
    23. 
    
  23. 
     
    
      
    
     
    
     
    
      
    
       http://tva1.sinaimg.cn/large/6a2a7a61ly1gy5fd3od4lg208w08wdvb.gif
      
    
     
    24. 
    
  24. 
     
    
      
    
     
    
     
    
      
    
       
      
    
     
    25. 
    
  25. 
     
    
      
    
     
    
     
    
      
    
       https://www.fabiaoqing.com/bqb/lists/page/3.html
      
    
     
    26. 
    
  26. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    27. 
    
  27. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    28. 
    
  28. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    29. 
    
  29. 
     
    
      
    
     
    
     
    
      
    
       def download_page(url):
      
    
     
    30. 
    
  30. 
     
    
      
    
     
    
     
    
      
        # url = 'https://www.fabiaoqing.com/biaoqing/lists/page/3.html'
      
    
     
    31. 
    
  31. 
     
    
      
    
     
    
     
    
      
    
           res1 = requests.get(url)
      
    
     
    32. 
    
  32. 
     
    
      
    
     
    
     
    
      
    
           temp = '<img class="ui image lazy" data-original="(.*?)" src="/Public/lazyload/img/transparent.gif" title="(.*?)" alt="(.*?)" style="max-height:188;margin: 0 auto"/>'
      
    
     
    33. 
    
  33. 
     
    
      
    
     
    
     
    
      
    
           result1 = re.findall(temp, res1.text)
      
    
     
    34. 
    
  34. 
     
    
      
    
     
    
     
    
      
        print(result1)
      
    
     
    35. 
    
  35. 
     
    
      
    
     
    
     
    
      
        for img in result1:
      
    
     
    36. 
    
  36. 
     
    
      
    
     
    
     
    
      
            print(img)
      
    
     
    37. 
    
  37. 
     
    
      
    
     
    
     
    
      
            # name = img[0]
      
    
     
    38. 
    
  38. 
     
    
      
    
     
    
     
    
      
            # new_name = re.sub(r'[\/:*?;"<>|\n]', '_', name)
      
    
     
    39. 
    
  39. 
     
    
      
    
     
    
     
    
      
            # download_photo(img[1], new_name)
      
    
     
    40. 
    
  40. 
     
    
      
    
     
    
     
    
      
    
               download_photo(img[1], img[0])
      
    
     
    41. 
    
  41. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    42. 
    
  42. 
     
    
      
    
     
    
     
    
      
    
       # download_page('https://www.fabiaoqing.com/biaoqing/lists/page/3.html')
      
    
     
    43. 
    
  43. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    44. 
    
  44. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    45. 
    
  45. 
     
    
      
    
     
    
     
    
      
    
       def download_all_page():
      
    
     
    46. 
    
  46. 
     
    
      
    
     
    
     
    
      
        for page in range(1, 50):
      
    
     
    47. 
    
  47. 
     
    
      
    
     
    
     
    
      
    
               pages = 'https://www.fabiaoqing.com/biaoqing/lists/page/' + str(page) + '.html'
      
    
     
    48. 
    
  48. 
     
    
      
    
     
    
     
    
      
    
               download_page(pages)
      
    
     
    49. 
    
  49. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    50. 
    
  50. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    51. 
    
  51. 
     
    
      
    
     
    
     
    
      
    
       download_all_page()
      
    
     
    52.

八、酷狗music


   
    
    
  1. 
     
    
      
    
     
    
     
    
      
    
       # coding=utf-8
      
    
     
    2. 
    
  2. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    3. 
    
  3. 
     
    
      
    
     
    
     
    
      
    
        作者：gaojs
      
    
     
    4. 
    
  4. 
     
    
      
    
     
    
     
    
      
    
        功能：
      
    
     
    5. 
    
  5. 
     
    
      
    
     
    
     
    
      
    
        新增功能：
      
    
     
    6. 
    
  6. 
     
    
      
    
     
    
     
    
      
    
        日期：2022/4/8 12:59
      
    
     
    7. 
    
  7. 
     
    
      
    
     
    
     
    
      
    
       """
      
    
     
    8. 
    
  8. 
     
    
      
    
     
    
     
    
      
    
       import os.path
      
    
     
    9. 
    
  9. 
     
    
      
    
     
    
     
    
      
    
       import pprint
      
    
     
    10. 
    
  10. 
     
    
      
    
     
    
     
    
      
    
       import re
      
    
     
    11. 
    
  11. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    12. 
    
  12. 
     
    
      
    
     
    
     
    
      
    
       import requests
      
    
     
    13. 
    
  13. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    14. 
    
  14. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    15. 
    
  15. 
     
    
      
    
     
    
     
    
      
    
       if not os.path.exists('music/'):
      
    
     
    16. 
    
  16. 
     
    
      
    
     
    
     
    
      
    
           os.mkdir('music/')
      
    
     
    17. 
    
  17. 
     
    
      
    
     
    
     
    
      
    
       url = 'https://www.kugou.com/yy/html/rank.html'
      
    
     
    18. 
    
  18. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    19. 
    
  19. 
     
    
      
    
     
    
     
    
      
    
       headers = {
      
    
     
    20. 
    
  20. 
     
    
      
    
     
    
     
    
      
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36'
      
    
     
    21. 
    
  21. 
     
    
      
    
     
    
     
    
      
    
       }
      
    
     
    22. 
    
  22. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    23. 
    
  23. 
     
    
      
    
     
    
     
    
      
    
       rsp = requests.get(url, headers=headers)
      
    
     
    24. 
    
  24. 
     
    
      
    
     
    
     
    
      
    
       # print(rsp.text)
      
    
     
    25. 
    
  25. 
     
    
      
    
     
    
     
    
      
    
       hash_list = re.findall('"Hash":"(.*?)"', rsp.text)
      
    
     
    26. 
    
  26. 
     
    
      
    
     
    
     
    
      
    
       album_list = re.findall('"album_id":(.*?),', rsp.text)
      
    
     
    27. 
    
  27. 
     
    
      
    
     
    
     
    
      
    
       # print(rsp.text)
      
    
     
    28. 
    
  28. 
     
    
      
    
     
    
     
    
      
    
       zip_list = zip(hash_list, album_list)
      
    
     
    29. 
    
  29. 
     
    
      
    
     
    
     
    
      
    
       for hash1, album_id in zip_list:
      
    
     
    30. 
    
  30. 
     
    
      
    
     
    
     
    
      
        # print(hash1, album_id)
      
    
     
    31. 
    
  31. 
     
    
      
    
     
    
     
    
      
     
      
    
     
    32. 
    
  32. 
     
    
      
    
     
    
     
    
      
    
           index_url = 'https://wwwapi.kugou.com/yy/index.php'
      
    
     
    33. 
    
  33. 
     
    
      
    
     
    
     
    
      
    
           data = {
      
    
     
    34. 
    
  34. 
     
    
      
    
     
    
     
    
      
            'r': 'play/getdata',
      
    
     
    35. 
    
  35. 
     
    
      
    
     
    
     
    
      
            'hash': hash1,
      
    
     
    36. 
    
  36. 
     
    
      
    
     
    
     
    
      
            'dfid': '34dlds4MjPyk0XgC5n0MobxL',
      
    
     
    37. 
    
  37. 
     
    
      
    
     
    
     
    
      
            'appid': '1014',
      
    
     
    38. 
    
  38. 
     
    
      
    
     
    
     
    
      
            'mid': 'fbcb28bbcbd1758696a1eb4363b645d6',
      
    
     
    39. 
    
  39. 
     
    
      
    
     
    
     
    
      
            'platid': '4',
      
    
     
    40. 
    
  40. 
     
    
      
    
     
    
     
    
      
            'album_id': album_id,
      
    
     
    41. 
    
  41. 
     
    
      
    
     
    
     
    
      
            '_': '1649395118742'
      
    
     
    42. 
    
  42. 
     
    
      
    
     
    
     
    
      
    
           }
      
    
     
    43. 
    
  43. 
     
    
      
    
     
    
     
    
      
    
           rsp1 = requests.get(url=index_url, params=data, headers=headers)
      
    
     
    44. 
    
  44. 
     
    
      
    
     
    
     
    
      
        # print(rsp1.json())
      
    
     
    45. 
    
  45. 
     
    
      
    
     
    
     
    
      
        # pprint.pprint(rsp1.json())
      
    
     
    46. 
    
  46. 
     
    
      
    
     
    
     
    
      
    
           audioname = rsp1.json()['data']['audio_name']
      
    
     
    47. 
    
  47. 
     
    
      
    
     
    
     
    
      
    
           playurl = rsp1.json()['data']['play_url']
      
    
     
    48. 
    
  48. 
     
    
      
    
     
    
     
    
      
        # print(audioname, playurl)
      
    
     
    49. 
    
  49. 
     
    
      
    
     
    
     
    
      
    
           music_content = requests.get(url=playurl, headers=headers).content
      
    
     
    50. 
    
  50. 
     
    
      
    
     
    
     
    
      
        with open('music\\' + audioname + '.mp3', 'wb') as f:
      
    
     
    51. 
    
  51. 
     
    
      
    
     
    
     
    
      
            print(f'*************************正在爬取歌曲{audioname}中***********************')
      
    
     
    52. 
    
  52. 
     
    
      
    
     
    
     
    
      
    
               f.write(music_content)
      
    
     
    53.

文章来源: blog.csdn.net，作者：懿曲折扇情，版权归原作者所有，如需转载，请联系作者。

原文链接：blog.csdn.net/qq_41332844/article/details/126837335

【版权声明】本文为华为云社区用户转载文章，如果您发现本社区中有涉嫌抄袭的内容，欢迎发送邮件进行举报，并提供相关证据，一经查实，本社区将立刻删除涉嫌侵权内容，举报邮箱： cloudbbs@huaweicloud.com
音视频
  • 点赞
  • 收藏
  • 关注作者

评论（0）

0/1000
抱歉，系统识别当前为高风险访问，暂不支持该操作

全部回复

上滑加载中

设置昵称

在此一键设置昵称，即可参与社区互动！

*长度不超过10个汉字或20个英文字符，设置后3个月内不可修改。

*长度不超过10个汉字或20个英文字符，设置后3个月内不可修改。

确认 取消

加入云驻计划，成为创作者

  • 华为云周边好礼
  • 免费体验产品
  • 特殊身份标识
  • 线下官方门票
  • 内部专家零距离
  • 与10000+优质创作者共同成长
立即加入