使用async批量获取博客链接

主要使用了superagent  cheerio  async

目的获取所有的博客链接，需要的参数有 博客地址 以及 总页码

  

   

    

     

    
    

     

      let superagent = require('superagent')
     
    
   

    

     

    
    

     

      let cheerio = require('cheerio')
     
    
   

    

     

    
    

     

      const async = require('async')
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      let baseBlogLink = 'https://fizzz.blog.csdn.net/'  // 博客地址
     
    
   

    

     

    
    

     

      let articleSelector = '.article-list h4 a' // 文章选择器
     
    
   

    

     

    
    

     

      let totalPage = 9 // 总页码
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      const setData = {
     
    
   

    

     

    
    

     
  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
     
    
   

    

     

    
    

     
  'Referrer': 'https://fizzz.blog.csdn.net/',
     
    
   

    

     

    
    

     
  'Content-Type': 'text/html; charset=UTF-8',
     
    
   

    

     

    
    

     

      }
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      // Pretend this is some complicated async factory
     
    
   

    

     

    
    

     

      var getOnePageBlogLink = function(pageIndex, callback) {
     
    
   

    

     

    
    

     

        pageIndex = pageIndex + 1
     
    
   

    

     

    
    

     

        superagent
     
    
   

    

     

    
    

     

          .get(`${baseBlogLink}/article/list/${pageIndex}`)
     
    
   

    

     

    
    

     

          .set(setData)
     
    
   

    

     

    
    

     

          .then(res => {
     
    
   

    

     

    
    

     
      let blogHrefArr = []
     
    
   

    

     

    
    

     
      let $ = cheerio.load(res.text)
     
    
   

    

     

    
    

     
      let allEle = $(articleSelector)
     
    
   

    

     

    
    

     
      let len = allEle.length
     
    
   

    

     

    
    

     
      if (len > 0) {
     
    
   

    

     

    
    

     
        console.log(`获取到${len}条博客记录`)
     
    
   

    

     

    
    

     
        for (let i = 0; i < len; i++) {
     
    
   

    

     

    
    

     
          let blogItem = {
     
    
   

    

     

    
    

     
            name: allEle.eq(i).text().replace(/\s+/g, ''),
     
    
   

    

     

    
    

     
            href: allEle.eq(i).attr('href'),
     
    
   

    

     

    
    

     

                }
     
    
   

    

     

    
    

     

                blogHrefArr.push(blogItem)
     
    
   

    

     

    
    

     

              }
     
    
   

    

     

    
    

     

            }
     
    
   

    

     

    
    

     
      callback(null, blogHrefArr)
     
    
   

    

     

    
    

     

          })
     
    
   

    

     

    
    

     

      };
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      async.times(totalPage, function(n, next) {
     
    
   

    

     

    
    

     
  getOnePageBlogLink(n, function(err, blogs) {
     
    
   

    

     

    
    

     
    next(err, blogs);
     
    
   

    

     

    
    

     

        });
     
    
   

    

     

    
    

     

      }, function(err, blogsArr) {
     
    
   

    

     

    
    

     
  const allBlogs = blogsArr.flat()
     
    
   

    

     

    
    

     
  console.log(allBlogs)
     
    
   

    

     

    
    

     
  console.log(allBlogs.length, 123)
     
    
   

    

     

    
    

     

      });
     
    
  
 

执行效果图

文章来源: fizzz.blog.csdn.net，作者：拿我格子衫来，版权归原作者所有，如需转载，请联系作者。

原文链接：fizzz.blog.csdn.net/article/details/108999633