使用async批量获取博客链接

主要使用了superagent  cheerio  async

目的获取所有的博客链接，需要的参数有 博客地址 以及 总页码

      let superagent = require('superagent')
      let cheerio = require('cheerio')
      const async = require('async')
      let baseBlogLink = 'https://fizzz.blog.csdn.net/'  // 博客地址
      let articleSelector = '.article-list h4 a' // 文章选择器
      let totalPage = 9 // 总页码
      const setData = {
       'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
       'Referrer': 'https://fizzz.blog.csdn.net/',
       'Content-Type': 'text/html; charset=UTF-8',
      }
      // Pretend this is some complicated async factory
      var getOnePageBlogLink = function(pageIndex, callback) {
        pageIndex = pageIndex + 1
        superagent
          .get(`${baseBlogLink}/article/list/${pageIndex}`)
          .set(setData)
          .then(res => {
           let blogHrefArr = []
           let $ = cheerio.load(res.text)
           let allEle = $(articleSelector)
           let len = allEle.length
           if (len > 0) {
             console.log(`获取到${len}条博客记录`)
             for (let i = 0; i < len; i++) {
               let blogItem = {
                 name: allEle.eq(i).text().replace(/\s+/g, ''),
                 href: allEle.eq(i).attr('href'),
                }
                blogHrefArr.push(blogItem)
              }
            }
           callback(null, blogHrefArr)
          })
      };
      async.times(totalPage, function(n, next) {
       getOnePageBlogLink(n, function(err, blogs) {
         next(err, blogs);
        });
      }, function(err, blogsArr) {
       const allBlogs = blogsArr.flat()
       console.log(allBlogs)
       console.log(allBlogs.length, 123)
      });
  
 

执行效果图

文章来源: fizzz.blog.csdn.net，作者：拿我格子衫来，版权归原作者所有，如需转载，请联系作者。

原文链接：fizzz.blog.csdn.net/article/details/108999633