抖音用户评论和ID的采集方法

随着5G时代的普及，各大运营商的套餐纷纷推出，使得流量成本进一步降低。同时5G线路高宽带、低延迟的技术特性和手机普及，让视频用户访问量快速增加。

由于大量的视频工具和拍摄APP的提供，让短视频的拍摄、制作包括发行都变得更加简单便捷，越来越多的普通人可以投入视频创作，创造了更丰富的互联网内容，视频作者从视频推广中能获取粉丝和推广收益。

目前抖音就是目前最重要的短视频平台之一，要及时获取上面的用户评论和ID，需要结合爬虫代理实现采集方案，下面的代码就能快速采集相关信息：

package htmlunit;

import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class HtmlunitDemo {
    // 代理服务器(产品官网 www.16yun.cn)
    final static String proxyHost = "t.16yun.cn";
    final static Integer proxyPort = 31111;

    // 代理验证信息
    final static String proxyUser = "USERNAME";
    final static String proxyPass = "PASSWORD";

    public static void main(String[] args) {

        CredentialsProvider credsProvider = new BasicCredentialsProvider();
        credsProvider.setCredentials(

        new AuthScope(proxyHost, proxyPort),
        new UsernamePasswordCredentials(proxyUser, proxyPass));


        WebClient webClient = new WebClient(BrowserVersion.CHROME,proxyHost, proxyPort);


        webClient.setCredentialsProvider(credsProvider);


        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setActiveXNative(false);
        webClient.getOptions().setCssEnabled(false);

        HtmlPage page = null;

        try {
            page = webClient.getPage("www.douyin.com");
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            webClient.close();
        }

        webClient.waitForBackgroundJavaScript(30000);


        String pageXml = page.asXml();

        System.out.println(pageXml);
    }
}