扒了一千多条关于文心一言的评论，我发现这届网友真的不好糊弄…

<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-core</artifactId>
    <version>0.8.0</version>
</dependency>
<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-extension</artifactId>
    <version>0.8.0</version>
</dependency>

public class WenxinProcessor implements PageProcessor {
    private Site site = Site.me()
            .setRetryTimes(3).setSleepTime(1000);

    @Override
    public void process(Page page){
        String title = page.getHtml()
                .xpath("//h1[@class='QuestionHeader-title']/text()").toString();
        String question= page.getHtml()
                .xpath("//div[@class='QuestionRichText']//tidyText()").toString();

        System.out.println(title);
        System.out.println(question);
    }

    public Site getSite(){
        return site;
    }

    public static void main(String[] args){
        Spider.create(new WenxinProcessor())
                .addUrl("https://www.zhihu.com/question/589929380")
                .thread(2)
                .run();
    }
}

@Override
public void process(Page page){
    String contentPath= "div[@class='QuestionAnswers-answers']"+
            "//div[@class='RichContent RichContent--unescapable']" +
            "//div[@class='RichContent-inner']"+
            "/tidyText()";
    List<String> answerList = page.getHtml().xpath(contentPath).all();
    for (int i = 0; i < answerList.size(); i++) {
        System.out.println("第"+(i+1)+"条回答:");
        System.out.println(answerList.get(i)+"\n=======");
    }
}

// 修改前
// private static final String DEFAULT_CONFIG_FILE = "/data/webmagic/webmagic-selenium/config.ini";
// 修改后
private static final String DEFAULT_CONFIG_FILE = "selenium.properties";

# What WebDriver to use for the tests
driver=chrome
# PhantomJS specific config (change according to your installation)
chrome_driver_loglevel=DEBUG

//模拟下拉，刷新页面
for (int i=0; i < 20; i++){
    System.out.println("休眠2s");
    try {
        //滚动到最底部
        ((JavascriptExecutor)webDriver)
                .executeScript("window.scrollTo(0,document.body.scrollHeight)");
        //休眠，等待加载页面
        Thread.sleep(2000);
        //往回滚一点，否则不加载
        ((JavascriptExecutor)webDriver)
                .executeScript("window.scrollBy(0,-300)");
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}

mvn clean install

<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-selenium</artifactId>
    <version>0.8.1.1-SNAPSHOT</version>
</dependency>

public static void main(String[] args){
    Spider.create(new WenxinProcessor())
            .addUrl("https://www.zhihu.com/question/589929380")
            .thread(2)
            .setDownloader(new SeleniumDownloader("D:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe")
                    .setSleepTime(1000))
            .run();
}

if (driver.equals(DRIVER_CHROME)) {
 mDriver = new ChromeDriver(sCaps);
}

if (driver.equals(DRIVER_CHROME)) {
 ChromeOptions options=new ChromeOptions();
 options.setHeadless(true);
 mDriver = new ChromeDriver(options);
}

((JavascriptExecutor)webDriver)
  .executeScript("document.getElementsByClassName('Button QuestionRichText-more')[0].click()");

@Slf4j
@AllArgsConstructor
public class WenxinPipeline implements Pipeline {
    private final ZhihuRepository zhihuRepository;

    @Override
    public void process(ResultItems resultItems, Task task){
        Map<String, Object> map = resultItems.getAll();
        String title = map.get("title").toString();
        String question = map.get("question").toString();
        List<String> answer = (List<String>) map.get("answer");

        ZhihuEntity zhihuEntity;
        for (String an : answer) {
            zhihuEntity = new ZhihuEntity();
            zhihuEntity.setTitle(title);
            zhihuEntity.setQuestion(question);
            zhihuEntity.setAnswer(an);
            try {
                zhihuRepository.save(zhihuEntity);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}

@GetMapping("wenxin")
public void wenxin(){
    new Thread(() -> {
        Request request = new Request("https://www.zhihu.com/question/589929380");
        WenxinProcessor4 wenxinProcessor = new WenxinProcessor4();
        Spider.create(wenxinProcessor)
                .addRequest(request)
                .addPipeline(new WenxinPipeline(zhihuRepository))
                .setDownloader(new SeleniumDownloader("D:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe")
                        .setSleepTime(1000))
                .run();
    }).start();
}

Spider.create(new WenxinProcessor4())
        .addUrl(new String[]{"https://www.zhihu.com/question/589941496",
              "https://www.zhihu.com/question/589904230","https://www.zhihu.com/question/589938328"})
        .addPipeline(new WenxinPipeline(zhihuRepository))
        .setDownloader(new SeleniumDownloader("D:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe")
                .setSleepTime(1000))
        .run();