大三上寒假15天--第13天

package my.webmagic;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;

public class Getgov implements PageProcessor{
    private Site site=Site.me().setRetrySleepTime(3).setSleepTime(100);
    /**
     * @param args
     */
    public Site getSite() {
        // TODO Auto-generated method stub
        return site;
    }

    public void process(Page page) {
        // TODO Auto-generated method stub
        page.putField("allhtml",page.getHtml().toString());
        String urlstr=null;
    for(int i=2;i<=50;i++){
            urlstr="http://www.beijing.gov.cn/so/s?qt=%E4%BF%A1%E4%BB%B6&tab=all&siteCode=1100000088&page="+i;
            page.addTargetRequest(urlstr);
    }
    page.addTargetRequests(page.getHtml().links().regex("view\\S+8").all());
    page.addTargetRequests(page.getHtml().regex("http://www.beijing.gov.cn/hudong/hdjl/com.web.\\S+.\\S+.flow\\?originalId=[^\"]+").all());
    System.out.println(page.getHtml().regex("http://www.beijing.gov.cn/hudong/hdjl/com.web.\\S+.\\S+.flow\\?originalId=[^\"]+").all());
        }
    public static void main(String[] args) {
        Spider.create(new Getgov())
        .addUrl("http://www.beijing.gov.cn/so/s?qt=%E4%BF%A1%E4%BB%B6&tab=all&siteCode=1100000088&page=1")
        .addPipeline(new FilePipeline("./"))
        .setScheduler(new FileCacheQueueScheduler("./"))
        .thread(5)
        .run();
        System.out.println("完成");
  }
}