遇见问题

        嘘嘘,不要让太多的人知道!!!! 

        代理ip是爬虫工资必要的消费,那么如何很好的利用各家服务商提供的免费代理IP呢?

使用方案

     讲解一下,就是在爬取之前先到各家服务商哪里爬取最新的可利用代理IP,然后放在ip池里,然后再去爬取目标网站。如此以来,维护好这个ip池,就可以源源不断的接收新的可以使用的代理ip,剩下的工作就是从网上四处寻找代理网站了。

给个demo吧

import com.alibaba.fastjson.JSON;import com.alibaba.fastjson.JSONArray;import com.alibaba.fastjson.JSONObject;import org.apache.http.HttpEntity;import org.apache.http.HttpHost;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClientBuilder;import org.apache.http.impl.client.HttpClients;import org.apache.http.util.EntityUtils;import java.util.ArrayList;import java.util.List;/** * 使用免费代理demo * * Created by zc on 2017/8/11. */public class HttpClientUtilTest {    public static void main(String[] args) throws Exception {        //第一步,爬取ip代理池        List<ProxyModel> proxyModelList = spriderProxyIp();        String ip = proxyModelList.get(2).getIp();        int port = proxyModelList.get(2).getPort();        System.out.println(ip + " " + port + " " + proxyModelList.get(2).getAnony());        //添加白名单        whilteList();        //请求目标地址        reqWeb(ip, port);    }    /**     * 请求目标地址     *     * @param ip   代理Ip     * @param port 端口     * @throws Exception 异常     */    private static void reqWeb(String ip, int port) throws Exception {        HttpClientBuilder build = HttpClients.custom();        HttpHost proxy = new HttpHost(ip, port);        CloseableHttpClient client = build.setProxy(proxy).build();        String url = "http://write.blog.csdn.net/postedit/77099632";        HttpGet request = new HttpGet(url);        CloseableHttpResponse response = client.execute(request);        HttpEntity entity = response.getEntity();        System.out.println(EntityUtils.toString(entity));    }    /**     * 添加自己外网ip到讯代理白名单     *     * @throws Exception 异常     */    private static void whilteList() throws Exception {        String url = "http://www.xdaili.cn/ipagent/whilteList/addIp?spiderId=dce0442efaac42618205f177c2xxxxip=xx.xx.xx.xx";        HttpGet request = new HttpGet(url);        CloseableHttpClient client = HttpClients.custom().build();        CloseableHttpResponse response = client.execute(request);        System.out.println(response.getStatusLine());    }    /**     * 爬取讯代理IP池     *     * @return ip集合     * @throws Exception 异常     */    private static List<ProxyModel> spriderProxyIp() throws Exception {        List<ProxyModel> proxyModelList = new ArrayList<>();        String url = "http://www.xdaili.cn/ipagent//freeip/getFreeIps?page=1&rows=10";        HttpGet request = new HttpGet(url);        CloseableHttpClient client = HttpClients.custom().build();        CloseableHttpResponse response = client.execute(request);        HttpEntity entity = response.getEntity();        String resTxt = EntityUtils.toString(entity);        JSONObject jsonObject = JSON.parseObject(resTxt);        JSONArray rows = jsonObject.getJSONArray("rows");        rows.stream().map(v -> (JSONObject) v).filter(v -> v.getString("anony").equals("高匿")).forEach(v -> {            ProxyModel model = new ProxyModel();            model.setIp(v.getString("ip"));            model.setPort(Integer.parseInt(v.getString("port")));            model.setResponsetime(v.getString("responsetime"));            model.setAnony(v.getString("anony"));            proxyModelList.add(model);        });        return proxyModelList;    }    public static class ProxyModel {        private String ip;        private int port;        private String responsetime;        private String anony;        public String getIp() {            return ip;        }        public void setIp(String ip) {            this.ip = ip;        }        public int getPort() {            return port;        }        public void setPort(int port) {            this.port = port;        }        public String getResponsetime() {            return responsetime;        }        public void setResponsetime(String responsetime) {            this.responsetime = responsetime;        }        public String getAnony() {            return anony;        }        public void setAnony(String anony) {            this.anony = anony;        }    }