船舶大数据爬虫采集长效固定ip在java使用中的解决方案

时间:2022-08-15   来源:枫之叶网络

随着跨境电商的迅速发展，海运船舶实时运输信息非常重要，很多船运需要采用爬虫技术进行数据抓取并使用（注意此处的信息为公开非敏感信息哦）。分布式采集和监控技术突飞猛进。java语言经常使用固定长效ip进行分布式数据采集和监控。以下是demon：

    import java.io.ByteArrayOutputStream;

    import java.io.InputStream;

    import java.net.Authenticator;

    import java.net.HttpURLConnection;

    import java.net.InetSocketAddress;

    import java.net.PasswordAuthentication;

    import java.net.Proxy;

    import java.net.URL;

    class ProxyAuthenticator extends Authenticator {

        private String user, password;

        public ProxyAuthenticator(String user, String password) {

            this.user     = user;

            this.password = password;

        }

        protected PasswordAuthentication getPasswordAuthentication() {

            return new PasswordAuthentication(user, password.toCharArray());

        }

    }

    public class ProxyTest {

        public static void main(String args[]) throws Exception {

            String targetUrl = "http://www.pachongdaili.com";

            System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");

            String proxyServer = "http://demon.pachongdaili.com";

            int proxyPort      = 65108;

            String proxyUser  = "user";

            String proxyPass  = "pwd";

            try {

                URL url = new URL(targetUrl);

                Authenticator.setDefault(new ProxyAuthenticator(proxyUser, proxyPass));

                InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);

                Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);

                HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);

                connection.setRequestProperty("Proxy-Switch-Ip","yes");

                byte[] response = readStream(connection.getInputStream());

                System.out.println(new String(response));

            } catch (Exception e) {

                System.out.println(e.getLocalizedMessage());

            }

        }

        public static byte[] readStream(InputStream inStream) throws Exception {

            ByteArrayOutputStream outSteam = new ByteArrayOutputStream();

            byte[] buffer = new byte[1024];

            int len = -1;

            while ((len = inStream.read(buffer)) != -1) {

                outSteam.write(buffer, 0, len);

            }

            outSteam.close();

            inStream.close();

            return outSteam.toByteArray();

        }

    }

              

当前地址：http://www.pachongdaili.com/support/a452.html 客服联系QQ：475685360