web crawler with java
https://ko.wikipedia.org/wiki/%EC%9B%B9_%ED%81%AC%EB%A1%A4%EB%9F%AC
웹 크롤러(web crawler)는 조직적, 자동화된 방법으로 월드 와이드 웹을 탐색하는 컴퓨터 프로그램입니다.. 웹 크롤러에 대한 다른 용어로는 앤트(ants), 자동 인덱서(automatic indexers), 봇(bots), 웜(worms), 웹 스파이더(web spider), 웹 로봇(web robot) 등이 있습니다.
java로 만든 일단 소스를 아래 공유합니다.
동작은 정해진 http주소를 받아서 web page를 가져와서 파일로 저장하는 소스 입니다. depth를 정해서 html페이지내에 여러개의 링크가 있으면 각 링크들을 방문해서 저장을 하게 됩니다.
결국 웹 페이지를 가져와 저장하는 부분밖에는 없지만 가져온 페이지를 파싱해서 필요한 데이터를 추출하고 분석하는 부분은 용도에 따라 다를 수 있으니 이용자의 몫으로 남겨 두겠습니다.
사용라이브러리는 아래와 같습니다.
httpclient 4.5
commons-cli-1.3.1.jar
jericho-html-3.4.jar
설명은 나중에 추가로 작성하도록 하겠습니다.
소스
package prj.dish; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.zip.GZIPInputStream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.client.utils.URIUtils; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import net.htmlparser.jericho.Config; import net.htmlparser.jericho.Element; import net.htmlparser.jericho.LoggerProvider; import net.htmlparser.jericho.Source; public class Webcrawler { private int maxDepth = 1; private int maxHostChange = 1; private String savePath; private String host; boolean DOMAIN_CHANGE = true; byte[] htmlByte = null; HashSet<String> visited = new HashSet<String>(); CloseableHttpClient httpclient = HttpClients.createDefault(); public static void main(String[] args) { System.out.println("Welcome !! Webcrawler"); Config.LoggerProvider=LoggerProvider.DISABLED; System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog"); if(args.length >= 1){ Options options = new Options(); Option savepath = new Option("s", "savepath", true, "input save folder file path"); savepath.setRequired(true); options.addOption(savepath); Option url = new Option("u", "url", true, "url ex) http://www.daum.net"); url.setRequired(true); options.addOption(url); Option depth = new Option("d", "depth", true, "max depth"); depth.setRequired(false); options.addOption(depth); Option changehostdepth = new Option("c", "changehostdepth", true, "change host depth"); changehostdepth.setRequired(false); options.addOption(changehostdepth); CommandLineParser parser = new DefaultParser(); HelpFormatter formatter = new HelpFormatter(); CommandLine cmd; try { cmd = parser.parse(options, args); } catch (ParseException e) { System.out.println(e.getMessage()); formatter.printHelp("Webcrawler", options); System.exit(1); return; } String saveFilePath = cmd.getOptionValue("savepath"); String urlPath = cmd.getOptionValue("url"); String depthParam = cmd.getOptionValue("depth"); if(depthParam==null || depthParam.isEmpty()) depthParam = "2"; String changehostdepthdepthParam = cmd.getOptionValue("changehostdepth"); if(changehostdepthdepthParam==null || changehostdepthdepthParam.isEmpty()) changehostdepthdepthParam = "1"; System.out.println(urlPath); Webcrawler crawler; crawler = new Webcrawler(); crawler.setSavePath(saveFilePath); crawler.setMaxDepth(Integer.valueOf(depthParam)); crawler.setMaxHostChange(Integer.valueOf(changehostdepthdepthParam)); crawler.run(urlPath); } System.out.println("End Webcrawler"); } private void run(String string) { host = string; connect( host, "/", 0, 0); } public String getString() { try { return new String(htmlByte, "UTF-8"); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } return null; } private String getHttp(String url) throws IOException, URISyntaxException{ String ret=null; try { HttpGet httpGet = new HttpGet(url); HttpClientContext context = HttpClientContext.create(); httpGet.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip"); CloseableHttpResponse response = httpclient.execute(httpGet,context); try { System.out.println(response.getStatusLine()); HttpEntity entity = response.getEntity(); Header contentEncoding = response.getFirstHeader("Content-Encoding"); if (contentEncoding != null && contentEncoding.getValue().equalsIgnoreCase("gzip")) { System.out.println("gziped"); htmlByte = inputStreamToByte( new GZIPInputStream(entity.getContent())); }else { htmlByte = inputStreamToByte(entity.getContent()); } HttpHost target = context.getTargetHost(); List<URI> redirectLocations = context.getRedirectLocations(); URI location = URIUtils.resolve(httpGet.getURI(), target, redirectLocations); System.out.println("Final HTTP location: " + location.toASCIIString()); ret = location.toASCIIString(); } finally { response.close(); } } finally { //httpclient.close(); } return ret; } private byte[] inputStreamToByte(InputStream in) { final int BUF_SIZE = 1024; ByteArrayOutputStream out = new ByteArrayOutputStream(); byte[] buffer = new byte[BUF_SIZE]; try { int length; while ((length = in.read(buffer)) != -1) out.write(buffer, 0, length); } catch (IOException e) { e.printStackTrace(); return null; } return out.toByteArray(); } private void connect(String lasturl, String addurl, int depth, int hostchange) { Source source = null; String newurl = null; int hostchanged = 0; if(addurl.startsWith("http://") || addurl.startsWith("https://")){ hostchanged = 1; if( maxHostChange <= hostchange+hostchanged) return; } if( maxDepth <= depth ){ return; } try { //if(DOMAIN_CHANGE){ lasturl = calcNextUrl(lasturl, addurl); //}else{ //lasturl = urlChg(host, lasturl, addurl); //} System.out.println("Get:["+depth+"]:"+lasturl); if( !visited.contains(lasturl) ){ visited.add(lasturl); }else{ System.out.println("visited !"); return; } //source=new Source(new URL(lasturl)); newurl = getHttp(lasturl); //fileSave(savePath + changeFileName(lasturl)+".htm",getString()); fileSave(savePath + changeFileName(lasturl)+".htm",htmlByte); source=new Source(getString()); } catch (Exception e) { e.printStackTrace(); return; } //System.out.println(source.getRenderer().toString()); List <Element> elements = source.getAllElements("a"); System.out.println("Len:("+htmlByte.length+"), A tag("+elements.size()+")"); for(int i = 0 ; i < elements.size(); i++){ Element ele = elements.get(i); String href = ele.getAttributeValue("href"); if(href==null || href.isEmpty()) continue; if(!DOMAIN_CHANGE){ if(href.startsWith("http://") || href.startsWith("https://")){ continue; } } if(href.startsWith("javascript:")){ continue; }else if(href.contains("#")){ continue; }else if(href.startsWith("<")){ continue; } connect(newurl,href,depth+1,hostchange+hostchanged); } } private void fileSave(String name, byte[] htmlByte) { FileOutputStream stream = null; try{ stream = new FileOutputStream(name); stream.write(htmlByte); } catch (Exception e) { } finally { try { stream.close(); } catch (IOException e) { } } } private String changeFileName(String lasturl) { lasturl=lasturl.replace('?', '_'); lasturl=lasturl.replace('*', '_'); lasturl=lasturl.replace('%', '_'); lasturl=lasturl.replace('.', '_'); lasturl=lasturl.replace('/', '_'); lasturl=lasturl.replace('\\', '_'); lasturl=lasturl.replace('\"', '_'); lasturl=lasturl.replace('\'', '_'); lasturl=lasturl.replace('|', '_'); lasturl=lasturl.replace('+', '_'); lasturl=lasturl.replace('-', '_'); lasturl=lasturl.replace(':', '_'); return lasturl; } private void setMaxDepth(int i) { maxDepth = i; } private void setMaxHostChange(int i) { maxHostChange = i; } private void setSavePath(String string) { savePath = string; if(!savePath.endsWith("/")) savePath=savePath+"/"; createDirectoryIfNeeded(string); String timeStamp = new SimpleDateFormat("yyyy.MM.dd.HH.mm").format(new Date()); savePath=savePath+timeStamp; createDirectoryIfNeeded(savePath); if(!savePath.endsWith("/")) savePath=savePath+"/"; } private void createDirectoryIfNeeded(String directoryName) { File theDir = new File(directoryName); if (!theDir.exists()) theDir.mkdirs(); } private void Webcrawler() { } public static String calcNextUrl(String thisurl, String add) { System.out.println("This:["+thisurl + "]Add:["+add+"]"); URI thisuri = URI.create(thisurl); String data = thisuri.getScheme() + "://" + thisuri.getHost(); if(thisuri.getPort()!=-1) data=":"+thisuri.getPort(); if(add.startsWith("/")) data=data+add; else if(add.startsWith("http")) data=add; else { data=thisurl; if(data.endsWith("/")) data=data+add; else data=data+"/"+add; } URI returi = URI.create(data); returi = returi.normalize(); if( !returi.toString().startsWith("http") ){ System.out.println("Error"); } return returi.toString(); } public static void fileSave(String name,String data) { try { File newTextFile = new File(name); FileWriter fw = new FileWriter(newTextFile); fw.write(data); fw.close(); } catch (IOException iox) { iox.printStackTrace(); } } }
실행할때 아래와 같이 옵션을 주어야 합니다.
-u 는 접속시도하는 url이 됩니다.
-s 는 저장할 폴더 이름입니다.
-d 이건 최대 html의 depth를 의미합니다. 숫자가 크면 링크를 많이 타고 갑니다. 따라서 적당하게 적습니다.
-c 링크를 타고 갈때 다른 site가 나올수 있는데 몇번이나 허용할지 최대 허용 갯수를 의미합니다.
-u http://www.daum.net -s save -d 2 -c 1
실행화면
Welcome !! Webcrawler http://www.daum.net This:[http://www.daum.net]Add:[/] Get:[0]:http://www.daum.net/ HTTP/1.1 200 OK Final HTTP location: http://www.daum.net/ Len:(154748), A tag(170) This:[http://www.daum.net/]Add:[/doc/top_accessibility.html] Get:[1]:http://www.daum.net/doc/top_accessibility.html HTTP/1.1 200 OK Final HTTP location: http://www.daum.net/doc/top_accessibility.html Len:(28899), A tag(40) End Webcrawler
코드 설명
http://swlock.blogspot.com/2017/01/web-crawler-with-java_20.html
