package test;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Proxy;
import java.net.SocketAddress;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.Map;
import java.util.Stack;
import java.util.regex.MatchResult;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class WebScraping {
/**
@param args
/
public static void main(String[] args) {
new WebScraping();
}
private Stack<String> parseStatuses = new Stack<String>();
public WebScraping() {
parseStatuses.push("none");
SocketAddress addr = new InetSocketAddress("**************",
8080);
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
InputStream isr = null;
InputStreamReader isrr = null;
BufferedReader br = null;
try {
URL url = new URL(urlStr);
URLConnection con = url.openConnection(proxy);
String chaset = getCharset(con);
isr = con.getInputStream();
isrr = new InputStreamReader(isr, chaset);
br = new BufferedReader(isrr);
String line = "";
StringBuilder lines = new StringBuilder();
while != null) {
lines.append(line);
}
parse(lines.toString());
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
br.close();
} catch (IOException e1) {
e1.printStackTrace();
}
try {
isrr.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
isr.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private String getCharset(URLConnection con) {
String contentType = con.getContentType();
String[] tokens = contentType.split("charset=");
if (tokens.length > 1) {
return tokens[1];
}
return "UTF-8";
}
private int parse(String line) {
StringBuilder buffer = new StringBuilder();
char[] c = line.toCharArray();
for (int i = 0; i < c.length; i++) {
if (c[i] == '<') {
parseText(buffer.toString());
buffer.setLength(0);
} else if (c[i] == '>') {
parseTag(buffer.toString());
buffer.setLength(0);
} else {
buffer.append(c[i]);
}
}
if (buffer.length() > 0) {
parseText(buffer.toString());
}
return 0;
}
private void parseTag(String str) {
if (str.startsWith("!--")) {
parseComment(str);
} else if (str.startsWith("!DOCTYPE")) {
parseDocType(str);
} else if (str.startsWith("?")) {
parsePI(str);
} else if (str.startsWith("/")) {
parseEndElement(str);
} else if (str.endsWith("/")) {
parseStartEndElement(str);
} else {
parseStartElement(str);
}
}
private void parseStartEndElement(String str) {
}
private void parseDocType(String str) {
}
private void parseStartElement(String str) {
Map<String, String> attrMap = detectAttrMap(str);
parseStatuses.push(attrMap.get(" "));
}
private Map<String, String> detectAttrMap(String str) {
Map<String, String> result = new HashMap<String, String>();
String[] temp = str.split(" ");
result.put(" ", temp[0]);
Pattern pattern = Pattern
.compile("(.+?)=[\\\"\\\'](.+?)[\\\"\\\']");
for (int i = 1; i < temp.length; i++) {
Matcher matcher = pattern.matcher(temp[i]);
if (matcher.matches()) {
result.put(matcher.group(1), matcher.group(2));
}
}
return result;
}
private void parseEndElement(String str) {
if (str.startsWith("/h3")) {
parseStatuses.pop();
}
}
private void parsePI(String str) {
}
private void parseComment(String str) {
}
private void parseText(String str) {
if (parseStatuses.peek().equals("h3")) {
System.out.println(str);
}
}
}
最終更新：2010年01月05日 01:28
