import java.util.*;import java.net.*;import java.io.*;import javax.swing.text.*;import javax.swing.text.html.*;
/** * That class implements a reusable spider. To use this * class you must have a class setup to recieve * the information found by the spider. This class must * implement the ISpiderReportable method. Written by * Jeff Heaton. Jeff Heaton is the author of "Programming * Spiders, Bots, and Aggregators" by Sybex. Jeff can be * contacted through his web site at http://www.jeffheaton.com. * * @author Jeff Heaton(http://www.jeffheaton.com) * @version 1.0 */public class Spider {
/** * A collection of URL's that resulted in an error. */ protected Collection workloadError = new ArrayList(3);
/** * A collection of URL's that are waiting to * be processed. */ protected Collection workloadWaiting = new ArrayList(3);
/** * A collection of URL's that were processed. */ protected Collection workloadProcessed = new ArrayList(3);
/** * The class that the spider should report its * URL's to. */ protected ISpiderReportable report;
/** * A flag that indicates if this process * should be canceled. */ protected boolean cancel = false;
/** * The constructor. * * @param report A class that implements the ISpiderReportable interface, * that will recieve information that the spider finds. */ public Spider(ISpiderReportable report) { this.report = report; }
/** * Get the URL's that resulted in an error. * * @return A collection of URL's. */ public Collection getWorkloadError() { return workloadError; }
/** * Get the URL's that were waiting to be processed. * You should add one URL to this collection to * begin the spider. * * @return A collection of URL's. */ public Collection getWorkloadWaiting() { return workloadWaiting; }
/** * Get the URL's that were processed by this spider. * * @return A collection of URL's. */ public Collection getWorkloadProcessed() { return workloadProcessed; }
/** * Clear all of the workloads. */ public void clear() { getWorkloadError().clear(); getWorkloadWaiting().clear(); getWorkloadProcessed().clear(); }
/** * Set a flag that will cause the begin * method to return before it is done. */ public void cancel() { cancel = true; }
/** * Add a URL for processing. * * @param url */ public void addURL(URL url) { if ( getWorkloadWaiting().contains(url) ) return; if ( getWorkloadError().contains(url) ) return; if ( getWorkloadProcessed().contains(url) ) return; log("Adding to workload: " + url ); getWorkloadWaiting().add(url); }
/** * Called internally to process a URL. * * @param url The URL to be processed. */ public void processURL(URL url) { try { log("Processing: " + url ); // get the URL's contents URLConnection connection = url.openConnection(); if ( (connection.getContentType()!=null) && !connection.getContentType().toLowerCase().startsWith("text/") ) { getWorkloadWaiting().remove(url); getWorkloadProcessed().add(url); log("Not processing because content type is: " + connection.getContentType() ); return; } // read the URL InputStream is = connection.getInputStream(); Reader r = new InputStreamReader(is); // parse the URL HTMLEditorKit.Parser parse = new HTMLParse().getParser(); parse.parse(r,new Parser(url),true); } catch ( IOException e ) { getWorkloadWaiting().remove(url); getWorkloadError().add(url); log("Error: " + url ); report.spiderURLError(url); return; } // mark URL as complete getWorkloadWaiting().remove(url); getWorkloadProcessed().add(url); log("Complete: " + url );
}
/** * Called to start the spider. */ public void begin() { cancel = false; while ( !getWorkloadWaiting().isEmpty() && !cancel ) { Object list[] = getWorkloadWaiting().toArray(); for ( int i=0;(i<list.length)&&!cancel;i++ ) processURL((URL)list[i]); } }
/** * A HTML parser callback used by this class to * detect links. * * @author Jeff Heaton * @version 1.0 */ protected class Parser extends HTMLEditorKit.ParserCallback { protected URL base;
public Parser(URL base) { this.base = base; }
public void handleSimpleTag(HTML.Tag t, MutableAttributeSet a,int pos) { String href = (String)a.getAttribute(HTML.Attribute.HREF); if( (href==null) && (t==HTML.Tag.FRAME) ) href = (String)a.getAttribute(HTML.Attribute.SRC); if ( href==null ) return;
int i = href.indexOf('#'); if ( i!=-1 ) href = href.substring(0,i);
if ( href.toLowerCase().startsWith("mailto:") ) { report.spiderFoundEMail(href); return; }
handleLink(base,href); }
public void handleStartTag(HTML.Tag t, MutableAttributeSet a,int pos) { handleSimpleTag(t,a,pos);// handle the same way
}
protected void handleLink(URL base,String str) { try { URL url = new URL(base,str); if ( report.spiderFoundURL(base,url) ) addURL(url); } catch ( MalformedURLException e ) { log("Found malformed URL: " + str ); } }
}
/** * Called internally to log information. * This basic method just writes the log * out to the stdout. * * @param entry The information to be written to the log. */ public void log(String entry) { System.out.println( (new Date()) + ":" + entry ); }}
