[事由：
想安装MinGW，安装文件mingw-get-inst-xxx.exe如果发现x/MinGW/var/cache/mingw-get/packages下有需要的文件，则不会去下载需要的文件。
(x是将要把MinGW安装到的目录，也就是说要事先建立好.../packages目录，并把下载好的文件放到这个目录下)
sourceforge上的MinGW在我住处下载速度为几K，发现mirrorservice.org比较快，于是想获得MinGW全部文件的url，且迅雷貌似不能再命令行下传入参数，快车貌似可以，但。。。没有用快车，命令行是linux的天地，当然要用linux下的下载工具了，于是用了单线程下载工具wget.exe。
]
rootUrl:
http://www.mirrorservice.org/sites/download.sourceforge.net/pub/sourceforge/m/project/mi/mingw/
1] 看下mingw在mirrorservice.org上的目录、文件组织结构
注意:含archive类型的html页面多一个a
2] 使用jsoup解析html
以rootUrl为根(或者叫起点)，用jsoup递归的把全部目录、文件的url弄出来
把所有的这些url保存成文件urls.txt, 用wget.exe下载
(
wget -i urls.txt
可以依次下载urls.txt中的每一行指定的文件
urls.txt有1400多行，实际操作时候, 将urls.txt分成若干个文件，分别执行wget -i urls_x.txt
wget.exe是命令行下单线程下载工具(这也是将urls.txt分成几个文件、执行多个wget -i 命令的原因之一了)(wget来源于linux), linux下多线程下载工具还有好多，但是貌似在windows下重写的(移植到windows下的)却没看到(我没找到)。
)
代码：
import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; import org.jsoup.Connection; import org.jsoup.Connection.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; public class Main { private static PrintStream ps; private static FileOutputStream fs; static{ try { fs = new FileOutputStream("E:/download_MinGW/urls.txt"); ps = new PrintStream(fs); } catch (FileNotFoundException e) { e.printStackTrace(); }//catch }//static public static void main(String[] args) { System.out.println("start time: "+new Date(System.currentTimeMillis()));; List<Resource> resources = new ArrayList<Resource>(); String url = "/sites/download.sourceforge.net/pub/sourceforge/m/project/mi/mingw/"; System.out.println("page mingw/:"); read_page(url, resources); ps.flush(); ps.close(); try { fs.close(); } catch (IOException e) { e.printStackTrace(); }//catch System.out.println("end time: "+new Date(System.currentTimeMillis()));; }//main public static int call_read_page_times = 0; public static void read_page(String url, List<Resource> resources) { Document doc = null; int timeout = 10 * 1000; int status = 404;//找不到 — 服务器找不到给定的资源；文档不存在。 int connTimes = 5; Response resp = null; System.out.println(call_read_page_times + " called"); call_read_page_times++; Connection conn = Jsoup.connect("http://www.mirrorservice.org" + url); conn.timeout(timeout); try { for (int i = 0; i < connTimes; i++) { try { resp = conn.execute(); } catch (SocketTimeoutException e) { continue; } status = resp.statusCode(); if (status == 200) break; }// for if (status != 200) { System.err.println("failed to connect"); return; } doc = resp.parse(); } catch (IOException e) { System.out.println("IOException:conn.execute()"); System.out.println("url:" + url); e.printStackTrace(); } Elements as = doc.getElementsByTag("a"); int i = 0; for (; i < as.size(); i++) { if ("Parent directory".equals(as.get(i).text().trim())) break; } i++; String url_in_page = null, type = null, name = null; for (; i < as.size(); i += 2) { if ("Customise display options".equals(as.get(i).text().trim())) break; url_in_page = as.get(i).attr("href"); type = as.get(i).getElementsByTag("IMG").get(0).attr("ALT"); if ("[archive]".equals(type)) i++; name = as.get(i + 1).text(); Resource resource = new Resource(url_in_page, type, name); resources.add(resource); System.out.println("url#: " + resource.url); System.out.println("type#: " + resource.type); System.out.println("name#: " + resource.name); System.out.println(); if ("[dir]".equals(type.trim())) { List<Resource> resources_in_page = new ArrayList<Resource>(); read_page(url_in_page, resources_in_page); }else{ ps.println("http://www.mirrorservice.org" + url_in_page); } }//for }//read_page public static void print(List<Resource> resources) { for (Resource resource : resources) { System.out.println("url: " + resource.url); System.out.println("type: " + resource.type); System.out.println("name: " + resource.name); System.out.println(); }//for }//print }//Main class Resource { // /sites/download.sourceforge.net/pub/sourceforge/m/project/mi/mingw/MSYS/wget/ public String url; // [dir] public String type; // wget public String name; public Resource(String url, String type, String name) { this.url = url; this.type = type; this.name = name; } }
