用C＃做一个网页数据采集工具

//提取产品列表页中产品最终页的网页        private void button1_Click(object sender, EventArgs e)        {            if (textBox1.Text.Trim() == "" || textBox2.Text.Trim() == "")            {                MessageBox.Show("网址和域名不能为空！", "信息提示", MessageBoxButtons.OK, MessageBoxIcon.Information);                return;            }            try            {                string Html = inc.GetHtml("http://www.shaoqun.com/");                //ArrayList al = inc.GetMatchesStr(Html, "<a[^>]*?>.*?</a>");                ArrayList al = inc.GetMatchesStr(Html, @"href/s*=/s*(?:[/'/""/s](?<1>[^/""/']*)[/'/""])");//提取链接

                StringBuilder sb = new StringBuilder();                foreach (object var in al)                {                    string a = var.ToString().Replace("/"", "").Replace("'", "");                    a = Regex.Replace(a, "href=", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);                    if (a.StartsWith("/"))                        a = textBox2.Text.Trim() + a;                    if (!a.StartsWith("http://"))                        a = "http://" + a;                    sb.Append(a + "/r/n");                }                textBox5.Text = sb.ToString();//把提取到网址输出到一个textBox，每个链接占一行

 

                MessageBox.Show("共提取" + al.Count.ToString() + "个链接", "信息提示", MessageBoxButtons.OK, MessageBoxIcon.Information);

            }            catch (Exception err)            {                MessageBox.Show("提取出错！原因：" + err.Message, "信息提示", MessageBoxButtons.OK, MessageBoxIcon.Information);            }

        }

 

 //把采集的产品页面html代码进行字符串处理，提取需要的代码，最后保存到本地一个access数据库中，同时提取产品图片地址并自动现在图片到本地images文件夹下        private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)        {            //填充产品表            Database.ExecuteNonQuery("delete from Tb_Product");            DataTable dt2 = new DataTable();            OleDbConnection conn = new OleDbConnection(Database.ConnectionStrings);            OleDbDataAdapter da = new OleDbDataAdapter("select * from Tb_Product", conn);            OleDbCommandBuilder cb = new OleDbCommandBuilder(da);            da.Fill(dt2);            dt2.Rows.Clear();

            BackgroundWorker worker = (BackgroundWorker)sender;//这个是做一个进度条

            string[] Urls = textBox5.Text.Trim().ToLower().Replace("/r/n", ",").Split(',');            DataTable dt = new DataTable();            StringBuilder ErrorStr = new StringBuilder();            string html = "", ImageDir = AppDomain.CurrentDomain.BaseDirectory + "Images//";

            //循环每次采集网址            for (int i = 0; i < Urls.Length; i++)            {                try                {                    if (!worker.CancellationPending)                    {                        if (Urls[i] == "")                            return;                        html = inc.GetHtml(Urls[i]);//获取该url的html代码                     DataRow NewRow = dt2.NewRow();

                        //产品名                        string ProductName = html.Substring(html.IndexOf("<title>") + 7);                        NewRow["ProductName"] = ProductName.Remove(ProductName.IndexOf("</title>")).Trim();

                        //产品编号                        NewRow["ModelId"] = NewRow["ProductName"].ToString().Substring(NewRow["ProductName"].ToString().IndexOf("Model:") + 6).Trim();

                        //产品介绍，这些都是根据不同网站的html做相应的修改                        string Introduce = html.Substring(html.IndexOf("Product Details") + 26);                        Introduce = Introduce.Remove(Introduce.IndexOf("</table>") + 8).Trim()

                        NewRow["Introduce"] = Introduce;

 

 

                            //下载图片                            string ProductImage = html.Substring(html.IndexOf("align=center><img") + 17);                            ProductImage = textBox2.Text.Trim() + ProductImage.Substring(ProductImage.IndexOf("src=/"") + 5);                            ProductImage = ProductImage.Remove(ProductImage.IndexOf("/""));                            try                            {                                inc.DownFile(ProductImage, ImageDir + ProductImage.Substring(ProductImage.LastIndexOf("/") + 1));                            }                            catch (Exception)                            {                                ErrorStr.Append("下载图片失败，图片地址：" + ImageDir + ProductImage.Substring(ProductImage.LastIndexOf("/") + 1) + "/r/n");                            }

                        dt2.Rows.Add(NewRow);

                        //Thread.Sleep(100);                        worker.ReportProgress((i + 1) * 100 / Urls.Length, i);                        toolStripStatusLabel1.Text = "处理进度:" + (i + 1).ToString() + "/" + Urls.Length.ToString();//进度条                    }

                }                catch (Exception err)                {                    ErrorStr.Append("采集错误：" + err.Message + ";网址：" + Urls[i] + "/r/n");                }            }            da.Update(dt2);            DataBind(dt2);            ShowError(ErrorStr.ToString());        }

         /// <summary>        /// ASPX页面生成静态Html页面

        /// </summary>        public static string GetHtml(string url)        {            StreamReader sr = null;            string str = null;            //读取远程路径            WebRequest request = WebRequest.Create(url);            HttpWebResponse response = (HttpWebResponse)request.GetResponse();            sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(response.CharacterSet));            str = sr.ReadToEnd();            sr.Close();            return str;        }

     // 提取HTML代码中的网址         public static ArrayList GetMatchesStr(string htmlCode, string strRegex)        {            ArrayList al = new ArrayList();

            Regex r = new Regex(strRegex, RegexOptions.IgnoreCase | RegexOptions.Multiline);            MatchCollection m = r.Matches(htmlCode);

            for (int i = 0; i < m.Count; i++)            {                bool rep = false;                string strNew = m[i].ToString();

                // 过滤重复的URL                 foreach (string str in al)                {                    if (strNew == str)                    {                        rep = true;                        break;                    }                }

                if (!rep) al.Add(strNew);            }

            al.Sort();

            return al;        }

        public static void DownFile(string Url, string Path)        {

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);            HttpWebResponse response = (HttpWebResponse)request.GetResponse();            Stream stream = response.GetResponseStream();            long size = response.ContentLength;            //创建文件流对象            using (FileStream fs = new FileStream(Path, FileMode.OpenOrCreate, FileAccess.Write))            {                byte[] b = new byte[1025];                int n = 0;                while ((n = stream.Read(b, 0, 1024)) > 0)                {                    fs.Write(b, 0, n);                }            }        }