proquest的采集程序demo测试截图
程序C#.net开发，程序使用了 Castle Activerecord, Nhibernate ,Spring.net, Memcached分布式数据排重,
自动分页，错误自动重试，超时处理等自能纠错技术，，
配置 部分
</configSections> <appSettings> <add key="startId" value="592" /> <add key="endId" value="600" /> <add key="startPId" value="0" /> <add key="Cookies" value="" /> <add key="DBID" value="10767" /> <add key="SINDEX" value="10" /> <add key="EINDEX" value="20" /> <add key="PM" value="53153" /> <add key="PC" value="6490441" /> <add key="D" value="437140591" /> <add key="backupData" value="53153,6490441,437140541,2315" /> </appSettings>
部分代码
rtn = Zfr.Common.Helper.GetPage(url, cookies, Encoding.GetEncoding("iso-8859-1"), url, null); if (rtn.IndexOf("LOGIN ERROR MESSAGE") != -1) { Zfr.Common.ConsoleProgram.ShowError("线程->LOGIN ERROR,自动登录"); System.Threading.Thread.Sleep(3000); GetCookies(rtn); goto p; } if (rtn.IndexOf("The remote name could not be resolved") != -1 || rtn.IndexOf("has timed out") != -1 || rtn.IndexOf("Bad Request") != -1) { error++; if (error < 4) { Zfr.Common.ConsoleProgram.ShowError("线程->错误,暂停4秒,自动重试"); System.Threading.Thread.Sleep(4000); goto p; } else { Zfr.Common.ConsoleProgram.ShowError("线程->错误,超过3次,回车跳过,继续"); Console.ReadLine(); }
---------------------------------
if (MemcachedBench.KeyExists("" + obj.HASH)) { Zfr.Common.ConsoleProgram.ShowError("KEY->重复:" + obj.HASH); } else {
--------------------------------------------------------------------
if (m.Success) { obj.magLink = m.Groups[0].Value; } DB<ProQuestItem>.Create(obj); Zfr.Common.ConsoleProgram.ShowSuccess("DATA->保存成功 ID:" + obj.ID); MemcachedBench.Set("" + obj.HASH, null); successCount++; SetConfig("backupData", string.Format("{0},{1},{2},{3}", pmid, pcid, did, obj.ID));
----------------------------------------------------------
void ShowSpeed() { TimeSpan ts = DateTime.Now - startTime; double s = ts.TotalSeconds; Zfr.Common.ConsoleProgram.Show(string.Format("INFO->{11}|{8} {9} {10}|{5} {6} {7}|C:{4} SP:{1} {2} {3} SPD:{0}", 24 * 60 * 60 * successCount / s, ts.Hours, ts.Minutes, ts.Seconds,successCount, pmid, pcid, did,sIndex,eIndex,cIndex,dbid)); }
------------------------------------------------
测试程序demo截图pro-qq1163551688.jpg
