最近公司网站搞检查，发现了一些问题，一直在用的是xenu工具，速度快，小巧（写了这么多年的MFC,真的很亲切啊，呵呵）
刚好这2天才学习python，所以自己写了一个spider，逐渐也对python有了一些了解，下面把源码分享出来，大家可以玩玩看
文件是utf-8格式，但如果加了中文注释，竟然不能debug了。。。无论你是不是在文件头2行加了#encoding=utf-8等方法，有人解决了告诉我一下哦
用python实现的网站spider 1.功能：检查死链和空title，链接目前只查属于站内的（包括主站和二级站,并且会过滤重复的URL，防止一个URL多次扫描） 2.当页面中有一些错误时，也会提供出错原因和具体位置,所以也可以检查一下网站是否标准 3.目前只查页面，不查资源，过滤了 .jpg|.jpeg|.avi|.png|.gif 4.依赖：必须安装python-3.2rc1（将来也有可能是正式版，可以自己改test.bat中的python的路径，不过不能装低于这个版本的，3.1版本loggingRotating方式有bug） 5.有些不能查出的死链：有时候死链会被重定向到别的页面，如果中国电信会直接跳到它的页面，此情况暂时无法处理，在公司测试没有此现象，adsl上网会这样 6.检查网页是否有语法错误，spider.py中 except HTMLParseError as e: # log.error("HTMLParseError : msg= %s, url= %s, pos= %s" % ( e.msg, url, lParser.getpos())) pass （前提是你已经了解了python的缩进语法）去掉第2行首的“#”即可，目前由于不少网页都有好些小问题，所以默认不开放 7.这是方便检查公司网站的小工具，在公司的几个项目网站上测试可用，不过没有在其他大型网站做过性能和兼容性测试，如果你有兴趣，源码都在里面，自己动手玩玩吧 8.如果有bug，请反馈给我，也可以自己debug找一下(我的开发环境eclipse + pydev) 出错说明： 1.URL出错，一般是死链接 [2011-02-09 11:12:11,392] ERROR : URLError : http://Hardware.global-supplier.com [2011-02-09 15:37:57,148] ERROR : HTTPError : code=400, msg= Bad Request, url= http://www.fashionalgifts.com/../company_info.html 2.页面解析时出错，下例是tag中内容有问题，pos代表页面的出错位置 [2011-02-09 11:12:17,119] ERROR : msg= malformed start tag, HTMLParseError : http://fashion.global-supplier.com/cosmetic-bags/pu-cosmetic-bag-gb-c09.html pos= (238, 27) 3.urlJoinError，一般是相对路径使用错误导致的，如下，第一个是页面URL，第二个是href中的内容，一级子目录竟然要返回上2级的目录 UrlJoin Error:http://www.fashionalgifts.com/company-culture.html/, ../../company_info.html 不过这样的URL在浏览器中是可以正常执行的。。。太强悍了，帮我们屏蔽错误 4.空title title is null, url=***
spider.py(主程序)
#Author:Paul Wang #Date:2011-01-26 #Description: from html.parser import HTMLParser from html.parser import HTMLParseError import urllib.request import myLogger import myHtmlParse import sys debugMode = 1 site = "http://www.geekzones.com/" charset = 'utf-8' logLever = 'debuG' try: def useage(): print(" python spider.py siteUrl charSet logLever") print(" example: python spider.py http://www.baidu.com utf-8 debug") def getInnerURL(url,parentUrl,siteName,log): global times global urlList lParser = myHtmlParse.MyParser(log) try: try: if(url[-1] != '/'): url += '/' if url in urlList or len(url) == 0: return times += 1 urlList.append(url) log.info('times = %d open url : %s' % (times , url)) lParser.currentLink = url; lParser.siteName = siteName; opener = urllib.request.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] htmlSource2 = opener.open(url).read() # req = urllib.request.Request(url) # htmlSource2 = urllib.request.urlopen(req).read() lParser.feed(htmlSource2.decode(charset)) lParser.filterRepeatLink() if(len(lParser.title) == 0): log.error('title is null, url= %s' % url) except urllib.error.HTTPError as e: log.error("HTTPError : code=%d, msg= %s, url= %s, parentUrl=%s" % ( e.code, e.msg, url, parentUrl)) except urllib.error.URLError as e: log.error("URLError : url= %s" % ( url)) except HTMLParseError as e: log.error("HTMLParseError : msg= %s, url= %s, pos= %s, parentUrl=%s" % ( e.msg, url, lParser.getpos(), parentUrl)) finally: lParser.close() for url1 in lParser.link: if url1 in urlList or len(url1) == 0: continue getInnerURL(url1, url, lParser.siteName,log) log = myLogger.myLogger('logging.config') if len(sys.argv) >= 3: site = sys.argv[1] charset = sys.argv[2] log.setLever(sys.argv[3]) else: useage() if debugMode != 1: sys.exit() if debugMode == 1: log.setLever("debUG") urlList = urllib.parse.urlparse(site) host = urlList.scheme + '://' + urlList.hostname siteName = host[ host.find('.') +1: ].rstrip('/') print(siteName) # sys.exit() times = 0; urlList = [] getInnerURL(site, '/',siteName,log) except KeyboardInterrupt: sys.exit()
mylogger.py
#Author:Paul Wang #Date:2011-01-26 #Description:logging wrapper import logging.config class myLogger: LEVELS = {'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL} def __init__(self, configName): print('init logger...') logging.config.fileConfig(configName) self.logger = logging.getLogger("simpleExample") def setLever(self, leverName): if (len(leverName)==0 ): self.logger.setLevel(logging.NOTSET) else: self.logger.setLevel( self.LEVELS.get(leverName.lower(), logging.NOTSET)) print('set lever %s...' % logging.getLevelName(self.logger.level) ) def writeLog(self, msg): self.logger.log(self.logger.level, msg) def debug(self, msg): self.logger.debug(msg) def info(self, msg): self.logger.info(msg) def warn(self, msg): self.logger.warn(msg) def error(self, msg): self.logger.error(msg) def critical(self, msg): self.logger.critical(msg) def __del__(self): print('destructor logger...')
myHtmlParse.py
#Author:Paul Wang #Date:2011-01-26 from html.parser import HTMLParser import urllib import re import myLogger class MyParser(HTMLParser): def __init__ (self,log): HTMLParser. __init__ (self) self.currentLink = '' self.link = [] self.exlink = [] self.host = '' self.siteName = '' #like 'baidu.com' self.title = '' self.titleFlag = 0 self.log = log def handle_starttag(self, tag, attrs): if tag == 'a' : if len(attrs)==0:pass for name, value in attrs: if name == 'href' : pattern = re.compile('(.jpg|.jpeg|.avi|.png|.gif)</p>,re.I) match = pattern.search(value) if match: continue if len(value) and value[0] != '#' and value.find('@') == -1: if value.find(self.siteName) != -1 / or value[0] == '/' / or value[0:2] == './' / or value[0:3] == '../': if(self.currentLink[-1] != '/'): self.currentLink += '/' url = urllib.request.urljoin(self.currentLink, value) if(url.find("..") != -1 ): self.log.error("UrlJoin Error:%s, %s" % (self.currentLink, value)) self.link.append( url) else: self.exlink.append(value.rstrip('/')) elif tag == 'title': if len(attrs)==0:pass self.titleFlag = 1 def handle_data(self, data): if self.titleFlag == 1 and len(self.title)== 0: self.title=data self.titleFlag = 0 def filterRepeatLink(self): # print('count1: %d' % len(self.link)) dict = {} for keys in self.link: dict.setdefault(keys, '111') # print('count2: %d' % len(dict)) self.link = dict.keys() # print('count2: %d' % len(self.link))
logging.config(配置档，用过log4c等的一看就明白了)
[loggers] keys=root,example [handlers] keys=consoleHandler,timedRotatingFileHandler [formatters] keys=simpleFormatter [formatter_simpleFormatter] #format=[%(asctime)s]%(name)s : %(message)s format=[%(asctime)s] %(levelname)s : %(message)s [logger_root] level=DEBUG handlers=consoleHandler,timedRotatingFileHandler [logger_example] level=DEBUG handlers=consoleHandler,timedRotatingFileHandler qualname=example propagate=0 [handler_consoleHandler] class=StreamHandler level=DEBUG formatter=simpleFormatter args=(sys.stdout,) [handler_rotateFileHandler] class=handlers.RotatingFileHandler level=DEBUG formatter=simpleFormatter args=('test.log', 'a', 1024*1024, 9) [handler_timedRotatingFileHandler] class=handlers.TimedRotatingFileHandler level=ERROR formatter=simpleFormatter args=('app.log','d')
