CMFLinkChecker/0040751000076400001440000000000007755461277013170 5ustar ctheuneusersCMFLinkChecker/Extensions/0040751000076400001440000000000007755461277015327 5ustar ctheuneusersCMFLinkChecker/Extensions/Install.py0100640000076400001440000000075707755434061017304 0ustar ctheuneusersfrom Products.CMFLinkChecker.LinkCheckerTool import LinkCheckerTool from StringIO import StringIO def install(self): out=StringIO() if not hasattr(self, "_isPortalRoot"): print >> out, "Must be installed in a CMF Site (read Plone)" return self.manage_addProduct['CMFLinkChecker'].manage_addTool(type='CMF Linkchecker Tool') print >> out, 'Installing portal_linkchecker' print >> out, 'Successfully installed CMFLinkChecker' return out.getvalue() CMFLinkChecker/Extensions/__init__.py0100640000076400001440000000003007755434061017415 0ustar ctheuneusers#poof, you're a product CMFLinkChecker/LinkCheckerTool.py0100640000076400001440000003665007755461503016561 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool $Id: LinkCheckerTool.py,v 1.4 2003/11/15 17:32:51 ctheune Exp $""" # Python stuff from types import StringTypes # Global Zope stuff from DateTime import DateTime from AccessControl import ClassSecurityInfo from OFS.SimpleItem import SimpleItem from Globals import InitializeClass, PersistentMapping, Persistent from Products.CMFCore.utils import UniqueObject from Products.CMFCore.CMFCorePermissions import ManagePortal from Products.PageTemplates.PageTemplateFile import PageTemplateFile # Sibling imports from interfaces import ILinkCheckerTool, ILinkInformation from LinkInfo import LinkInfo import retrievers import zLOG # Threading stuff from threading import Thread try: from threading import BoundedSemaphore except ImportError: from threading import Semaphore BoundedSemaphore = Semaphore import urllib, httplib import httpclient import Queue import time # Threading and transactions are a pain import Globals import OFS.Application import ZODB.ZApplication # Link checker stuff from httplib import ResponseNotReady from httpcodes import http_codes retrieval_queue = Queue.Queue(0) # Make this an infinite synchronized queue thread_stop = 0 active_threads = 5 thread_count = 0 pool_threads = BoundedSemaphore(value=active_threads) class TooManyThreadsRunning(Exception): pass class UnknownLink(Exception): pass class LinkCheckerTool(UniqueObject, SimpleItem): """A link checker tool. Encapsulating mechanisms for finding links in the site, testing their availability and notifying their owners.""" __implements__ = (ILinkCheckerTool, ) id = 'portal_linkchecker' meta_type = 'CMF Linkchecker Tool' security = ClassSecurityInfo() plone_tool = 1 manage_options = ( ({'label' : 'Overview', 'action' : 'manage_overview'}, {'label' : 'Configure', 'action' : 'manage_configure'}, {'label' : 'Linklist', 'action' : 'manage_checklinks'}, {'label' : 'Linkchecking', 'action' : 'manage_checker'}, ) + SimpleItem.manage_options ) maximumCacheAge = 60*60 # 1 hour gracefulUpdateAge = 60*45 # 45 minutes defaultURIPrefix = "" # No prefix by default # # ZMI methods # security.declareProtected(ManagePortal, 'manage_overview') manage_overview = PageTemplateFile('www/explainLinkcheckerTool', globals(), __name__='manage_overview') security.declareProtected(ManagePortal, 'manage_configure') manage_configure= PageTemplateFile('www/configureLinkcheckerTool', globals(), __name__='manage_configure') security.declareProtected(ManagePortal, 'manage_checklinks') manage_checklinks = PageTemplateFile('www/checkLinks', globals(), __name__='manage_checklinks') security.declareProtected(ManagePortal, 'manage_checker') manage_checker = PageTemplateFile('www/checker', globals(), __name__='manage_checker') security.declareProtected(ManagePortal, 'edit_configuration') def edit_configuration(self, maxage, graceage, defaultURIPrefix): """Update the configuration options.""" self.setMaximumCacheAge(maxage) self.setGracefulUpdateAge(graceage) self.setDefaultURIPrefix(defaultURIPrefix) if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_configure') security.declareProtected(ManagePortal, 'manage_checkLink') def manage_checkLink(self, url): """ZMI method for checking a link.""" if not self._cache.has_key(url): if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checklinks?manage_tabs_message=No+such+link') return data = self._cache[url] self.checkLink(data) if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checklinks?manage_tabs_message=Link+checked') def manage_forceCheckLink(self, url): """ZMI method for forcing the check of a link.""" if not self._cache.has_key(url): if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checklinks?manage_tabs_message=No+such+link') return data = self._cache[url] self.__queueAddLink(data) if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checklinks?manage_tabs_message=Check+forced') security.declareProtected(ManagePortal, 'manage_checkAll') def manage_checkAll(self): """ZMI method for checking all links.""" self.checkAllLinks() if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect(self.absolute_url()+'/manage_checker?manage_tabs_message=All+links+checked+or+queued.') security.declareProtected(ManagePortal, 'manage_checkAll') def manage_forceCheckAll(self): """ZMI method for checking all links.""" self.forceCheckAllLinks() if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect(self.absolute_url()+'/manage_checker?manage_tabs_message=All+links+forced+into+que.') security.declareProtected(ManagePortal, 'manage_findLinks') def manage_findLinks(self): """ZMI method for finding new links.""" count = self.findLinks() if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect(self.absolute_url()+'/manage_checklinks?manage_tabs_message=%s+links+added.+%s+links+removed.' % count) def manage_activateWorkerThread(self): """ZMI method for activating a new worker thread.""" try: self.activateWorkerThread() message = "Activated worker thread." except TooManyThreadsRunning: message = "Already %s running." % active_threads if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checker?manage_tabs_message=%s' % message) # # Interface methods # def __init__(self): self._retrievers = PersistentMapping() self._cache = PersistentMapping() # register all global retrievers for type, retriever in retrievers.retrievers.items(): self.registerRetriever(type, retriever) ## registry functions for link information retrieval security.declareProtected(ManagePortal, 'registerRetriever') def registerRetriever(self, portal_type, function_ref): assert not self._retrievers.has_key(portal_type), \ "Duplicate portal_type registration (%s)" % portal_type self._retrievers[portal_type] = function_ref def getRetrievableTypes(self): return self._retrievers.keys() def getRetriever(self, type): return self._retrievers[type] security.declareProtected(ManagePortal, 'checkLink') def checkLink(self, data): if not self._cache.has_key(data.getCacheKey()): raise UnknownLink now = time.time() if data.lastcheck: age = data.lastcheck.millis() / 1000 else: age = 0 # Cache invalid? Requeue an update and return "unknown" if now - age > self.maximumCacheAge: self.__queueAddLink(data) return "unknown" # Update required? if now - age > self.gracefulUpdateAge: self.__queueAddLink(data) return data.state security.declareProtected(ManagePortal, 'checkAllLinks') def checkAllLinks(self): for link in self._cache.values(): self.checkLink(link) security.declareProtected(ManagePortal, 'checkAllLinks') def forceCheckAllLinks(self): for link in self._cache.values(): self.__queueAddLink(link) security.declareProtected(ManagePortal, 'findLinks') def findLinks(self): count = 0 links = [] stale = self._cache.keys() # gather all objects that are of a type we can check for links for key in self.getRetrievableTypes(): objects = self.portal_catalog(portal_type=key) retriever = self.getRetriever(key) for ob in objects: ob = ob.getObject() for link in retriever(ob): if not self._cache.has_key(link.getCacheKey()): self._cache[link.getCacheKey()] = link count = count + 1 if link.getCacheKey() in stale: del stale[stale.index(link.getCacheKey())] # Update the URLS, they may need context. XXX this is icky for ob in self._cache.values(): ob = ob.__of__(self) oldhash = ob.getCacheKey() ob.setUrl(ob.url, self.REQUEST) if oldhash != ob.getCacheKey(): del self._cache[oldhash] self._cache[ob.getCacheKey()] = ob if ob.getCacheKey() in stale: del stale[stale.index(ob.getCacheKey())] # Everything in stale is stale. Kick it from the cache stalecount = len(stale) for x in stale: del self._cache[x] return count, stalecount security.declareProtected(ManagePortal, 'showLinks') def showLinks(self): """Show a grouped list of link data.""" data = self._cache.values() # group them all grouped = {} for x in data: group = x.getGroupId() if not grouped.has_key(group): grouped[group] = [] grouped[group].append(x) return grouped return self._cache.values() def checkWholeSite(self): for link in self.findLinks(): self.checkLink(link) security.declareProtected(ManagePortal, 'setMaximumCacheAge') def setMaximumCacheAge(self, age): self.maximumCacheAge = age security.declareProtected(ManagePortal, 'setGracefulUpdateAge') def setGracefulUpdateAge(self, age): assert age < self.maximumCacheAge, \ "Graceful update age is larger than maximum cache age." self.gracefulUpdateAge = age security.declareProtected(ManagePortal, 'setDefaultURIPrefix') def setDefaultURIPrefix(self, defaultURIPrefix): assert type(defaultURIPrefix) in StringTypes self.defaultURIPrefix = defaultURIPrefix security.declareProtected(ManagePortal, 'queueSize') def queueSize(self): global retrieval_queue return retrieval_queue.qsize() security.declareProtected(ManagePortal, 'activateWorkerThread') def activateWorkerThread(self): if not pool_threads.acquire(0): raise TooManyThreadsRunning global thread_count thread_count += 1 thread = URLCheckerThread(retrieval_queue) thread.start() security.declareProtected(ManagePortal, 'countWorkingThreads') def countWorkingThreads(self): global thread_count return thread_count ### Private methods security.declarePrivate('__queueAddLink') def __queueAddLink(self, info): """Schedules a new check. Info must be a ILinkInfo object.""" retrieval_queue.put((self.getPhysicalPath(), info.getCacheKey())) info.queued = 1 InitializeClass(LinkCheckerTool) #### Threading code. # We only use one worker thread. Object references # should be stable to work with. class URLCheckerThread(Thread): def __init__(self, queue): Thread.__init__(self, name="URLChecker") self.queue = queue def run(self): zLOG.LOG('CMFLinkChecker', 0, 'Initialized', '\tWorker thread has been initialized.\n') # try to get a zodb connection check_app = None while 1: if check_app: get_transaction().commit() check_app._p_jar.sync() check_app._p_jar.close() check_app = None # we are doing cooperative multitasking # this thread doesn't need too much power time.sleep(0.1) try: linkchecker, key = self.queue.get(block=0) except Queue.Empty: time.sleep(1) continue # We got work, so setup the database connection. DB=Globals.DB app=ZODB.ZApplication.ZApplicationWrapper( DB, 'Application', OFS.Application.Application, (), Globals.VersionNameName) check_app=app() o = check_app # find the linkchecker object for p in linkchecker: if p: o=getattr(o,p) linkchecker = o item = linkchecker._cache[key] # Links need to revalidate everytime otherwise they are in alert state. item.state_class = "alert" item.queued = 0 item.lastcheck = DateTime() # only check http links. other links are ... hmm # ... unknown ... type, url = urllib.splittype(item.url) if not type == 'http': item.state = 'unknown' item.state_class = 'unknown' item.lasttime = 0 item.headers = {} continue # get a connection to the server host, path = urllib.splithost(url) if host is None: item.state = "invalid" item.state_class = "unknown" item.reason = "Invalid Link: No hostname defined" continue host, port = urllib.splitnport(host, 80) conn = httpclient.myHTTPConnection(host, port, 2) starttime = time.time() # request the link try: conn.request('GET', path) except: import sys item.state = "unavailable" item.reason = sys.exc_value continue # look into the response response = None while not response: try: now = time.time() response = conn.getresponse() except ResponseNotReady: if starttime - now > 4: break time.sleep(0.2) conn.close() response.close() item.lasttime = now - starttime item.headers = {} # what if we didn't got a response? if not response: item.state = 'failed' continue item.state = response.status if not http_codes.has_key(item.state): item.state_class = 'unknown' continue translated_code = http_codes.get(item.state) if translated_code[0]: item.state_class="green" item.reason = translated_code[1] zLOG.LOG('CMFLinkChecker', 0, 'Stopped', '\tWorker thread stopped.\n') global thread_count thread_count -= 1 CMFLinkChecker/LinkInfo.py0100640000076400001440000000447007755461505015247 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool - link information object $Id: LinkInfo.py,v 1.3 2003/11/15 17:32:53 ctheune Exp $""" from Globals import Persistent from Acquisition import Implicit from OFS.SimpleItem import SimpleItem from interfaces import ILinkInformation from urllib import urlencode from Zope import app from Products.CMFCore.utils import getToolByName from httpcodes import http_codes app_ = None class LinkInfo(SimpleItem): __implements__ = (ILinkInformation, ) __allow_access_to_unprotected_subobjects__ = 1 # XXX make accessor functions owner = None object = None url = None lastcheck = None lasttime = None queued = 0 state = "unknown" state_class = "alert" headers = {} reason = "" def getCacheKey(self): """Construct a key that's good for identifying this link. Returns a key that is usable as a query string and a unique identifier for this object.""" return str(tuple(self.object) + (self.url, )) def getObject(self): """Return a reference to the object.""" object = self.__getApp().unrestrictedTraverse(self.object) return object def __getApp(self): global app_ if not app_: app_ = app() return app_ def setUrl(self, url, REQUEST=None): """Set the URL. Sets the URL to the given value, trying to make relative paths absolute. (Requires a REQUEST object)""" self.url = url if REQUEST is None: return # Try to normalize relative links oldurl = self.url if self.url.startswith("/"): # Maybe we have a default prefix on the tool? lt = getToolByName(self, 'portal_linkchecker', None) self.url = getattr(lt, 'defaultURIPrefix', REQUEST.BASE0) + self.url elif not ':' in self.url: try: self.url = REQUEST.BASE0+'/'.join(self.object)+'/'+self.url except: pass def getGroupId(self): return http_codes.get(self.state, [0, 'unknown'])[1] CMFLinkChecker/README.txt0100640000076400001440000000643407755454453014670 0ustar ctheuneusersCMFLinkchecker What is it? CMFLinkchecker is a CMF/Plone portal tool that can be used to monitor links within content of the portal. It is not used to monitor links that are generated by navigation code. It gathers links on demand automatically from different content types (Documents, News, Links ...) and puts them into it's monitoring list. It then tries to retrieve the object that is specified by the link (currently only HTTP-Links are supported) and remembers the status code. A colored and grouped list of all links of the portal gives you a detailed and high level overview on the status of the links on your site. How to install Extract the archive into your Zope's 'Products' directory and restart your Zope server. Easy way Use the "CMFQuickInstaller":http://sourceforge.net/projects/collective/CMFQuickInstaller to install it. Manual way Go to your CMF (or Plone) site and take a look in the Add menu. You should encounter a new entry saying "CMFLinkChecker tool". Select it. As there is only one tool in this package, you will have to select it in the upcoming screen a second time and hit the button. Now you should have a "portal_linkchecker" tool within your CMF site. Getting started Go within the ZMI into your linkchecker instance ('portal_linkchecker'). There are a couple of interesting tabs. Tab 1: Configure The 'maximum cache age' determines the period of time a result of a linkcheck is considered valid. The 'graceful update age' is the period of time that is still a valid result but will already be rechecked. The 'Default URI prefix' is added to relative links to determine a complete URL (e.g. add 'http://www.yourdomain.com:1234' here). Tab 2: Linklist The linklist gives you feedback about the links in your site and let's you gather new links. The "Check"-Links will put the link into the queue if there is no valid (either 'unknown' or timed out) result for it currently. The "Force"-Links will put the link into the queue without considering the age of the current result. The "Find new links" button will ask the catalog of your site for contents of types that we have link fetching adapters for and update the list of links (remove stale and add new). Tab 3: Linkchecking This page gives an overview of the number of concurrent threads working on your checking queue as well as the current number of items in the queue. Using the 'Check all links' and 'Force all links' buttons you can 'Check' or 'Force' all links at once. (See description of 'Tab 2: Linklist'). By default you can activate up to 5 concurrent threads for checking links. What? Problems? If you encounter any problems or have an inquiry about the product, feel free to contact Christian Theune:"ct@gocept.com" or put a bug report in http://bugs.gocept.com. Any feedback is appreciated. Commercial support is available from gocept. Contact via "mail@gocept.com":mail@gocept.com or +49 3496 30 99 112. (Located in Germany, we speak german and english) CMFLinkChecker/__init__.py0100640000076400001440000000116707737470361015275 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool initalisation code $Id: __init__.py,v 1.1.1.1 2003/10/04 07:04:17 ctheune Exp $""" from Products.CMFCore import utils import LinkCheckerTool tools = (LinkCheckerTool.LinkCheckerTool, ) def initialize(context): utils.ToolInit('CMFLinkChecker Tool', tools=tools, product_name='CMFLinkChecker', icon="tool.gif").initialize(context) CMFLinkChecker/httpclient.py0100640000076400001440000000271007737471213015704 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker customized http connection. (yet unused) $Id: httpclient.py,v 1.2 2003/10/04 07:11:07 ctheune Exp $""" import socket import httplib import time class TimeOut(Exception): pass myHTTPConnection = httplib.HTTPConnection class my2HTTPConnection(httplib.HTTPConnection): def __init__(self, host, port, timeout): httplib.HTTPConnection.__init__(self, host, port) self.timeout = timeout def send(self,str): if self.sock is None: if self.auto_open: self.connect() else: raise httplib.NotConnected() # that's the new code self.sock.setblocking(0) time1 = time.time() while 1: try: self.sock.sendall(str) except socket.error, v: if v[0] == 32: # Broken pipe self.close() raise if v[0] == 35: # Would block continue time.sleep(0.001) time2 = time.time() if time2-time1 > self.timeout: raise "Timeout", time2-time1 if self.debuglevel > 0: print "send:", repr(str) CMFLinkChecker/httpcodes.py0100640000076400001440000000444407755461505015534 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """HTTP 1.1 status code definitions $Id: httpcodes.py,v 1.1 2003/11/15 17:32:53 ctheune Exp $""" http_codes = { 100 : (1, 'Continue'), 101 : (1, 'Switching Protocols'), 200 : (1, 'OK'), 201 : (1, 'Created'), 202 : (1, 'Accepted'), 203 : (1, 'Non-Authoritative Information'), 204 : (1, 'No Content'), 205 : (1, 'Reset Content'), 206 : (1, 'Partial Content'), 300 : (1, 'Multiple Choices'), 301 : (1, 'Moved Permanently'), 302 : (1, 'Found'), 303 : (1, 'See Other'), 304 : (1, 'Not Modified'), 305 : (1, 'Use Proxy'), 306 : (1, '(Unused)'), 307 : (1, 'Temporary Redirect'), 400 : (0, 'Bad Request'), 401 : (0, 'Unauthorized'), 402 : (0, 'Payment required'), 403 : (0, 'Forbidden'), 404 : (0, 'Not Found'), 405 : (0, 'Method Not Allowed'), 406 : (0, 'Not Acceptable'), 407 : (0, 'Proxy Authentication Required'), 408 : (0, 'Request Timeout'), 409 : (0, 'Conflict'), 410 : (0, 'Gone'), 411 : (0, 'Length Required'), 412 : (0, 'Precondition failed'), 413 : (0, 'Request Entity Too Large'), 414 : (0, 'Request-URI Too Long'), 415 : (0, 'Unsupported Media Type'), 416 : (0, 'Request Range Not Satisfiable'), 417 : (0, 'Expectation Failed'), 500 : (0, 'Internal Server Error'), 501 : (0, 'Not Implemented'), 502 : (0, 'Bad Gateway'), 503 : (0, 'Service Unavailable'), 504 : (0, 'Gateway Timeout'), 505 : (0, 'HTTP Version Not Supported') } CMFLinkChecker/interfaces.py0100640000076400001440000000564507755442371015666 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool interface definitions $Id: interfaces.py,v 1.2 2003/11/15 15:23:37 ctheune Exp $""" from Interface import Interface, Attribute class ILinkCheckerTool(Interface): maximumCacheAge = Attribute("Integer that defines the age of valid information in seconds.") gracefulUpdateAge = Attribute("Integer that defines the age when an update is desirable.") def registerRetriever(portal_type, function_ref): """Registers the function_ref to be called on objects of portal_type. The signature of function_ref must be accepting a single object which is the context. The return value is a list of ILinkInformation objects. """ def checkLink(url): """Checks the given URL. This result may be cached to allow intelligent load balancing for the blocking calls. Returns a boolean value which means true for a link that is online and false for a link that is offline.""" def showLinks(): """Show the information about the known links. Doesn't trigger updates or anything.""" def checkAllLinks(): """Schedules checks for all links. Takes all links an puts them in the queue.""" def findLinks(): """Find new links in the site. Finds new links from all objects that are retrievable and store them in the cache. No link check is done here.""" def setMaximumCacheAge(age): """Sets the maximum cache age.""" def setGracefulUpdateAge(age): """Sets the graceful update age. Assures that the graceful update age is lower than the maximum cache age.""" def setDefaultURIPrefix(defaultURIPrefix): """Set the default URI prefix that will be appended to relative URLs. If set to a value, it will prepend REQUEST.URL0.""" def activateWorkerThread(): """Activate the worker thread. Starts the worker thread (if it's not running already).""" class ILinkInformation(Interface): """Information about a link.""" owner = Attribute("Contains a CMFMember who is the owner of the object.") object = Attribute("Contains the object the link came from.") url = Attribute("Contains the URL the link refers to.") lastcheck = Attribute("DateTime when the last check was performed.") lasttime = Attribute("Time (in seconds) the last check took.") state = Attribute("A HTTP Response Code.") reason = Attribute("A verbose reason describing the error") headers = Attribute("Dictionary of the headers of the last response.") def __init__(object, owner, url): """Constructor.""" CMFLinkChecker/retrievers.py0100640000076400001440000000524107755454453015731 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool - link retriever functions $Id: retrievers.py,v 1.3 2003/11/15 16:49:47 ctheune Exp $""" from Products.CMFDefault.Link import Link from Products.CMFDefault.Document import Document from Products.CMFCalendar.Event import Event from StructuredText import Basic from StructuredText.DocumentClass import StructuredTextLink, DocumentClass from LinkInfo import LinkInfo import htmllib from sgmllib import SGMLParseError import formatter def retrieverLink(object): """Usable for CMFDefault.Link objects.""" assert isinstance(object, Link) info = LinkInfo() info.object = object.getPhysicalPath() info.setUrl(object.getRemoteUrl()) return [info] def retrieverEvent(object): """Usable for CMFDefault.Event objects.""" assert isinstance(object, Event) info = LinkInfo() info.object = object.getPhysicalPath() info.setUrl(object.event_url) return [info] def retrieveDocument(object): """Usable for CMFDefault.Document objects (in HTML and STX mode)""" assert isinstance(object, Document) if object.text_format == 'structured-text': links = retrieveSTX(object.text) links = [link.href for link in links] elif object.text_format == 'html': links = retrieveHTML(object.text) else: links = [] retrieved = [] for link in links: info = LinkInfo() info.object = object.getPhysicalPath() info.setUrl(link) retrieved.append(info) return retrieved def retrieveSTX(text): """Retrieve links from STX data.""" data = Basic(text) data = DocumentClass()(data) stack = data.getChildren() links = [] while stack: element = stack.pop() if isinstance(element, StructuredTextLink): links.append(element) try: stack.extend(element.getChildren()) except AttributeError: pass return links def retrieveHTML(text): parser = htmllib.HTMLParser(formatter.NullFormatter()) try: parser.feed(text) except SGMLParseError: # SGMLLib seems to die on bad HTML sometimes. (At least with python2.1) pass parser.close() return tuple(parser.anchorlist) # simple registries retrievers = { 'Link' : retrieverLink, 'Event' : retrieverEvent, 'Favorite' : retrieverLink, 'Document' : retrieveDocument, 'News Item' : retrieveDocument } CMFLinkChecker/tool.gif0100640000076400001440000000024607737470361014625 0ustar ctheuneusersGIF89almoxzx皜VVWabaKLJ!Made with GIMP! ,AI \+Ua@$ o#L(D  8 >|W;CMFLinkChecker/version.txt0100640000076400001440000000000407755462333015377 0ustar ctheuneusers0.2 CMFLinkChecker/doc/0040751000076400001440000000000007755461277013735 5ustar ctheuneusersCMFLinkChecker/doc/TODO.txt0100640000076400001440000000204607737470365015240 0ustar ctheuneusersBugs & Needs: - Make a number of threads start automatically - Follow 302 Ideas: - Get deeper into the non-blocking code. - Make the link checkers modular so we can have link checkers for ftp and other protocols as well - Make retrievers that gather links to images as well - Have a heartbeat thread that schedules checks automatically - Make it check invalid links more often - Make it check stuff thats has gone over the grace period - Have a management balanced score card - Invalid links per Author (current / over a period of time) - Invalid links per content section - Current invalid links - Save the availability history of a link - Have mailout-reports (daily/monthly) - sitewide - by author - Have report pages: - By object - By author - Sitewide - Make links in the html code tell the visitor that the target is currently down CMFLinkChecker/doc/CHANGES0100640000076400001440000000113607755461505014720 0ustar ctheuneusers HEAD - Some usability tweaks on the link list as proposed by Alexander Limi. - Fixed UnboundLocalError in Document retriever (#821984) - Fixed possible crash of checker thread when urlsplit didn't find a hostname - Bug 1503: Included a modified version for prepending a default element on URLs without hostname ("relative" links). Thanks to Sebastian Stark. - Updated documentation - Catching errors when parsing HTML now to avoid breaking the link discovery process. 0.1 - added support for HTML mode of CMF Default Documents CMFLinkChecker/doc/LICENSE0100640000076400001440000003545107737470365014745 0ustar ctheuneusers GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS CMFLinkChecker/doc/ROADMAP0100640000076400001440000000122307737470774014740 0ustar ctheuneusers1.0 - Stabilize 0.5 - Mark links (if possible) that they are unavailable 0.4 - Introduce balanced score cards a whole site 0.3 - Integrate with Plone UI for content editors - Add mail support when links go offline (Authors/Administrators) - refactor utility to be usable without CMF/Plone - Add UI for managing the content-/meta-type mapping to the link fetchers 0.2 - Stabilize / Check performance and blocking - Integrate rescheduling of the links - Use ZopeQueue for outfactored thread management (more stable, better trusted imho) - Add ftp checking support CMFLinkChecker/doc/Scheduling.txt0100640000076400001440000000370007737470365016556 0ustar ctheuneusersScheduling Meta:: Author: Christian Theune Valid for: CMF Link checker CVS: $Id: Scheduling.txt,v 1.1.1.1 2003/10/04 07:04:21 ctheune Exp $ Status: work in progress Describes a scheduling mechanism to avoid long locks. Reason Having a large site with thousands of references to external it can be a task with a long duration to check. Approach Therefore a threaded approach is used where each request for a link beeing checked is scheduled in a queue and run asynchronously. If a check is requested and there exists information about the availability that is not older than MAX_CACHE_AGE the information is returned and no scheduling is done. If the response is within the MAX_CACHE_AGE but out of the GRACEFUL_UPDATE_AGE then the result is returned and a new check is scheduled. If the response in the CACHE is older than MAX_CACHE_AGE the string "unknown" is returned and a check with a high priority is scheduled. If the "lock" parameter is given, the request for a link check will wait until a valid result has been retrieve Avoiding bandwith wastage Retrieving the whole HTTP Responses on URLS can be quite expensive. If the server is capable of the HEAD request this should be used. Otherwise a partial HTTP Response is to be accepted. Transfer should be stopped after getting all relevant headers. This also minimizes the risk of DOS casualities. Information stored The cache carries this information along: URL -- The full URL that is checked and used as a key on the cache LastCheck -- A DateTime object representing the point in time when the last check has been performed. State -- The result of the last check. Can be one of ["ok", "timeout", "network"] or an integer representing an HTTP Response Error code. Time -- Time the last check took Headers -- HTTP Response Headers CMFLinkChecker/www/0040751000076400001440000000000007755461277014014 5ustar ctheuneusersCMFLinkChecker/www/checkLinks.zpt0100640000076400001440000000510007755461505016614 0ustar ctheuneusersheader tabs
  Object Link State Last update Queue  
Group: unknown
asdf http:/asdfdsa/asdf minutes ago Check, Force
footer CMFLinkChecker/www/checker.zpt0100640000076400001440000000130207737470365016146 0ustar ctheuneusersheader tabs

Queue

Size:

Threading

threads running.
footer CMFLinkChecker/www/configureLinkcheckerTool.zpt0100640000076400001440000000152607755442372021531 0ustar ctheuneusersheader tabs

Configure portal_linkchecker Tool

Timeouts



Relative links


footer CMFLinkChecker/www/explainLinkcheckerTool.zpt0100640000076400001440000000117007737470366021207 0ustar ctheuneusersheader tabs

portal_linkchecker Tool

This tool provides a service to check all catalogued items for invalid links.

Following objects are automatically checked currently:

  • CMF/Plone Links
  • CMF/Plone Documents (in STX and HTML mode)
  • CMF/Plone News Items
  • CMF/Plone Events
  • CMF/Plone Favorites
footer