CMFLinkChecker/0040751000076400001440000000000007770127444013160 5ustar ctheuneusersCMFLinkChecker/Extensions/0040751000076400001440000000000007770127440015313 5ustar ctheuneusersCMFLinkChecker/Extensions/Install.py0100640000076400001440000000623307770126003017266 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool $Id: Install.py,v 1.3 2003/12/17 19:18:27 ctheune Exp $""" from Products.CMFLinkChecker.LinkCheckerTool import LinkCheckerTool from Products.CMFCore.utils import getToolByName from Products.CMFCore.DirectoryView import addDirectoryViews from Products.CMFLinkChecker import linkchecker_globals from StringIO import StringIO import string def install(self): out=StringIO() if not hasattr(self, "_isPortalRoot"): print >> out, "Must be installed in a CMF Site (read Plone)" return print >> out, 'Installing portal_linkchecker' self.portal_actions.manage_aproviders(apname="portal_linkchecker", add_provider=1, REQUEST=None) self.manage_addProduct['CMFLinkChecker'].manage_addTool(type='CMF Linkchecker Tool') self.portal_catalog.addIndex("portal_linkchecker_uid", "FieldIndex") self.left_slots = list(self.left_slots) + ['here/portlet_links/macros/portlet',] # XXX Register with form validation # Register configlet portal_conf=getToolByName(self,'portal_controlpanel') portal_conf.registerConfiglet( 'Link Checker' , 'CMFLinkChecker' , 'string:${portal_url}/lc_configlet' , '' # a condition , 'Manage portal' # access permission , 'Products' # section to which the configlet should be added: #(Plone,Products,Members) , 1 # visibility , 'CMFLinkChecker' , 'plone_images/site_icon.gif' # icon in control_panel, put your own icon in the # /skins folder of your product and change #'site_icon.gif' to 'yourfile' , 'An utility for monitoring external links in your site. Helps you to avoid dead links.' , None ) # Register skins addDirectoryViews(self.portal_skins, 'skins', linkchecker_globals) skins = self.portal_skins.getSkinSelections() for skin in skins: path = self.portal_skins.getSkinPath(skin) path = map(string.strip, path.split(',')) if 'linkchecker' not in path: try: path.insert(path.index('content'), 'linkchecker') except ValueError: path.append('linkchecker') path = ','.join(path) self.portal_skins.addSkinSelection(skin, path) print >> out, skin + ",", else: print >> out, "skipped" + skin, print >> out, 'Successfully installed CMFLinkChecker' return out.getvalue() def uninstall(self): portal_conf=getToolByName(self,'portal_controlpanel') portal_conf.unregisterConfiglet('CMFLinkChecker') portal_cat = getToolByName(self, 'portal_catalog') portal_cat.delIndex('portal_linkchecker_uid') CMFLinkChecker/Extensions/__init__.py0100640000076400001440000000003007755434061017415 0ustar ctheuneusers#poof, you're a product CMFLinkChecker/LinkCheckerTool.py0100640000076400001440000006215407770126003016545 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool $Id: LinkCheckerTool.py,v 1.6 2003/12/17 19:18:27 ctheune Exp $""" # Python stuff from types import StringTypes from md5 import md5 from random import shuffle # Global Zope stuff from DateTime import DateTime from AccessControl import ClassSecurityInfo from OFS.Folder import Folder from Globals import InitializeClass, PersistentMapping, Persistent from Products.CMFCore.utils import UniqueObject from Products.CMFCore.CMFCorePermissions import ManagePortal from Products.PageTemplates.PageTemplateFile import PageTemplateFile from Products.CMFCore.ActionProviderBase import ActionProviderBase from Products.CMFCore.ActionInformation import ActionInformation from Products.CMFCore.Expression import Expression from AccessControl import getSecurityManager # Sibling imports from interfaces import ILinkCheckerTool, ILinkInformation from LinkInfo import LinkInfo import retrievers import zLOG # Threading stuff from threading import Thread try: from threading import BoundedSemaphore except ImportError: from threading import Semaphore BoundedSemaphore = Semaphore import urllib, httplib import httpclient import Queue import time # Threading and transactions are a pain import Globals import OFS.Application import ZODB.ZApplication # Link checker stuff from httplib import ResponseNotReady from httpcodes import http_codes retrieval_queue = Queue.Queue(0) # Make this an infinite synchronized queue thread_stop = 0 active_threads = 5 thread_count = 0 pool_threads = BoundedSemaphore(value=active_threads) class TooManyThreadsRunning(Exception): pass class UnknownLink(Exception): pass class LinkCheckerTool(UniqueObject, ActionProviderBase, Folder): """A link checker tool. Encapsulating mechanisms for finding links in the site, testing their availability and notifying their owners.""" __implements__ = (ILinkCheckerTool, ) id = 'portal_linkchecker' meta_type = 'CMF Linkchecker Tool' security = ClassSecurityInfo() plone_tool = 1 manage_options = ( ({'label' : 'Overview', 'action' : 'manage_overview'},) + ActionProviderBase.manage_options + ({'label' : 'Configure', 'action' : 'manage_configure'}, {'label' : 'Linklist', 'action' : 'manage_checklinks'}, {'label' : 'Linkchecking', 'action' : 'manage_checker'},) + Folder.manage_options ) _actions = [ActionInformation(id='linkchecker_object_status', title='Links', action=Expression(text='string:${object_url}/lc_object_status'), category='object', visible=1), ActionInformation(id='linkchecker_member_overview', title='My Links', action=Expression(text='string:${portal_url}/lc_my_dead_links'), category='user', visible=1), ActionInformation(id='linkchecker_balanced_scorecard', title='Balanced Score Card: Links', category='portal_tabs', visible=1, action=Expression(text='string:${portal_url}/lc_balanced_score_card'))] maximumCacheAge = 60*60 # 1 hour gracefulUpdateAge = 60*45 # 45 minutes checkingPeriod = 60*5 # 5 minutes defaultURIPrefix = "" # No prefix by default # # ZMI methods # security.declareProtected(ManagePortal, 'manage_overview') manage_overview = PageTemplateFile('www/explainLinkcheckerTool', globals(), __name__='manage_overview') security.declareProtected(ManagePortal, 'manage_configure') manage_configure= PageTemplateFile('www/configureLinkcheckerTool', globals(), __name__='manage_configure') security.declareProtected(ManagePortal, 'manage_checklinks') manage_checklinks = PageTemplateFile('www/checkLinks', globals(), __name__='manage_checklinks') security.declareProtected(ManagePortal, 'manage_checker') manage_checker = PageTemplateFile('www/checker', globals(), __name__='manage_checker') security.declareProtected(ManagePortal, 'edit_configuration') def edit_configuration(self, maxage, graceage, defaultURIPrefix): """Update the configuration options.""" self.setMaximumCacheAge(maxage) self.setGracefulUpdateAge(graceage) self.setDefaultURIPrefix(defaultURIPrefix) if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_configure') security.declareProtected(ManagePortal, 'manage_checkLink') def manage_checkLink(self, id): """ZMI method for checking a link.""" if not self._cache.has_key(url): if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checklinks?manage_tabs_message=No+such+link') link = getattr(self, id) self.checkLink(link) if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checklinks?manage_tabs_message=Link+checked') def manage_forceCheckLink(self, url): """ZMI method for forcing the check of a link.""" if not self._cache.has_key(url): if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checklinks?manage_tabs_message=No+such+link') return link = getattr(self, id) self.__queueAddLink(link) if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checklinks?manage_tabs_message=Check+forced') security.declareProtected(ManagePortal, 'manage_checkAll') def manage_checkAll(self): """ZMI method for checking all links.""" self.checkAllLinks() if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect(self.absolute_url()+'/manage_checker?manage_tabs_message=All+links+checked+or+queued.') security.declareProtected(ManagePortal, 'manage_checkAll') def manage_forceCheckAll(self): """ZMI method for checking all links.""" self.forceCheckAllLinks() if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect(self.absolute_url()+'/manage_checker?manage_tabs_message=All+links+forced+into+que.') security.declareProtected(ManagePortal, 'manage_findLinks') def manage_findLinks(self): """ZMI method for finding new links.""" count = self.findLinks() if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect(self.absolute_url()+'/manage_checklinks?manage_tabs_message=%s+links+added.+%s+links+removed.' % count) security.declareProtected(ManagePortal, 'manage_activateWorkerThread') def manage_activateWorkerThread(self): """ZMI method for activating a new worker thread.""" try: self.activateWorkerThread() message = "Activated worker thread." except TooManyThreadsRunning: message = "Already %s running." % active_threads if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checker?manage_tabs_message=%s' % message) security.declareProtected(ManagePortal, 'manage_activateSchedulerThread') def manage_activateSchedulerThread(self): """ZMI method for activating a new scheduler thread.""" self.activateSchedulerThread() message = "Scheduler+activated" if hasattr(self.REQUEST, 'RESPONSE'): self.REQUEST.RESPONSE.redirect('manage_checker?manage_tabs_message=%s' % message) # # Interface methods # def __init__(self): self._retrievers = PersistentMapping() # register all global retrievers for type, retriever in retrievers.retrievers.items(): self.registerRetriever(type, retriever) security.declarePrivate("getLinkId") def getLinkId(self, url): """The ID is the md5 sum of the link.""" if url is None: import pdb; pdb.set_trace() return md5(url).hexdigest() security.declarePrivate("getInfoForURL") def getInfoForURL(self, url): """Get an info object for this url""" id = self.getLinkId(url) info = getattr(self, id, None) if info is None: info = LinkInfo(url) self._setObject(id, info) info = getattr(self, id) return info def manage_afterAdd(self, container, object): # The link catalog that manages links self.manage_addProduct['ZCatalog'].manage_addZCatalog('link_catalog', 'Link Catalog') self.link_catalog.addIndex('url', 'FieldIndex') self.link_catalog.addIndex('state', 'KeywordIndex') self.link_catalog.addIndex('objects', 'KeywordIndex') ########################################################################## ## UID methods security.declarePrivate('getUIDForObject') def getUIDForObject(self, object): try: return object.portal_linkchecker_uid except AttributeError: uid_a = object.getId() uid_b = "" uid = lambda: "%s-%s" % (uid_a, uid_b) while uid() in self.portal_catalog.\ Indexes['portal_linkchecker_uid'].uniqueValues(): uid_b = time.time() object.portal_linkchecker_uid = uid() self.portal_catalog.reindexObject(object) return object.portal_linkchecker_uid security.declarePrivate('getObjectForUID') def getObjectForUID(self, uid): results = self.portal_catalog(portal_linkchecker_uid=uid) if not results: return None else: return results[0].getObject() ## registry functions for link information retrieval security.declareProtected(ManagePortal, 'registerRetriever') def registerRetriever(self, portal_type, function_ref): assert not self._retrievers.has_key(portal_type), \ "Duplicate portal_type registration (%s)" % portal_type self._retrievers[portal_type] = function_ref def getRetrievableTypes(self): return self._retrievers.keys() def getRetriever(self, type): return self._retrievers[type] security.declareProtected(ManagePortal, 'checkLink') def checkLink(self, link): now = time.time() if link.lastcheck: age = link.lastcheck.millis() / 1000 else: age = 0 # Cache invalid? Requeue an update and return "unknown" if now - age > self.maximumCacheAge: self.__queueAddLink(link) return "grey" # Update required? if now - age > self.gracefulUpdateAge: self.__queueAddLink(link) return "grey" return link.state security.declareProtected(ManagePortal, 'checkAllLinks') def checkAllLinks(self): for link in self.objectValues(["Link Information"]): self.checkLink(link) security.declareProtected(ManagePortal, 'checkAllLinks') def forceCheckAllLinks(self): for link in self.objectValues(["Link Information"]): self.__queueAddLink(link) security.declarePrivate('clearLinks') def clearLinks(self): """Removes all links that aren't referenced anymore.""" to_delete = [] for x in self.objectValues(["Link Information"]): if len(x.objects) == 0: to_delete.append(x.getId()) self.manage_delObjects(to_delete) security.declareProtected(ManagePortal, 'findLinks') def findLinks(self): count = 0 links = [] # Clear all uid references for x in self.objectValues(["Link Information"]): x.clearObjects() # gather all objects that are of a type we can check for links for key in self.getRetrievableTypes(): objects = self.portal_catalog(portal_type=key) retriever = self.getRetriever(key) for ob in objects: ob = ob.getObject() # resolve catalog retriever(self, ob) # Delete old references self.clearLinks() return 0, 0 # XXX this might be wrong security.declareProtected('Modify portal content', 'findLinksForObject') def findLinksForObject(self, object): # Delete old references for this object links = self.link_catalog(objects=[self.getUIDForObject(object)]) for x in links: x = x.getObject() x.removeObject(object) # Update links type = object.Type() if not type in self.getRetrievableTypes(): return retriever = self.getRetriever(type) retriever(self, object) # Remove stale links self.clearLinks() security.declareProtected(ManagePortal, 'showLinks') def showLinks(self): """Show a grouped list of link data.""" data = self._cache.values() # group them all grouped = {} for x in data: group = x.getGroupId() if not grouped.has_key(group): grouped[group] = [] grouped[group].append(x) return grouped return self._cache.values() security.declareProtected(ManagePortal, 'setMaximumCacheAge') def setMaximumCacheAge(self, age): self.maximumCacheAge = age security.declareProtected(ManagePortal, 'setPeriod') def setPeriod(self, period): self.checkingPeriod = int(period) security.declareProtected(ManagePortal, 'setGracefulUpdateAge') def setGracefulUpdateAge(self, age): assert age < self.maximumCacheAge, \ "Graceful update age is larger than maximum cache age." self.gracefulUpdateAge = age security.declareProtected(ManagePortal, 'setDefaultURIPrefix') def setDefaultURIPrefix(self, defaultURIPrefix): assert type(defaultURIPrefix) in StringTypes self.defaultURIPrefix = defaultURIPrefix security.declareProtected(ManagePortal, 'queueSize') def queueSize(self): global retrieval_queue return retrieval_queue.qsize() security.declareProtected(ManagePortal, 'activateWorkerThread') def activateWorkerThread(self): if not pool_threads.acquire(0): raise TooManyThreadsRunning global thread_count thread_count += 1 thread = URLCheckerThread(retrieval_queue) thread.start() security.declareProtected(ManagePortal, 'activateSchedulerThread') def activateSchedulerThread(self): thread = LinkSchedulerThread(retrieval_queue, self.getPhysicalPath()) thread.start() security.declareProtected(ManagePortal, 'countWorkingThreads') def countWorkingThreads(self): global thread_count return thread_count ### Private methods security.declarePrivate('__queueAddLink') def __queueAddLink(self, info): """Schedules a new check. Info must be a ILinkInfo object.""" zLOG.LOG("CMFLinkChecker", zLOG.BLATHER, "Queue", "\tManually queueing %s\n" % info.url) retrieval_queue.put((self.getPhysicalPath(), info.getId())) info.queued = 1 ### UI convenience methods security.declarePublic("isManaged") def isManaged(self, object): """Returns true/false for objects if they can be checked.""" try: type = object.Type() except: return 0 return type in self.getRetrievableTypes() security.declareProtected("Modify portal content", "getLinksForObject") def getLinksForObject(self, object): """Return a list of link information objects that are valid for this object.""" links = self.link_catalog(objects=[self.getUIDForObject(object)]) links = [ x.getObject() for x in links ] return links security.declareProtected("View", "getLinksForAuthenticatedMember") def getGroupedLinksForAuthenticatedMember(self): """Returns a list of link information objects that are valid for the authenticated user.""" sm = getSecurityManager() user = sm.getUser() links = self.portal_catalog(Creator=user.getId()) uid = [ self.getUIDForObject(x.getObject()) for x in links ] links = self.link_catalog(uid=uid) links = [ x.getObject() for x in links ] links = [ x for x in links if x is not None ] groups = {'red':[], 'orange':[], 'grey':[], 'green':[]} for link in links: groups[link.state].append(link) return groups security.declareProtected("Manage portal", "getBalancedScoreCard") def getBalancedScoreCard(self): status = {} # Total amount of links (distinct) status['total_distinct'] = len(self.objectValues(["Link Information"])) # Total amount of objects status['total_documents'] = len(self.link_catalog.Indexes['objects'].uniqueValues()) # Documents per link values = [] for x in self.objectValues(["Link Information"]): values.append(len(x.objects)) status['documents_per_link_min'] = min(values) status['documents_per_link_max'] = max(values) status['documents_per_link_avg'] = "%.02f" % reduce(lambda x,y:(x+y)/2.0, values) # Total amount of links status['total'] = reduce(lambda x,y:x+y, values) # Links per document documents = {} for x in self.objectValues(["Link Information"]): for ob in x.objects: count = documents.get(ob, 0) count += 1 documents[ob] = count documents = documents.values() status['links_per_document_min'] = min(documents) status['links_per_document_max'] = max(documents) status['links_per_document_avg'] = "%.02f" % reduce(lambda x,y:(x+y)/2.0, documents) # Links per state (total) states = {'red':0, 'grey':0, 'green':0, 'orange':0} for x in self.objectValues(["Link Information"]): states[x.state] += 1 for x in states.keys(): status['total_%s_links' % x] = states[x] status['relative_%s_links' % x] = "%0.2f" % (float(states[x]) / status['total_distinct'] * 100) # Documents with state ... states = {'red': [], 'grey': [], 'green': [], 'orange': []} for x in self.objectValues(["Link Information"]): # We need to make it this complicated because we don't want to count documents # multiple times for y in x.objects: if y not in states[x.state]: states[x.state].append(y) for x in states.keys(): status['total_%s_documents' % x] = len(states[x]) status['relative_%s_documents' % x] = "%.02f" % (float(len(states[x])) / status['total_documents'] * 100) return status InitializeClass(LinkCheckerTool) #### Threading code. # We only use one worker thread. Object references # should be stable to work with. class LinkSchedulerThread(Thread): def __init__(self, queue, linkchecker): Thread.__init__(self, name="LinkScheduler") self.queue = queue self.linkchecker = linkchecker def run(self): zLOG.LOG('CMFLinkChecker', 0, 'Initialized', '\tScheduling thread has been initialized.\n') # calculate timing for scheduler check_app = None links = [] while 1: if check_app: get_transaction().abort() # We don't do write-requests anyway check_app._p_jar.sync() check_app._p_jar.close() check_app = None DB=Globals.DB app=ZODB.ZApplication.ZApplicationWrapper( DB, 'Application', OFS.Application.Application, (), Globals.VersionNameName) check_app=app() o = check_app # find the linkchecker object for p in self.linkchecker: if p: o=getattr(o,p) linkchecker = o if not links: # try to get a zodb connection period = linkchecker.checkingPeriod links = linkchecker.objectIds(["Link Information"]) shuffle(links) pause = period/float(len(links)) zLOG.LOG("CMFLinkChecker", zLOG.BLATHER, "Automatic Scheduler", "\tPause set to %s seconds\n" % pause) link = links.pop() link = getattr(linkchecker, link) zLOG.LOG("CMFLinkChecker", zLOG.BLATHER, "Automatic Scheduler", "\tQueuing %s\n" % link.url) self.queue.put((linkchecker.getPhysicalPath(), link.getId())) time.sleep(pause) class URLCheckerThread(Thread): def __init__(self, queue): Thread.__init__(self, name="URLChecker") self.queue = queue def run(self): zLOG.LOG('CMFLinkChecker', 0, 'Initialized', '\tWorker thread has been initialized.\n') # try to get a zodb connection check_app = None while 1: if check_app: get_transaction().commit() check_app._p_jar.sync() check_app._p_jar.close() check_app = None # we are doing cooperative multitasking # this thread doesn't need too much power time.sleep(0.1) try: linkchecker, key = self.queue.get(block=0) except Queue.Empty: time.sleep(1) continue # We got work, so setup the database connection. DB=Globals.DB app=ZODB.ZApplication.ZApplicationWrapper( DB, 'Application', OFS.Application.Application, (), Globals.VersionNameName) check_app=app() o = check_app # find the linkchecker object for p in linkchecker: if p: o=getattr(o,p) linkchecker = o try: item = getattr(linkchecker, key) except AttributeError: continue zLOG.LOG("CMFLinkChecker", zLOG.BLATHER, "Checker", "\tChecking %s\n" % item.url) # Links need to revalidate everytime otherwise they are in alert state. item.queued = 0 item.lastcheck = DateTime() # only check http links. other links are ... hmm # ... unknown ... type, url = urllib.splittype(item.url) if not type == 'http': item.state = 'grey' item.reason = "Protocol not supported by CMFLinkChecker" item.lasttime = 0 item.headers = {} continue # get a connection to the server host, path = urllib.splithost(url) if host is None: item.state = "red" item.reason = "Invalid Link: No hostname defined" continue host, port = urllib.splitnport(host, 80) conn = httpclient.myHTTPConnection(host, port, 2) starttime = time.time() # request the link try: conn.request('GET', path) except: import sys item.state = "red" item.reason = sys.exc_value[1] # this is http specific continue # look into the response response = None while not response: try: now = time.time() response = conn.getresponse() except ResponseNotReady: if starttime - now > 4: break except IOError: break time.sleep(0.2) conn.close() if response is not None: response.close() item.lasttime = now - starttime item.headers = {} # what if we didn't got a response? if not response: item.state = 'red' item.reason = 'unknown' continue if not http_codes.has_key(response.status): item.state = 'grey' item.reason = 'unknown HTTP response' continue item.state, item.reason = http_codes.get(response.status) zLOG.LOG('CMFLinkChecker', 0, 'Stopped', '\tWorker thread stopped.\n') global thread_count thread_count -= 1 CMFLinkChecker/LinkInfo.py0100640000076400001440000000650607770126003015235 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool - link information object $Id: LinkInfo.py,v 1.5 2003/12/17 19:18:27 ctheune Exp $""" from Globals import Persistent from Acquisition import Implicit from OFS.SimpleItem import SimpleItem from interfaces import ILinkInformation from urllib import urlencode from Products.CMFCore.utils import getToolByName from httpcodes import http_codes class LinkInfo(SimpleItem): meta_type = "Link Information" __implements__ = (ILinkInformation, ) __allow_access_to_unprotected_subobjects__ = 1 # XXX make accessor functions owner = None objects = [] url = None lastcheck = None lasttime = None queued = 0 state = "grey" headers = {} reason = "" def __init__(self, url): self.objects = [] self.setUrl(url) def _p_resolveConflict(self, oldState, savedState, newState): return newState def getId(self): return self.aq_parent.getLinkId(self.url) def title(self): return self.url def addObject(self, object): """Add this object to the object list.""" uid = self.portal_linkchecker.getUIDForObject(object) if not uid in self.objects: self.objects.append(uid) self._p_changed = 1 self.index() def removeObject(self, object): """Remove this uid from the object list.""" uid = self.portal_linkchecker.getUIDForObject(object) if uid in self.objects: self.objects.remove(uid) self._p_changed = 1 self.index() def clearObjects(self): """Remove all uid references from the object list.""" self.objects = [] self._p_changed = 1 self.index() def getObjects(self): """Return a reference to the object.""" objects = [ self.portal_linkchecker.getObjectForUID(object) for object in self.objects ] return objects def setUrl(self, url, REQUEST=None): """Set the URL. Sets the URL to the given value, trying to make relative paths absolute. (Requires a REQUEST object)""" self.url = url if REQUEST is None: return # Try to normalize relative links oldurl = self.url if self.url.startswith("/"): # Maybe we have a default prefix on the tool? lt = getToolByName(self, 'portal_linkchecker', None) self.url = getattr(lt, 'defaultURIPrefix', REQUEST.BASE0) + self.url elif not ':' in self.url: try: self.url = REQUEST.BASE0+'/'.join(self.object)+'/'+self.url except: pass self.index() def getGroupId(self): return self.state # Catalog support def manage_afterAdd(self, container, object): self.index() def manage_beforeDelete(self, item, container): self.unindex() def index(self): self.link_catalog.catalog_object(self, '/'.join(self.getPhysicalPath())) def unindex(self): self.link_catalog.uncatalog_object('/'.join(self.getPhysicalPath())) CMFLinkChecker/README.txt0100640000076400001440000000670207770122550014651 0ustar ctheuneusersCMFLinkchecker What is it? CMFLinkchecker is a CMF/Plone portal tool that can be used to monitor links within content of the portal. It is not used to monitor links that are generated by navigation code. It gathers links on demand automatically from different content types (Documents, News, Links ...) and puts them into it's monitoring list. It then tries to retrieve the object that is specified by the link (currently only HTTP-Links are supported) and remembers the status code. A colored and grouped list of all links of the portal gives you a detailed and high level overview on the status of the links on your site. How to install Extract the archive into your Zope's 'Products' directory and restart your Zope server. Easy way Use the "CMFQuickInstaller":http://sourceforge.net/projects/collective/CMFQuickInstaller to install it. Manual way Create an external method called "install_lc" in your CMF site. Module: CMFLinkChecker.Install Function: install Click the test button. Now you should have a "portal_linkchecker" tool within your CMF site. Additional steps To automatically register changes on objects in the link database, you need to add the "lc_updatelinks" script as a validator for the managed content types (e.g. in portal_properties/navigation_properties or as a CMFFormController override). Getting started Go within the ZMI into your linkchecker instance ('portal_linkchecker'). There are a couple of interesting tabs. Tab 1: Configure The 'maximum cache age' determines the period of time a result of a linkcheck is considered valid. The 'graceful update age' is the period of time that is still a valid result but will already be rechecked. The 'Default URI prefix' is added to relative links to determine a complete URL (e.g. add 'http://www.yourdomain.com:1234' here). Tab 2: Linklist The linklist gives you feedback about the links in your site and let's you gather new links. The "Check"-Links will put the link into the queue if there is no valid (either 'grey' or timed out) result for it currently. The "Force"-Links will put the link into the queue without considering the age of the current result. The "Find new links" button will ask the catalog of your site for contents of types that we have link fetching adapters for and update the list of links (remove stale and add new). Tab 3: Linkchecking This page gives an overview of the number of concurrent threads working on your checking queue as well as the current number of items in the queue. Using the 'Check all links' and 'Force all links' buttons you can 'Check' or 'Force' all links at once. (See description of 'Tab 2: Linklist'). By default you can activate up to 5 concurrent threads for checking links. What? Problems? If you encounter any problems or have an inquiry about the product, feel free to contact Christian Theune:"ct@gocept.com" or put a bug report in http://bugs.gocept.com. Any feedback is appreciated. Commercial support is available from gocept. Contact via "mail@gocept.com":mail@gocept.com or +49 3496 30 99 112. (Located in Germany, we speak german and english) CMFLinkChecker/__init__.py0100640000076400001440000000144407770122550015262 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool initalisation code $Id: __init__.py,v 1.2 2003/12/17 18:50:16 ctheune Exp $""" # Zope imports from Products.CMFCore import utils, DirectoryView # Linkchecker imports import LinkCheckerTool linkchecker_globals = globals() tools = (LinkCheckerTool.LinkCheckerTool, ) # Register skin directories DirectoryView.registerDirectory('skins', linkchecker_globals) def initialize(context): utils.ToolInit('CMFLinkChecker Tool', tools=tools, product_name='CMFLinkChecker', icon="tool.gif").initialize(context) CMFLinkChecker/httpclient.py0100640000076400001440000000271007737471213015704 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker customized http connection. (yet unused) $Id: httpclient.py,v 1.2 2003/10/04 07:11:07 ctheune Exp $""" import socket import httplib import time class TimeOut(Exception): pass myHTTPConnection = httplib.HTTPConnection class my2HTTPConnection(httplib.HTTPConnection): def __init__(self, host, port, timeout): httplib.HTTPConnection.__init__(self, host, port) self.timeout = timeout def send(self,str): if self.sock is None: if self.auto_open: self.connect() else: raise httplib.NotConnected() # that's the new code self.sock.setblocking(0) time1 = time.time() while 1: try: self.sock.sendall(str) except socket.error, v: if v[0] == 32: # Broken pipe self.close() raise if v[0] == 35: # Would block continue time.sleep(0.001) time2 = time.time() if time2-time1 > self.timeout: raise "Timeout", time2-time1 if self.debuglevel > 0: print "send:", repr(str) CMFLinkChecker/httpcodes.py0100640000076400001440000000476307770122550015527 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """HTTP 1.1 status code definitions $Id: httpcodes.py,v 1.2 2003/12/17 18:50:16 ctheune Exp $""" http_codes = { 100 : ('green', 'Continue'), 101 : ('green', 'Switching Protocols'), 200 : ('green', 'OK'), 201 : ('green', 'Created'), 202 : ('green', 'Accepted'), 203 : ('green', 'Non-Authoritative Information'), 204 : ('green', 'No Content'), 205 : ('green', 'Reset Content'), 206 : ('green', 'Partial Content'), 300 : ('green', 'Multiple Choices'), 301 : ('green', 'Moved Permanently'), 302 : ('green', 'Found'), 303 : ('green', 'See Other'), 304 : ('green', 'Not Modified'), 305 : ('green', 'Use Proxy'), 306 : ('green', '(Unused)'), 307 : ('green', 'Temporary Redirect'), 400 : ('red', 'Bad Request'), 401 : ('red', 'Unauthorized'), 402 : ('red', 'Payment required'), 403 : ('red', 'Forbidden'), 404 : ('red', 'Not Found'), 405 : ('red', 'Method Not Allowed'), 406 : ('red', 'Not Acceptable'), 407 : ('red', 'Proxy Authentication Required'), 408 : ('red', 'Request Timeout'), 409 : ('red', 'Conflict'), 410 : ('red', 'Gone'), 411 : ('red', 'Length Required'), 412 : ('red', 'Precondition failed'), 413 : ('red', 'Request Entity Too Large'), 414 : ('red', 'Request-URI Too Long'), 415 : ('red', 'Unsupported Media Type'), 416 : ('red', 'Request Range Not Satisfiable'), 417 : ('red', 'Expectation Failed'), 500 : ('orange', 'Internal Server Error'), 501 : ('red', 'Not Implemented'), 502 : ('red', 'Bad Gateway'), 503 : ('orange', 'Service Unavailable'), 504 : ('orange', 'Gateway Timeout'), 505 : ('red', 'HTTP Version Not Supported') } CMFLinkChecker/interfaces.py0100640000076400001440000000650007770122550015644 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool interface definitions $Id: interfaces.py,v 1.3 2003/12/17 18:50:16 ctheune Exp $""" from Interface import Interface, Attribute class ILinkCheckerTool(Interface): maximumCacheAge = Attribute("Integer that defines the age of valid information in seconds.") gracefulUpdateAge = Attribute("Integer that defines the age when an update is desirable.") def getLinkId(url): """Return an id that identifies this link within the linkchecker.""" def registerRetriever(portal_type, function_ref): """Registers the function_ref to be called on objects of portal_type. The signature of function_ref must be accepting a single object which is the context. The return value is a list of ILinkInformation objects. """ def checkLink(url): """Checks the given URL. This result may be cached to allow intelligent load balancing for the blocking calls. Returns a boolean value which means true for a link that is online and false for a link that is offline.""" def showLinks(): """Show the information about the known links. Doesn't trigger updates or anything.""" def checkAllLinks(): """Schedules checks for all links. Takes all links an puts them in the queue.""" def findLinks(): """Find new links in the site. Finds new links from all objects that are retrievable and store them in the cache. No link check is done here.""" def setMaximumCacheAge(age): """Sets the maximum cache age.""" def setGracefulUpdateAge(age): """Sets the graceful update age. Assures that the graceful update age is lower than the maximum cache age.""" def setDefaultURIPrefix(defaultURIPrefix): """Set the default URI prefix that will be appended to relative URLs. If set to a value, it will prepend REQUEST.URL0.""" def activateWorkerThread(): """Activate the worker thread. Starts the worker thread (if it's not running already).""" # Convenience methods for creating UI stuff def getLinksForObject(object): """Returns a list of ILinkInformation objects that correspond to the given object.""" class ILinkInformation(Interface): """Information about a link.""" objects = Attribute("Contains a list of UIDs for objects containing this link.") url = Attribute("Contains the URL the link refers to.") lastcheck = Attribute("DateTime when the last check was performed.") lasttime = Attribute("Time (in seconds) the last check took.") state = Attribute("A HTTP Response Code.") reason = Attribute("A verbose reason describing the error") headers = Attribute("Dictionary of the headers of the last response.") def addObject(object): """Add this uid to the object list.""" def removeObject(object): """Remove this uid from the object list.""" def getObjects(): """Returns a list of objects referencing this link.""" CMFLinkChecker/lchtmllib.py0100640000076400001440000000115307770122550015472 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """Customized HTML parser to find images in an imglist (similar to anchorlist) $Id: lchtmllib.py,v 1.1 2003/12/17 18:50:16 ctheune Exp $""" from htmllib import HTMLParser class LCHTMLParser(HTMLParser): imglist = [] def do_img(self, attrs): for attrname, value in attrs: if attrname == "src": self.imglist.append(value) CMFLinkChecker/retrievers.py0100640000076400001440000000540707770126003015715 0ustar ctheuneusers############################################################################## # # Copyright (c) 2003 gocept gmbh & co. kg. All rights reserved. # # See also LICENSE.txt # ############################################################################## """CMF link checker tool - link retriever functions $Id: retrievers.py,v 1.5 2003/12/17 19:18:27 ctheune Exp $""" from Products.CMFDefault.Link import Link from Products.CMFDefault.Document import Document from Products.CMFCalendar.Event import Event from StructuredText import Basic from StructuredText.DocumentClass import StructuredTextLink, DocumentClass from LinkInfo import LinkInfo import lchtmllib from sgmllib import SGMLParseError import formatter def retrieverLink(linkchecker, object): """Usable for CMFDefault.Link objects.""" assert isinstance(object, Link) url = object.getRemoteUrl() info = linkchecker.getInfoForURL(url) info.addObject(object) def retrieverEvent(linkchecker, object): """Usable for CMFDefault.Event objects.""" assert isinstance(object, Event) url = object.event_url info = linkchecker.getInfoForURL(url) info.addObject(object) def retrieveDocument(linkchecker, object): """Usable for CMFDefault.Document objects (in HTML and STX mode)""" assert isinstance(object, Document) # Handle document modes if object.text_format == 'structured-text': links = retrieveSTX(object.text) elif object.text_format == 'html': links = retrieveHTML(object.text) else: links = [] # Handle link registration for link in links: info = linkchecker.getInfoForURL(link) info.addObject(object) def retrieveSTX(text): """Retrieve links from STX data.""" data = Basic(text) data = DocumentClass()(data) stack = data.getChildren() links = [] while stack: element = stack.pop() if isinstance(element, StructuredTextLink): # Handle images seperately element = element.href if element.startswith('img:'): element = element[4:] links.append(element) try: stack.extend(element.getChildren()) except AttributeError: pass return links def retrieveHTML(text): parser = lchtmllib.LCHTMLParser(formatter.NullFormatter()) try: parser.feed(text) except SGMLParseError: # SGMLLib seems to die on bad HTML sometimes. (At least with python2.1) pass parser.close() return tuple(parser.anchorlist) + tuple(parser.imglist) # simple registries retrievers = { 'Link' : retrieverLink, 'Event' : retrieverEvent, 'Favorite' : retrieverLink, 'Document' : retrieveDocument, 'News Item' : retrieveDocument } CMFLinkChecker/tool.gif0100640000076400001440000000024607737470361014625 0ustar ctheuneusersGIF89almoxzx皜VVWabaKLJ!Made with GIMP! ,AI \+Ua@$ o#L(D  8 >|W;CMFLinkChecker/version.txt0100640000076400001440000000001007770127656015377 0ustar ctheuneusers0.3beta CMFLinkChecker/doc/0040751000076400001440000000000007770127440013721 5ustar ctheuneusersCMFLinkChecker/doc/STATES.txt0100640000076400001440000000136307770122550015462 0ustar ctheuneusers====================== Link State Definitions ====================== The following state names are defined forthe use within CMFLinkChecker. They are protocol independent. green Means everything is ok with this link, nothing to worry about The link will work as expected by the author. orange The link doesn't work but the response hints that this might be a temporary problem. (E.g. DNS resolution failures) red The link doesn't work and the response from the server hints that this is a hard problem (E.g. HTTP 404) grey There is no information about this link currently available. In addition to the link state there is a 'reason' field that stores (protocol dependent) information why the link is currently in the given state. CMFLinkChecker/doc/CHANGES0100640000076400001440000000273307770126003014710 0ustar ctheuneusers 0.2beta - Some code cleanup (legal stuff, coding style) - Debugging output available via zLOG.BLATHER - Introduced the automatic scheduler. You can start it by activating a scheduler thread in the ZMI (Link Checking). It assures to check every known link at least once within the given period of time as configured in the configlet. - Introduced "configlet" to configure the automatic scheduler and default URI prefix. - Introduced "balanced score card" that gives strategic information about the current state of links in the site. - Introduced "My (broken) Links" which gives an author an overview on all his broken links. - Using a UID to maintain stable references to the objects. - Using a ZCatalog to query LinkInfo objects. - Using an ObjectManager for the LinkInfo objects now. - Some usability tweaks on the link list as proposed by Alexander Limi. - Fixed UnboundLocalError in Document retriever (#821984) - Fixed possible crash of checker thread when urlsplit didn't find a hostname - Bug 1503: Included a modified version for prepending a default element on URLs without hostname ("relative" links). Thanks to Sebastian Stark. - Updated documentation - Catching errors when parsing HTML now to avoid breaking the link discovery process. 0.1 - added support for HTML mode of CMF Default Documents CMFLinkChecker/doc/LICENSE0100640000076400001440000003545107737470365014745 0ustar ctheuneusers GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS CMFLinkChecker/doc/ROADMAP0100640000076400001440000000263207770122550014724 0ustar ctheuneusersfuture - refactor utility to be usable without CMF/Plone - Use ZopeQueue for outfactored thread management (more stable, better trusted imho) 1.0 January 2004 - /F10/ Support more protocols - FTP support - HTTPS support - /F50/ Intime update of link lists - Add UI for managing the content-/meta-type mapping to the link fetchers - /F30/ Mail notifications on link status changes (Authors/Administrators) - Batched Mail notification - UI for user options - I18N support (incl. german translation) 0.3 December 2003 - /F60/ Plone 2.0 Integration - Plone 2.0 configlet - /F40/ Integrate with Plone UI for content editors - List of links and their status for a document (object) - List of all defect links for a document (object) as portlet - List of all defect links for the current user (user action "My links") - Balanced score card including statistics for the whole page - /F15/ Add support for checking references to images - /F20/ Regularly updates - Integration of a scheduling algorithm that guarantees every link to be checked within a fixed period of time - Stabilize / Check performance and blocking 0.2 November 2003 - Bug fixing CMFLinkChecker/doc/Scheduling.txt0100640000076400001440000000370007737470365016556 0ustar ctheuneusersScheduling Meta:: Author: Christian Theune Valid for: CMF Link checker CVS: $Id: Scheduling.txt,v 1.1.1.1 2003/10/04 07:04:21 ctheune Exp $ Status: work in progress Describes a scheduling mechanism to avoid long locks. Reason Having a large site with thousands of references to external it can be a task with a long duration to check. Approach Therefore a threaded approach is used where each request for a link beeing checked is scheduled in a queue and run asynchronously. If a check is requested and there exists information about the availability that is not older than MAX_CACHE_AGE the information is returned and no scheduling is done. If the response is within the MAX_CACHE_AGE but out of the GRACEFUL_UPDATE_AGE then the result is returned and a new check is scheduled. If the response in the CACHE is older than MAX_CACHE_AGE the string "unknown" is returned and a check with a high priority is scheduled. If the "lock" parameter is given, the request for a link check will wait until a valid result has been retrieve Avoiding bandwith wastage Retrieving the whole HTTP Responses on URLS can be quite expensive. If the server is capable of the HEAD request this should be used. Otherwise a partial HTTP Response is to be accepted. Transfer should be stopped after getting all relevant headers. This also minimizes the risk of DOS casualities. Information stored The cache carries this information along: URL -- The full URL that is checked and used as a key on the cache LastCheck -- A DateTime object representing the point in time when the last check has been performed. State -- The result of the last check. Can be one of ["ok", "timeout", "network"] or an integer representing an HTTP Response Error code. Time -- Time the last check took Headers -- HTTP Response Headers CMFLinkChecker/doc/TODO.txt0100640000076400001440000000100007770122550015210 0ustar ctheuneusersTODO ==== Bugs & Needs: - Make a number of threads start automatically - Follow 302 Ideas: - Get deeper into the non-blocking code. - Make the link checkers modular so we can have link checkers for ftp and other protocols as well - Save the availability history of a link - Have mailout-reports (daily/monthly) - sitewide - by author - Make links in the html code tell the visitor that the target is currently down CMFLinkChecker/skins/0040751000076400001440000000000007770127440014303 5ustar ctheuneusersCMFLinkChecker/skins/linkchecker/0040751000076400001440000000000007770127440016565 5ustar ctheuneusersCMFLinkChecker/skins/linkchecker/document_updatelinks.cpy0100640000076400001440000000037507770122550023522 0ustar ctheuneusers## Script (Python) "lc_updatelinks" ##bind container=container ##bind context=context ##bind namespace= ##bind script=script ##bind subpath=traverse_subpath ##parameters= ##title=Discussion parent breadcrumbs context.lc_updatelinks() return state CMFLinkChecker/skins/linkchecker/lc_balanced_score_card.pt0100640000076400001440000000753707770122550023533 0ustar ctheuneusers

Balanced score card: Links

This is the balanced score card for the links on your site. It gives you strategic information about the links on your site.

Total links

Total:
Distinct:

Total documents

Total:

Documents per link

Minimum:
Maximum:
Average:

Links per document

Minimum:
Maximum:
Average:

Links in the state ...

Red: (%)
Orange: (%)
Green: (%)
Grey: (%)

Documents containing links in the state ...

Red: (%)
Orange: (%)
Green: (%)
Grey: (%)
 CMFLinkChecker/skins/linkchecker/lc_configlet.pt0100640000076400001440000000657707770122550021573 0ustar ctheuneusers

Link Checker Setup

Link Checker setup for this portal.

Link Checker Details
The period of time after which all links should have been checked at least once.
seconds
This is used as a prefix for absolute links without hostname (e.g. /foo/bar).
Validation error output
CMFLinkChecker/skins/linkchecker/lc_configlet_set.py0100640000076400001440000000070407770122550022435 0ustar ctheuneusers## Script (Python) "prefs_mailhost_set" ##bind container=container ##bind context=context ##bind namespace= ##bind script=script ##bind subpath=traverse_subpath ##parameters=checkingPeriod, defaultURIPrefix ##title=set mailhost prefs ## context.portal_linkchecker.setPeriod(checkingPeriod) context.portal_linkchecker.setDefaultURIPrefix(defaultURIPrefix) context.REQUEST.RESPONSE.redirect('lc_configlet?portal_status_message=Changes+saved.') return CMFLinkChecker/skins/linkchecker/lc_my_dead_links.pt0100640000076400001440000000437307770122550022413 0ustar ctheuneusers

My broken links

Overview of all your broken links and image references.

Status red links

 Link   State   Description   Last update   Documents 
http://www.google.de Unknown Unknown Unknown
CMFLinkChecker/skins/linkchecker/lc_object_status.pt0100640000076400001440000000344307770122550022457 0ustar ctheuneusers

Link status overview

Overview on the status of links and images of this object.

 Link   State   Description   Last update 
http://www.google.de Unknown Unknown Unknown
CMFLinkChecker/skins/linkchecker/lc_updatelinks.py0100640000076400001440000000063407770122550022135 0ustar ctheuneusers## Script (Python) "lc_updatelinks" ##bind container=container ##bind context=context ##bind namespace= ##bind script=script ##bind subpath=traverse_subpath ##parameters= ##title=Discussion parent breadcrumbs lc = context.portal_linkchecker lc.findLinksForObject(context) for x in lc.getLinksForObject(context): lc.checkLink(x) context.REQUEST.RESPONSE.redirect(context.absolute_url()+'/lc_object_status') CMFLinkChecker/skins/linkchecker/portlet_links.pt0100640000076400001440000000153707770122550022023 0ustar ctheuneusers
Broken links
Refresh
CMFLinkChecker/www/0040751000076400001440000000000007770127440014000 5ustar ctheuneusersCMFLinkChecker/www/checkLinks.zpt0100640000076400001440000000512107770122551016607 0ustar ctheuneusersheader tabs
  Object Link State Last update Queue  
Group: unknown
asdf http:/asdfdsa/asdf minutes ago Check, Force
footer CMFLinkChecker/www/checker.zpt0100640000076400001440000000146007770122551016137 0ustar ctheuneusersheader tabs

Queue

Size:

Threading

threads running.
footer CMFLinkChecker/www/configureLinkcheckerTool.zpt0100640000076400001440000000152607755442372021531 0ustar ctheuneusersheader tabs

Configure portal_linkchecker Tool

Timeouts



Relative links


footer CMFLinkChecker/www/explainLinkcheckerTool.zpt0100640000076400001440000000117007737470366021207 0ustar ctheuneusersheader tabs

portal_linkchecker Tool

This tool provides a service to check all catalogued items for invalid links.

Following objects are automatically checked currently:

  • CMF/Plone Links
  • CMF/Plone Documents (in STX and HTML mode)
  • CMF/Plone News Items
  • CMF/Plone Events
  • CMF/Plone Favorites
footer