Ticket #207: Ticket207.patch

twistedcaldav/index.py

@@ -26 +26 @@
 __all__ = [
     "Index",
     "IndexSchedule",
+    "IndexedSearchException",
 ]
 
 import datetime
@@ -83 +84 @@
     which is not reserved.
     """
 
+class IndexedSearchException(ValueError):
+    pass
+
+
 class AbstractCalendarIndex(AbstractSQLDatabase):
     """
     Calendar collection index abstract base class that defines the apis for the index.
@@ -246 +251 @@
         statement += ")"
         results = self._db_values_for_sql(statement, *names)
         return results
-    
-    def searchValid(self, filter):
-        if isinstance(filter, caldavxml.Filter):
-            qualifiers = calendarquery.sqlcalendarquery(filter)
-        else:
-            qualifiers = None
-            
-        return qualifiers is not None
 
-    def search(self, filter):
+    def testAndUpdateIndex(self, mindate):
+        # Find out if the index is expanded far enough.
+        names = self.notExpandedBeyond(mindate)
+        # Actually expand recurrance max
+        for name in names:
+            self.reExpandResource(name,mindate)
+
+    def indexedSearch(self, filter):
         """
         Finds resources matching the given qualifiers.
         @param filter: the L{Filter} for the calendar-query to execute.
@@ -266 +270 @@
         """
         # FIXME: Don't forget to use maximum_future_expansion_duration when we
         # start caching...
-        
+
         # Make sure we have a proper Filter element and get the partial SQL statement to use.
         if isinstance(filter, caldavxml.Filter):
-            qualifiers = calendarquery.sqlcalendarquery(filter)
+            # Get a tuple of (SQL-qualifier, hint-to-update-index:date)
+            qualifier = calendarquery.sqlcalendarquery(filter)
+            if qualifier is not None:
+                sqlqualifiers = qualifier[0]
+                if qualifier[1] is not None:
+                    # bring index up to a a given date if it's not already
+                    self.testAndUpdateIndex(qualifier[1])
+            else:
+                # We cannot handle this filter in an indexed search
+                raise IndexedSearchException()
         else:
-            qualifiers = None
-        if qualifiers is not None:
-            rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + qualifiers[0], *qualifiers[1])
+            sqlqualifiers = None
+
+        # Perform the search
+        if sqlqualifiers is not None:
+            rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + sqlqualifiers[0],
+                                       *sqlqualifiers[1])
         else:
-            rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
-            
+            raise IndexedSearchException()
+
+        # Check result for missing resources
+        for row in rowiter:
+            name = row[0]
+            if self.resource.getChild(name.encode("utf-8")):
+                yield row
+            else:
+                log.err("Calendar resource %s is missing from %s. Removing from index."
+                        % (name, self.resource))
+                self.deleteResource(name)
+
+    def bruteforceSearch(self):
+        """
+        List the whole index and tests for existance, updating the index.
+        @return: all resources in the index.
+        """
+        # List all resources
+        rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
+
+        # Check result for missing resources
         for row in rowiter:
             name = row[0]
             if self.resource.getChild(name.encode("utf-8")):
@@ -393 +428 @@
                 """
             )
 
-    def _add_to_db(self, name, calendar, cursor = None):
+    def notExpandedBeyond(self, mindate):
+        """
+        Gives all resources which has not been expanded beyond a given date in the index
+        """
+        return self._db_values_for_sql("select NAME from RESOURCE where RECURRANCE_MAX <= :1", mindate)
+
+    def reExpandResource(self,name,until_date):
+        """
+        Given a resource name, remove it from the database and re-add it
+        with a longer expansion.
+        """
+        calendar = self.resource.getChild(name).iCalendar()
+
+        oldUID = self.resourceUIDForName(name)
+        if oldUID is not None:
+            self._delete_from_db(name, oldUID)
+        self._add_to_db(name, calendar,until_date)
+        self._db_commit()
+
+
+    def _add_to_db(self, name, calendar, expand_wish = None, cursor = None):
         """
         Records the given calendar resource in the index with the given name.
         Resource names and UIDs must both be unique; only one resource name may
@@ -406 +461 @@
         """
         uid = calendar.resourceUID()
 
-        expand_max = datetime.date.today() + default_future_expansion_duration
+        expand_default = datetime.date.today() + default_future_expansion_duration
+        expand_max     = datetime.date.today() + maximum_future_expansion_duration
+
+        expand = expand_default
+        if expand_wish is not None:
+            if expand_wish > expand_default:
+                expand = expand_wish
+            if expand > expand_max:
+                # Will not handle
+                raise IndexedSearchException
 
-        instances = calendar.expandTimeRanges(expand_max)
+        instances = calendar.expandTimeRanges(expand)
         for key in instances:
             instance = instances[key]
             start = instance.start.replace(tzinfo=utc)

twistedcaldav/method/report_calquery.py

@@ -35 +35 @@
 from twistedcaldav.caldavxml import caldav_namespace
 from twistedcaldav.customxml import TwistedCalendarAccessProperty
 from twistedcaldav.method import report_common
+from twistedcaldav.index import IndexedSearchException
 
 import urllib
 
@@ -166 +167 @@
 
             # Check for disabled access
             if filteredaces is not None:
-                # See whether the filter is valid for an index only query
-                index_query_ok = calresource.index().searchValid(filter)
-            
-                # Get list of children that match the search and have read access
-                names = [name for name, ignore_uid, ignore_type in calresource.index().search(filter)]
+                # Variable to remember whether the filter is valid for an index only query
+                index_query_ok = None
+                try:
+                    # Get list of children that match the search and have read access
+                    names = [name for name, ignore_uid, ignore_type in calresource.index().indexedSearch(filter)]
+                    index_query_ok = True
+                except IndexedSearchException:
+                    names = [name for name, ignore_uid, ignore_type in calresource.index().bruteforceSearch()]
+                    index_query_ok = False
+
                 if not names:
                     yield None
                     return
-                  
+
                 # Now determine which valid resources are readable and which are not
                 ok_resources = []
                 d = calresource.findChildrenFaster(
@@ -189 +195 @@
                 x = waitForDeferred(d)
                 yield x
                 x.getResult()
-                
+
                 for child, child_uri in ok_resources:
                     child_uri_name = child_uri[child_uri.rfind("/") + 1:]
                     child_path_name = urllib.unquote(child_uri_name)

twistedcaldav/method/report_common.py

@@ -47 +47 @@
 from twistedcaldav.dateops import clipPeriod, normalizePeriodList, timeRangesOverlap
 from twistedcaldav.ical import Component, Property, iCalendarProductID
 from twistedcaldav.instance import InstanceList
+from twistedcaldav.index import IndexedSearchException
 
 from vobject.icalendar import utc
 
@@ -346 +347 @@
     yield filteredaces
     filteredaces = filteredaces.getResult()
 
-    for name, uid, type in calresource.index().search(filter): #@UnusedVariable
+    try:
+        resources = calresource.index().indexedSearch(filter)
+    except IndexedSearchException:
+        resources = calresource.index().bruteforceSearch()
+
+    for name, uid, type in resources: #@UnusedVariable
         
         # Check privileges - must have at least CalDAV:read-free-busy
         child = waitForDeferred(request.locateChildResource(calresource, name))

twistedcaldav/query/calendarquery.py

@@ -24 +24 @@
 __version__ = "0.0"
 
 __all__ = [
-    "calendarquery",
+    "CalendarQueryExpressionGenerator",
     "sqlcalendarquery",
 ]
 
@@ -42 +42 @@
 #FIELD_DESCRIPTION = "RESOURCE.DESCRIPTION"
 #FIELD_LOCATION = "RESOURCE.LOCATION"
 
-def calendarquery(filter):
+class CalendarQueryExpressionGenerator(object):
     """
-    Convert the supplied calendar-query into an expression tree.
-
-    @param filter: the L{Filter} for thw calendar-query to convert.
-    @return: a L{baseExpression} for the expression tree.
+    Can parse CalDAV XML filters to expression structures and
+    Hold information about the expression useful for clients
     """
-    
-    # Lets assume we have a valid filter from the outset.
-    
-    # Top-level filter contains exactly one comp-filter element
-    assert len(filter.children) == 1
-    vcalfilter = filter.children[0]
-    assert isinstance(vcalfilter, caldavxml.ComponentFilter)
-    assert vcalfilter.filter_name == "VCALENDAR"
-    
-    if len(vcalfilter.children) > 0:
-        # Only comp-filters are handled
-        for _ignore in [x for x in vcalfilter.children if not isinstance(x, caldavxml.ComponentFilter)]:
+
+    def __init__(self):
+        self.enddate = None
+
+    def calendarquery(self, filter):
+        """
+        Convert the supplied calendar-query into an expression tree.
+
+        @param filter: the L{Filter} for thw calendar-query to convert.
+        @return: a L{baseExpression} for the expression tree.
+        """
+
+        # Lets assume we have a valid filter from the outset.
+
+        # Top-level filter contains exactly one comp-filter element
+        assert len(filter.children) == 1
+        vcalfilter = filter.children[0]
+        assert isinstance(vcalfilter, caldavxml.ComponentFilter)
+        assert vcalfilter.filter_name == "VCALENDAR"
+
+        if len(vcalfilter.children) > 0:
+            # Only comp-filters are handled
+            for _ignore in [x for x in vcalfilter.children if not isinstance(x, caldavxml.ComponentFilter)]:
+                raise ValueError
+
+            return self.compfilterListExpression(vcalfilter.children)
+        else:
+            return expression.allExpression()
+
+    def compfilterListExpression(self, compfilters):
+        """
+        Create an expression for a list of comp-filter elements.
+
+        @param compfilters: the C{list} of L{ComponentFilter} elements.
+        @return: a L{baseExpression} for the expression tree.
+        """
+
+        if len(compfilters) == 1:
+            return self.compfilterExpression(compfilters[0])
+        else:
+            return expression.orExpression([self.compfilterExpression(c) for c in compfilters])
+
+    def compfilterExpression(self, compfilter):
+        """
+        Create an expression for a single comp-filter element.
+
+        @param compfilter: the L{ComponentFilter} element.
+        @return: a L{baseExpression} for the expression tree.
+        """
+
+        # Handle is-not-defined case
+        if not compfilter.defined:
+            # Test for TYPE != <<component-type name>>
+            return expression.isnotExpression(FIELD_TYPE, compfilter.filter_name, True)
+
+        expressions = []
+        if isinstance(compfilter.filter_name, str):
+            expressions.append(expression.isExpression(FIELD_TYPE, compfilter.filter_name, True))
+        else:
+            expressions.append(expression.inExpression(FIELD_TYPE, compfilter.filter_name, True))
+
+        # Handle time-range
+        if compfilter.qualifier and isinstance(compfilter.qualifier, caldavxml.TimeRange):
+            start, end, startfloat, endfloat = getTimerangeArguments(compfilter.qualifier)
+            expressions.append(expression.timerangeExpression(start, end, startfloat, endfloat))
+            self.enddate = compfilter.qualifier.end.date()
+
+        # Handle properties - we can only do UID right now
+        props = []
+        for p in [x for x in compfilter.filters if isinstance(x, caldavxml.PropertyFilter)]:
+            props.append(self.propfilterExpression(p))
+        if len(props) > 1:
+            propsExpression = expression.orExpression[props]
+        elif len(props) == 1:
+            propsExpression = props[0]
+        else:
+            propsExpression = None
+
+        # Handle embedded components - we do not right now as our Index does not handle them
+        comps = []
+        for _ignore in [x for x in compfilter.filters if isinstance(x, caldavxml.ComponentFilter)]:
             raise ValueError
-        
-        return compfilterListExpression(vcalfilter.children)
-    else:
-        return expression.allExpression()
+        if len(comps) > 1:
+            compsExpression = expression.orExpression[comps]
+        elif len(comps) == 1:
+            compsExpression = comps[0]
+        else:
+            compsExpression = None
 
-def compfilterListExpression(compfilters):
-    """
-    Create an expression for a list of comp-filter elements.
-    
-    @param compfilters: the C{list} of L{ComponentFilter} elements.
-    @return: a L{baseExpression} for the expression tree.
-    """
-    
-    if len(compfilters) == 1:
-        return compfilterExpression(compfilters[0])
-    else:
-        return expression.orExpression([compfilterExpression(c) for c in compfilters])
+        # Now build compound expression
+        if ((propsExpression is not None) and (compsExpression is not None)):
+            expressions.append(expression.orExpression([propsExpression, compsExpression]))
+        elif propsExpression is not None:
+            expressions.append(propsExpression)
+        elif compsExpression is not None:
+            expressions.append(compsExpression)
+
+        # Now build return expression
+        return expression.andExpression(expressions)
+
+    def propfilterExpression(self, propfilter):
+        """
+        Create an expression for a single prop-filter element.
+
+        @param propfilter: the L{PropertyFilter} element.
+        @return: a L{baseExpression} for the expression tree.
+        """
 
-def compfilterExpression(compfilter):
-    """
-    Create an expression for a single comp-filter element.
-    
-    @param compfilter: the L{ComponentFilter} element.
-    @return: a L{baseExpression} for the expression tree.
-    """
-    
-    # Handle is-not-defined case
-    if not compfilter.defined:
-        # Test for TYPE != <<component-type name>>
-        return expression.isnotExpression(FIELD_TYPE, compfilter.filter_name, True)
-        
-    expressions = []
-    if isinstance(compfilter.filter_name, str):
-        expressions.append(expression.isExpression(FIELD_TYPE, compfilter.filter_name, True))
-    else:
-        expressions.append(expression.inExpression(FIELD_TYPE, compfilter.filter_name, True))
-    
-    # Handle time-range    
-    if compfilter.qualifier and isinstance(compfilter.qualifier, caldavxml.TimeRange):
-        start, end, startfloat, endfloat = getTimerangeArguments(compfilter.qualifier)
-        expressions.append(expression.timerangeExpression(start, end, startfloat, endfloat))
-        
-    # Handle properties - we can only do UID right now
-    props = []
-    for p in [x for x in compfilter.filters if isinstance(x, caldavxml.PropertyFilter)]:
-        props.append(propfilterExpression(p))
-    if len(props) > 1:
-        propsExpression = expression.orExpression[props]
-    elif len(props) == 1:
-        propsExpression = props[0]
-    else:
-        propsExpression = None
-        
-    # Handle embedded components - we do not right now as our Index does not handle them
-    comps = []
-    for _ignore in [x for x in compfilter.filters if isinstance(x, caldavxml.ComponentFilter)]:
-        raise ValueError
-    if len(comps) > 1:
-        compsExpression = expression.orExpression[comps]
-    elif len(comps) == 1:
-        compsExpression = comps[0]
-    else:
-        compsExpression = None
-
-    # Now build compound expression
-    if ((propsExpression is not None) and (compsExpression is not None)):
-        expressions.append(expression.orExpression([propsExpression, compsExpression]))
-    elif propsExpression is not None:
-        expressions.append(propsExpression)
-    elif compsExpression is not None:
-        expressions.append(compsExpression)
+        # Only handle UID right now
+        if propfilter.filter_name != "UID":
+            raise ValueError
 
-    # Now build return expression
-    return expression.andExpression(expressions)
+        # Handle is-not-defined case
+        if not propfilter.defined:
+            # Test for <<field>> != "*"
+            return expression.isExpression(FIELD_UID, "", True)
 
-def propfilterExpression(propfilter):
-    """
-    Create an expression for a single prop-filter element.
-    
-    @param propfilter: the L{PropertyFilter} element.
-    @return: a L{baseExpression} for the expression tree.
-    """
-    
-    # Only handle UID right now
-    if propfilter.filter_name != "UID":
-        raise ValueError
-
-    # Handle is-not-defined case
-    if not propfilter.defined:
-        # Test for <<field>> != "*"
-        return expression.isExpression(FIELD_UID, "", True)
-    
-    # Handle time-range - we cannot do this with our Index right now
-    if propfilter.qualifier and isinstance(propfilter.qualifier, caldavxml.TimeRange):
-        raise ValueError
-    
-    # Handle text-match
-    tm = None
-    if propfilter.qualifier and isinstance(propfilter.qualifier, caldavxml.TextMatch):
-        if propfilter.qualifier.negate:
-            tm = expression.notcontainsExpression(propfilter.filter_name, str(propfilter.qualifier), propfilter.qualifier)
+        # Handle time-range - we cannot do this with our Index right now
+        if propfilter.qualifier and isinstance(propfilter.qualifier, caldavxml.TimeRange):
+            raise ValueError
+
+        # Handle text-match
+        tm = None
+        if propfilter.qualifier and isinstance(propfilter.qualifier, caldavxml.TextMatch):
+            if propfilter.qualifier.negate:
+                tm = expression.notcontainsExpression(propfilter.filter_name, str(propfilter.qualifier), propfilter.qualifier)
+            else:
+                tm = expression.containsExpression(propfilter.filter_name, str(propfilter.qualifier), propfilter.qualifier)
+
+        # Handle embedded parameters - we do not right now as our Index does not handle them
+        params = []
+        for _ignore in propfilter.filters:
+            raise ValueError
+        if len(params) > 1:
+            paramsExpression = expression.orExpression[params]
+        elif len(params) == 1:
+            paramsExpression = params[0]
         else:
-            tm = expression.containsExpression(propfilter.filter_name, str(propfilter.qualifier), propfilter.qualifier)
-    
-    # Handle embedded parameters - we do not right now as our Index does not handle them
-    params = []
-    for _ignore in propfilter.filters:
-        raise ValueError
-    if len(params) > 1:
-        paramsExpression = expression.orExpression[params]
-    elif len(params) == 1:
-        paramsExpression = params[0]
-    else:
-        paramsExpression = None
-
-    # Now build return expression
-    if (tm is not None) and (paramsExpression is not None):
-        return expression.andExpression([tm, paramsExpression])
-    elif tm is not None:
-        return tm
-    elif paramsExpression is not None:
-        return paramsExpression
-    else:
-        return None
+            paramsExpression = None
+
+        # Now build return expression
+        if (tm is not None) and (paramsExpression is not None):
+            return expression.andExpression([tm, paramsExpression])
+        elif tm is not None:
+            return tm
+        elif paramsExpression is not None:
+            return paramsExpression
+        else:
+            return None
 
+# Utility function
 def getTimerangeArguments(timerange):
     """
     Get start/end and floating start/end (adjusted for timezone offset) values from the
@@ -209 +220 @@
 
     return str(start), str(end), str(startfloat), str(endfloat)
 
+# return SQL qualifiers and info to index.
 def sqlcalendarquery(filter):
     """
-    Convert the supplied calendar-query into a oartial SQL statement.
+    Convert the supplied calendar-query into a partial SQL statement.
 
-    @param filter: the L{Filter} for thw calendar-query to convert.
+    @param filter: the L{Filter} for the calendar-query to convert.
     @return: a C{tuple} of (C{str}, C{list}), where the C{str} is the partial SQL statement,
             and the C{list} is the list of argument substitutions to use with the SQL API execute method.
             Or return C{None} if it is not possible to create an SQL query to fully match the calendar-query.
     """
     try:
-        expression = calendarquery(filter)
+        expression_generator = CalendarQueryExpressionGenerator()
+        expression = expression_generator.calendarquery(filter)
         sql = sqlgenerator.sqlgenerator(expression)
-        return sql.generate()
+        qualifiers = (sql.generate(),expression_generator.enddate)
+        return qualifiers
     except ValueError:
+        # Indicate an unsupported filter
         return None
 
 

twistedcaldav/static.py

@@ -187 +187 @@
             filteredaces = filteredaces.getResult()
 
             # Must verify ACLs which means we need a request object at this point
-            for name, uid, type in self.index().search(None): #@UnusedVariable
+            for name, uid, type in self.index().bruteforceSearch(None): #@UnusedVariable
                 try:
                     child = waitForDeferred(request.locateChildResource(self, name))
                     yield child