diff -x '*.svn' -x '*.pyc' -ru ./twistedcaldav/index.py ../../DCS1.2/CalendarServer-1.2/twistedcaldav/index.py --- ./twistedcaldav/index.py 2008-08-20 10:30:02.000000000 +0200 +++ ../../DCS1.2/CalendarServer-1.2/twistedcaldav/index.py 2008-08-22 16:48:11.000000000 +0200 @@ -26,6 +26,7 @@ __all__ = [ "Index", "IndexSchedule", + "IndexedSearchException", ] import datetime @@ -83,6 +84,10 @@ which is not reserved. """ +class IndexedSearchException(ValueError): + pass + + class AbstractCalendarIndex(AbstractSQLDatabase): """ Calendar collection index abstract base class that defines the apis for the index. @@ -246,16 +251,15 @@ statement += ")" results = self._db_values_for_sql(statement, *names) return results - - def searchValid(self, filter): - if isinstance(filter, caldavxml.Filter): - qualifiers = calendarquery.sqlcalendarquery(filter) - else: - qualifiers = None - - return qualifiers is not None - def search(self, filter): + def testAndUpdateIndex(self, mindate): + # Find out if the index is expanded far enough. + names = self.notExpandedBeyond(mindate) + # Actually expand recurrance max + for name in names: + self.reExpandResource(name,mindate) + + def indexedSearch(self, filter): """ Finds resources matching the given qualifiers. @param filter: the L{Filter} for the calendar-query to execute. @@ -266,17 +270,48 @@ """ # FIXME: Don't forget to use maximum_future_expansion_duration when we # start caching... - + # Make sure we have a proper Filter element and get the partial SQL statement to use. if isinstance(filter, caldavxml.Filter): - qualifiers = calendarquery.sqlcalendarquery(filter) + # Get a tuple of (SQL-qualifier, hint-to-update-index:date) + qualifier = calendarquery.sqlcalendarquery(filter) + if qualifier is not None: + sqlqualifiers = qualifier[0] + if qualifier[1] is not None: + # bring index up to a a given date if it's not already + self.testAndUpdateIndex(qualifier[1]) + else: + # We cannot handle this filter in an indexed search + raise IndexedSearchException() else: - qualifiers = None - if qualifiers is not None: - rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + qualifiers[0], *qualifiers[1]) + sqlqualifiers = None + + # Perform the search + if sqlqualifiers is not None: + rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + sqlqualifiers[0], + *sqlqualifiers[1]) else: - rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE") - + raise IndexedSearchException() + + # Check result for missing resources + for row in rowiter: + name = row[0] + if self.resource.getChild(name.encode("utf-8")): + yield row + else: + log.err("Calendar resource %s is missing from %s. Removing from index." + % (name, self.resource)) + self.deleteResource(name) + + def bruteforceSearch(self): + """ + List the whole index and tests for existance, updating the index. + @return: all resources in the index. + """ + # List all resources + rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE") + + # Check result for missing resources for row in rowiter: name = row[0] if self.resource.getChild(name.encode("utf-8")): @@ -393,7 +428,27 @@ """ ) - def _add_to_db(self, name, calendar, cursor = None): + def notExpandedBeyond(self, mindate): + """ + Gives all resources which has not been expanded beyond a given date in the index + """ + return self._db_values_for_sql("select NAME from RESOURCE where RECURRANCE_MAX <= :1", mindate) + + def reExpandResource(self,name,until_date): + """ + Given a resource name, remove it from the database and re-add it + with a longer expansion. + """ + calendar = self.resource.getChild(name).iCalendar() + + oldUID = self.resourceUIDForName(name) + if oldUID is not None: + self._delete_from_db(name, oldUID) + self._add_to_db(name, calendar,until_date) + self._db_commit() + + + def _add_to_db(self, name, calendar, expand_wish = None, cursor = None): """ Records the given calendar resource in the index with the given name. Resource names and UIDs must both be unique; only one resource name may @@ -406,9 +461,18 @@ """ uid = calendar.resourceUID() - expand_max = datetime.date.today() + default_future_expansion_duration + expand_default = datetime.date.today() + default_future_expansion_duration + expand_max = datetime.date.today() + maximum_future_expansion_duration + + expand = expand_default + if expand_wish is not None: + if expand_wish > expand_default: + expand = expand_wish + if expand > expand_max: + # Will not handle + raise IndexedSearchException - instances = calendar.expandTimeRanges(expand_max) + instances = calendar.expandTimeRanges(expand) for key in instances: instance = instances[key] start = instance.start.replace(tzinfo=utc) diff -x '*.svn' -x '*.pyc' -ru ./twistedcaldav/method/report_calquery.py ../../DCS1.2/CalendarServer-1.2/twistedcaldav/method/report_calquery.py --- ./twistedcaldav/method/report_calquery.py 2008-08-20 10:29:59.000000000 +0200 +++ ../../DCS1.2/CalendarServer-1.2/twistedcaldav/method/report_calquery.py 2008-08-21 17:56:25.000000000 +0200 @@ -35,6 +35,7 @@ from twistedcaldav.caldavxml import caldav_namespace from twistedcaldav.customxml import TwistedCalendarAccessProperty from twistedcaldav.method import report_common +from twistedcaldav.index import IndexedSearchException import urllib @@ -166,15 +167,20 @@ # Check for disabled access if filteredaces is not None: - # See whether the filter is valid for an index only query - index_query_ok = calresource.index().searchValid(filter) - - # Get list of children that match the search and have read access - names = [name for name, ignore_uid, ignore_type in calresource.index().search(filter)] + # Variable to remember whether the filter is valid for an index only query + index_query_ok = None + try: + # Get list of children that match the search and have read access + names = [name for name, ignore_uid, ignore_type in calresource.index().indexedSearch(filter)] + index_query_ok = True + except IndexedSearchException: + names = [name for name, ignore_uid, ignore_type in calresource.index().bruteforceSearch()] + index_query_ok = False + if not names: yield None return - + # Now determine which valid resources are readable and which are not ok_resources = [] d = calresource.findChildrenFaster( @@ -189,7 +195,7 @@ x = waitForDeferred(d) yield x x.getResult() - + for child, child_uri in ok_resources: child_uri_name = child_uri[child_uri.rfind("/") + 1:] child_path_name = urllib.unquote(child_uri_name) diff -x '*.svn' -x '*.pyc' -ru ./twistedcaldav/method/report_common.py ../../DCS1.2/CalendarServer-1.2/twistedcaldav/method/report_common.py --- ./twistedcaldav/method/report_common.py 2008-08-20 10:29:59.000000000 +0200 +++ ../../DCS1.2/CalendarServer-1.2/twistedcaldav/method/report_common.py 2008-08-21 17:55:18.000000000 +0200 @@ -47,6 +47,7 @@ from twistedcaldav.dateops import clipPeriod, normalizePeriodList, timeRangesOverlap from twistedcaldav.ical import Component, Property, iCalendarProductID from twistedcaldav.instance import InstanceList +from twistedcaldav.index import IndexedSearchException from vobject.icalendar import utc @@ -346,7 +347,12 @@ yield filteredaces filteredaces = filteredaces.getResult() - for name, uid, type in calresource.index().search(filter): #@UnusedVariable + try: + resources = calresource.index().indexedSearch(filter) + except IndexedSearchException: + resources = calresource.index().bruteforceSearch() + + for name, uid, type in resources: #@UnusedVariable # Check privileges - must have at least CalDAV:read-free-busy child = waitForDeferred(request.locateChildResource(calresource, name)) diff -x '*.svn' -x '*.pyc' -ru ./twistedcaldav/query/calendarquery.py ../../DCS1.2/CalendarServer-1.2/twistedcaldav/query/calendarquery.py --- ./twistedcaldav/query/calendarquery.py 2008-07-02 12:54:08.000000000 +0200 +++ ../../DCS1.2/CalendarServer-1.2/twistedcaldav/query/calendarquery.py 2008-08-22 16:20:27.000000000 +0200 @@ -24,7 +24,7 @@ __version__ = "0.0" __all__ = [ - "calendarquery", + "CalendarQueryExpressionGenerator", "sqlcalendarquery", ] @@ -42,151 +42,162 @@ #FIELD_DESCRIPTION = "RESOURCE.DESCRIPTION" #FIELD_LOCATION = "RESOURCE.LOCATION" -def calendarquery(filter): +class CalendarQueryExpressionGenerator(object): """ - Convert the supplied calendar-query into an expression tree. - - @param filter: the L{Filter} for thw calendar-query to convert. - @return: a L{baseExpression} for the expression tree. + Can parse CalDAV XML filters to expression structures and + Hold information about the expression useful for clients """ - - # Lets assume we have a valid filter from the outset. - - # Top-level filter contains exactly one comp-filter element - assert len(filter.children) == 1 - vcalfilter = filter.children[0] - assert isinstance(vcalfilter, caldavxml.ComponentFilter) - assert vcalfilter.filter_name == "VCALENDAR" - - if len(vcalfilter.children) > 0: - # Only comp-filters are handled - for _ignore in [x for x in vcalfilter.children if not isinstance(x, caldavxml.ComponentFilter)]: + + def __init__(self): + self.enddate = None + + def calendarquery(self, filter): + """ + Convert the supplied calendar-query into an expression tree. + + @param filter: the L{Filter} for thw calendar-query to convert. + @return: a L{baseExpression} for the expression tree. + """ + + # Lets assume we have a valid filter from the outset. + + # Top-level filter contains exactly one comp-filter element + assert len(filter.children) == 1 + vcalfilter = filter.children[0] + assert isinstance(vcalfilter, caldavxml.ComponentFilter) + assert vcalfilter.filter_name == "VCALENDAR" + + if len(vcalfilter.children) > 0: + # Only comp-filters are handled + for _ignore in [x for x in vcalfilter.children if not isinstance(x, caldavxml.ComponentFilter)]: + raise ValueError + + return self.compfilterListExpression(vcalfilter.children) + else: + return expression.allExpression() + + def compfilterListExpression(self, compfilters): + """ + Create an expression for a list of comp-filter elements. + + @param compfilters: the C{list} of L{ComponentFilter} elements. + @return: a L{baseExpression} for the expression tree. + """ + + if len(compfilters) == 1: + return self.compfilterExpression(compfilters[0]) + else: + return expression.orExpression([self.compfilterExpression(c) for c in compfilters]) + + def compfilterExpression(self, compfilter): + """ + Create an expression for a single comp-filter element. + + @param compfilter: the L{ComponentFilter} element. + @return: a L{baseExpression} for the expression tree. + """ + + # Handle is-not-defined case + if not compfilter.defined: + # Test for TYPE != <> + return expression.isnotExpression(FIELD_TYPE, compfilter.filter_name, True) + + expressions = [] + if isinstance(compfilter.filter_name, str): + expressions.append(expression.isExpression(FIELD_TYPE, compfilter.filter_name, True)) + else: + expressions.append(expression.inExpression(FIELD_TYPE, compfilter.filter_name, True)) + + # Handle time-range + if compfilter.qualifier and isinstance(compfilter.qualifier, caldavxml.TimeRange): + start, end, startfloat, endfloat = getTimerangeArguments(compfilter.qualifier) + expressions.append(expression.timerangeExpression(start, end, startfloat, endfloat)) + self.enddate = compfilter.qualifier.end.date() + + # Handle properties - we can only do UID right now + props = [] + for p in [x for x in compfilter.filters if isinstance(x, caldavxml.PropertyFilter)]: + props.append(self.propfilterExpression(p)) + if len(props) > 1: + propsExpression = expression.orExpression[props] + elif len(props) == 1: + propsExpression = props[0] + else: + propsExpression = None + + # Handle embedded components - we do not right now as our Index does not handle them + comps = [] + for _ignore in [x for x in compfilter.filters if isinstance(x, caldavxml.ComponentFilter)]: raise ValueError - - return compfilterListExpression(vcalfilter.children) - else: - return expression.allExpression() + if len(comps) > 1: + compsExpression = expression.orExpression[comps] + elif len(comps) == 1: + compsExpression = comps[0] + else: + compsExpression = None -def compfilterListExpression(compfilters): - """ - Create an expression for a list of comp-filter elements. - - @param compfilters: the C{list} of L{ComponentFilter} elements. - @return: a L{baseExpression} for the expression tree. - """ - - if len(compfilters) == 1: - return compfilterExpression(compfilters[0]) - else: - return expression.orExpression([compfilterExpression(c) for c in compfilters]) + # Now build compound expression + if ((propsExpression is not None) and (compsExpression is not None)): + expressions.append(expression.orExpression([propsExpression, compsExpression])) + elif propsExpression is not None: + expressions.append(propsExpression) + elif compsExpression is not None: + expressions.append(compsExpression) + + # Now build return expression + return expression.andExpression(expressions) + + def propfilterExpression(self, propfilter): + """ + Create an expression for a single prop-filter element. + + @param propfilter: the L{PropertyFilter} element. + @return: a L{baseExpression} for the expression tree. + """ -def compfilterExpression(compfilter): - """ - Create an expression for a single comp-filter element. - - @param compfilter: the L{ComponentFilter} element. - @return: a L{baseExpression} for the expression tree. - """ - - # Handle is-not-defined case - if not compfilter.defined: - # Test for TYPE != <> - return expression.isnotExpression(FIELD_TYPE, compfilter.filter_name, True) - - expressions = [] - if isinstance(compfilter.filter_name, str): - expressions.append(expression.isExpression(FIELD_TYPE, compfilter.filter_name, True)) - else: - expressions.append(expression.inExpression(FIELD_TYPE, compfilter.filter_name, True)) - - # Handle time-range - if compfilter.qualifier and isinstance(compfilter.qualifier, caldavxml.TimeRange): - start, end, startfloat, endfloat = getTimerangeArguments(compfilter.qualifier) - expressions.append(expression.timerangeExpression(start, end, startfloat, endfloat)) - - # Handle properties - we can only do UID right now - props = [] - for p in [x for x in compfilter.filters if isinstance(x, caldavxml.PropertyFilter)]: - props.append(propfilterExpression(p)) - if len(props) > 1: - propsExpression = expression.orExpression[props] - elif len(props) == 1: - propsExpression = props[0] - else: - propsExpression = None - - # Handle embedded components - we do not right now as our Index does not handle them - comps = [] - for _ignore in [x for x in compfilter.filters if isinstance(x, caldavxml.ComponentFilter)]: - raise ValueError - if len(comps) > 1: - compsExpression = expression.orExpression[comps] - elif len(comps) == 1: - compsExpression = comps[0] - else: - compsExpression = None - - # Now build compound expression - if ((propsExpression is not None) and (compsExpression is not None)): - expressions.append(expression.orExpression([propsExpression, compsExpression])) - elif propsExpression is not None: - expressions.append(propsExpression) - elif compsExpression is not None: - expressions.append(compsExpression) + # Only handle UID right now + if propfilter.filter_name != "UID": + raise ValueError - # Now build return expression - return expression.andExpression(expressions) + # Handle is-not-defined case + if not propfilter.defined: + # Test for <> != "*" + return expression.isExpression(FIELD_UID, "", True) -def propfilterExpression(propfilter): - """ - Create an expression for a single prop-filter element. - - @param propfilter: the L{PropertyFilter} element. - @return: a L{baseExpression} for the expression tree. - """ - - # Only handle UID right now - if propfilter.filter_name != "UID": - raise ValueError - - # Handle is-not-defined case - if not propfilter.defined: - # Test for <> != "*" - return expression.isExpression(FIELD_UID, "", True) - - # Handle time-range - we cannot do this with our Index right now - if propfilter.qualifier and isinstance(propfilter.qualifier, caldavxml.TimeRange): - raise ValueError - - # Handle text-match - tm = None - if propfilter.qualifier and isinstance(propfilter.qualifier, caldavxml.TextMatch): - if propfilter.qualifier.negate: - tm = expression.notcontainsExpression(propfilter.filter_name, str(propfilter.qualifier), propfilter.qualifier) + # Handle time-range - we cannot do this with our Index right now + if propfilter.qualifier and isinstance(propfilter.qualifier, caldavxml.TimeRange): + raise ValueError + + # Handle text-match + tm = None + if propfilter.qualifier and isinstance(propfilter.qualifier, caldavxml.TextMatch): + if propfilter.qualifier.negate: + tm = expression.notcontainsExpression(propfilter.filter_name, str(propfilter.qualifier), propfilter.qualifier) + else: + tm = expression.containsExpression(propfilter.filter_name, str(propfilter.qualifier), propfilter.qualifier) + + # Handle embedded parameters - we do not right now as our Index does not handle them + params = [] + for _ignore in propfilter.filters: + raise ValueError + if len(params) > 1: + paramsExpression = expression.orExpression[params] + elif len(params) == 1: + paramsExpression = params[0] else: - tm = expression.containsExpression(propfilter.filter_name, str(propfilter.qualifier), propfilter.qualifier) - - # Handle embedded parameters - we do not right now as our Index does not handle them - params = [] - for _ignore in propfilter.filters: - raise ValueError - if len(params) > 1: - paramsExpression = expression.orExpression[params] - elif len(params) == 1: - paramsExpression = params[0] - else: - paramsExpression = None - - # Now build return expression - if (tm is not None) and (paramsExpression is not None): - return expression.andExpression([tm, paramsExpression]) - elif tm is not None: - return tm - elif paramsExpression is not None: - return paramsExpression - else: - return None + paramsExpression = None + + # Now build return expression + if (tm is not None) and (paramsExpression is not None): + return expression.andExpression([tm, paramsExpression]) + elif tm is not None: + return tm + elif paramsExpression is not None: + return paramsExpression + else: + return None +# Utility function def getTimerangeArguments(timerange): """ Get start/end and floating start/end (adjusted for timezone offset) values from the @@ -209,20 +220,24 @@ return str(start), str(end), str(startfloat), str(endfloat) +# return SQL qualifiers and info to index. def sqlcalendarquery(filter): """ - Convert the supplied calendar-query into a oartial SQL statement. + Convert the supplied calendar-query into a partial SQL statement. - @param filter: the L{Filter} for thw calendar-query to convert. + @param filter: the L{Filter} for the calendar-query to convert. @return: a C{tuple} of (C{str}, C{list}), where the C{str} is the partial SQL statement, and the C{list} is the list of argument substitutions to use with the SQL API execute method. Or return C{None} if it is not possible to create an SQL query to fully match the calendar-query. """ try: - expression = calendarquery(filter) + expression_generator = CalendarQueryExpressionGenerator() + expression = expression_generator.calendarquery(filter) sql = sqlgenerator.sqlgenerator(expression) - return sql.generate() + qualifiers = (sql.generate(),expression_generator.enddate) + return qualifiers except ValueError: + # Indicate an unsupported filter return None diff -x '*.svn' -x '*.pyc' -ru ./twistedcaldav/static.py ../../DCS1.2/CalendarServer-1.2/twistedcaldav/static.py --- ./twistedcaldav/static.py 2008-08-22 16:42:51.000000000 +0200 +++ ../../DCS1.2/CalendarServer-1.2/twistedcaldav/static.py 2008-08-21 11:31:30.000000000 +0200 @@ -187,7 +187,7 @@ filteredaces = filteredaces.getResult() # Must verify ACLs which means we need a request object at this point - for name, uid, type in self.index().search(None): #@UnusedVariable + for name, uid, type in self.index().bruteforceSearch(): try: child = waitForDeferred(request.locateChildResource(self, name)) yield child