Logo Search packages:      
Sourcecode: calibre version File versions

util.py

00001 """base classes and helper functions for css and stylesheets packages
"""
__all__ = []
__docformat__ = 'restructuredtext'
__version__ = '$Id: util.py 1781 2009-07-19 12:30:49Z cthedot $'

from helper import normalize
from itertools import ifilter
import css
import codec
import codecs
import errorhandler
import tokenize2
import types
import xml.dom

try:
    from _fetchgae import _defaultFetcher
except ImportError, e:
    from _fetch import _defaultFetcher
    
log = errorhandler.ErrorHandler()

00024 class _BaseClass(object):
    """
    Base class for Base, Base2 and _NewBase.

    **Base and Base2 will be removed in the future!**
    """
    _log = errorhandler.ErrorHandler()
    _prods = tokenize2.CSSProductions

    def _checkReadonly(self):
        "Raise xml.dom.NoModificationAllowedErr if rule/... is readonly"
        if hasattr(self, '_readonly') and self._readonly:
            raise xml.dom.NoModificationAllowedErr(
                u'%s is readonly.' % self.__class__)
            return True
        return False

00041     def _valuestr(self, t):
        """
        Return string value of t (t may be a string, a list of token tuples
        or a single tuple in format (type, value, line, col).
        Mainly used to get a string value of t for error messages.
        """
        if not t:
            return u''
        elif isinstance(t, basestring):
            return t
        else:
            return u''.join([x[1] for x in t])


00055 class _NewBase(_BaseClass):
    """
    New base class for classes using ProdParser.

    **Currently CSSValue and related ones only.**
    """
    def __init__(self):
        self._seq = Seq()

00064     def _setSeq(self, newseq):
        """Set value of ``seq`` which is readonly."""
        newseq._readonly = True
        self._seq = newseq

    def _tempSeq(self, readonly=False):
        "Get a writeable Seq() which is used to set ``seq`` later"
        return Seq(readonly=readonly)

    seq = property(lambda self: self._seq,
                   doc="Internal readonly attribute, **DO NOT USE**!")


00077 class Base(_BaseClass):
    """
    **Superceded by _NewBase**

    **Superceded by Base2 which is used for new seq handling class.**

    Base class for most CSS and StyleSheets classes

    Contains helper methods for inheriting classes helping parsing

    ``_normalize`` is static as used by Preferences.
    """
    __tokenizer2 = tokenize2.Tokenizer()

    # for more on shorthand properties see
    # http://www.dustindiaz.com/css-shorthand/
    # format: shorthand: [(propname, mandatorycheck?)*]
    _SHORTHANDPROPERTIES = {
            u'background': [],
            u'background-position': [],
            u'border': [],
            u'border-left': [],
            u'border-right': [],
            u'border-top': [],
            u'border-bottom': [],
            #u'border-color': [], # list or single but same values
            #u'border-style': [], # list or single but same values
            #u'border-width': [], # list or single but same values
            u'cue': [],
            u'font': [],
            u'list-style': [],
            #u'margin': [], # list or single but same values
            u'outline': [],
            #u'padding': [], # list or single but same values
            u'pause': []
            }

    @staticmethod
00115     def _normalize(x):
        """
        normalizes x, namely:

        - remove any \ before non unicode sequences (0-9a-zA-Z) so for
          x=="c\olor\" return "color" (unicode escape sequences should have
          been resolved by the tokenizer already)
        - lowercase
        """
        return normalize(x)

00126     def _splitNamespacesOff(self, text_namespaces_tuple):
        """
        returns tuple (text, dict-of-namespaces) or if no namespaces are
        in cssText returns (cssText, {})

        used in Selector, SelectorList, CSSStyleRule, CSSMediaRule and
        CSSStyleSheet
        """
        if isinstance(text_namespaces_tuple, tuple):
            return text_namespaces_tuple[0], _SimpleNamespaces(self._log,
                                                    text_namespaces_tuple[1])
        else:
            return text_namespaces_tuple, _SimpleNamespaces(log=self._log)

00140     def _tokenize2(self, textortokens):
        """
        returns tokens of textortokens which may already be tokens in which
        case simply returns input
        """
        if not textortokens:
            return None
        elif isinstance(textortokens, basestring):
            # needs to be tokenized
            return self.__tokenizer2.tokenize(
                 textortokens)
        elif types.GeneratorType == type(textortokens):
            # already tokenized
            return textortokens
        elif isinstance(textortokens, tuple):
            # a single token (like a comment)
            return [textortokens]
        else:
            # already tokenized but return generator
            return (x for x in textortokens)

    def _nexttoken(self, tokenizer, default=None):
        "returns next token in generator tokenizer or the default value"
        try:
            return tokenizer.next()
        except (StopIteration, AttributeError):
            return default

    def _type(self, token):
        "returns type of Tokenizer token"
        if token:
            return token[0]
        else:
            return None

    def _tokenvalue(self, token, normalize=False):
        "returns value of Tokenizer token"
        if token and normalize:
            return Base._normalize(token[1])
        elif token:
            return token[1]
        else:
            return None

00184     def _stringtokenvalue(self, token):
        """
        for STRING returns the actual content without surrounding "" or ''
        and without respective escapes, e.g.::

             "with \" char" => with " char
        """
        if token:
            value = token[1]
            return value.replace('\\' + value[0], value[0])[1: - 1]
        else:
            return None

00197     def _uritokenvalue(self, token):
        """
        for URI returns the actual content without surrounding url()
        or url(""), url('') and without respective escapes, e.g.::

             url("\"") => "
        """
        if token:
            value = token[1][4: - 1].strip()
            if value and (value[0] in '\'"') and (value[0] == value[ - 1]):
                # a string "..." or '...'
                value = value.replace('\\' + value[0], value[0])[1: - 1]
            return value
        else:
            return None

00213     def _tokensupto2(self,
                     tokenizer,
                     starttoken=None,
                     blockstartonly=False, # {
                     blockendonly=False, # }
                     mediaendonly=False,
                     importmediaqueryendonly=False, # ; or STRING
                     mediaqueryendonly=False, # { or STRING
                     semicolon=False, # ;
                     propertynameendonly=False, # :
                     propertyvalueendonly=False, # ! ; }
                     propertypriorityendonly=False, # ; }
                     selectorattendonly=False, # ]
                     funcendonly=False, # )
                     listseponly=False, # ,
                     separateEnd=False # returns (resulttokens, endtoken)
                     ):
        """
        returns tokens upto end of atrule and end index
        end is defined by parameters, might be ; } ) or other

        default looks for ending "}" and ";"
        """
        ends = u';}'
        endtypes = ()
        brace = bracket = parant = 0 # {}, [], ()

        if blockstartonly: # {
            ends = u'{'
            brace = - 1 # set to 0 with first {
        elif blockendonly: # }
            ends = u'}'
            brace = 1
        elif mediaendonly: # }
            ends = u'}'
            brace = 1 # rules } and mediarules }
        elif importmediaqueryendonly:
            # end of mediaquery which may be ; or STRING
            ends = u';'
            endtypes = ('STRING',)
        elif mediaqueryendonly:
            # end of mediaquery which may be { or STRING
            # special case, see below
            ends = u'{'
            brace = - 1 # set to 0 with first {
            endtypes = ('STRING',)
        elif semicolon:
            ends = u';'
        elif propertynameendonly: # : and ; in case of an error
            ends = u':;'
        elif propertyvalueendonly: # ; or !important
            ends = u';!'
        elif propertypriorityendonly: # ;
            ends = u';'
        elif selectorattendonly: # ]
            ends = u']'
            if starttoken and self._tokenvalue(starttoken) == u'[':
                bracket = 1
        elif funcendonly: # )
            ends = u')'
            parant = 1
        elif listseponly: # ,
            ends = u','

        resulttokens = []
        if starttoken:
            resulttokens.append(starttoken)
        if tokenizer:
            for token in tokenizer:
                typ, val, line, col = token
                if 'EOF' == typ:
                    resulttokens.append(token)
                    break
                if u'{' == val:
                    brace += 1
                elif u'}' == val:
                    brace -= 1
                elif u'[' == val:
                    bracket += 1
                elif u']' == val:
                    bracket -= 1
                # function( or single (
                elif u'(' == val or \
                     Base._prods.FUNCTION == typ:
                    parant += 1
                elif u')' == val:
                    parant -= 1

                resulttokens.append(token)

                if (brace == bracket == parant == 0) and (
                    val in ends or typ in endtypes):
                    break
                elif mediaqueryendonly and brace == - 1 and (
                     bracket == parant == 0) and typ in endtypes:
                     # mediaqueryendonly with STRING
                    break
        if separateEnd:
            # TODO: use this method as generator, then this makes sense
            if resulttokens:
                return resulttokens[: - 1], resulttokens[ - 1]
            else:
                return resulttokens, None
        else:
            return resulttokens

00319     def _adddefaultproductions(self, productions, new=None):
        """
        adds default productions if not already present, used by
        _parse only

        each production should return the next expected token
        normaly a name like "uri" or "EOF"
        some have no expectation like S or COMMENT, so simply return
        the current value of self.__expected
        """
        def ATKEYWORD(expected, seq, token, tokenizer=None):
            "default impl for unexpected @rule"
            if expected != 'EOF':
                # TODO: parentStyleSheet=self
                rule = css.CSSUnknownRule()
                rule.cssText = self._tokensupto2(tokenizer, token)
                if rule.wellformed:
                    seq.append(rule)
                return expected
            else:
                new['wellformed'] = False
                self._log.error(u'Expected EOF.', token=token)
                return expected

        def COMMENT(expected, seq, token, tokenizer=None):
            "default implementation for COMMENT token adds CSSCommentRule"
            seq.append(css.CSSComment([token]))
            return expected

        def S(expected, seq, token, tokenizer=None):
            "default implementation for S token, does nothing"
            return expected

        def EOF(expected=None, seq=None, token=None, tokenizer=None):
            "default implementation for EOF token"
            return 'EOF'

        p = {'ATKEYWORD': ATKEYWORD,
             'COMMENT': COMMENT,
             'S': S,
             'EOF': EOF # only available if fullsheet
             }
        p.update(productions)
        return p

00364     def _parse(self, expected, seq, tokenizer, productions, default=None,
               new=None, initialtoken=None):
        """
        puts parsed tokens in seq by calling a production with
            (seq, tokenizer, token)

        expected
            a name what token or value is expected next, e.g. 'uri'
        seq
            to add rules etc to
        tokenizer
            call tokenizer.next() to get next token
        productions
            callbacks {tokentype: callback}
        default
            default callback if tokentype not in productions
        new
            used to init default productions
        initialtoken
            will be used together with tokenizer running 1st this token
            and then all tokens in tokenizer

        returns (wellformed, expected) which the last prod might have set
        """
        wellformed = True

        if initialtoken:
            # add initialtoken to tokenizer
            def tokens():
                "Build new tokenizer including initialtoken"
                yield initialtoken
                for item in tokenizer:
                    yield item
            fulltokenizer = (t for t in tokens())
        else:
            fulltokenizer = tokenizer

        if fulltokenizer:
            prods = self._adddefaultproductions(productions, new)
            for token in fulltokenizer:
                p = prods.get(token[0], default)
                if p:
                    expected = p(expected, seq, token, tokenizer)
                else:
                    wellformed = False
                    self._log.error(u'Unexpected token (%s, %s, %s, %s)' % token)
        return wellformed, expected


00413 class Base2(Base, _NewBase):
    """
    **Superceded by _NewBase.**

    Base class for new seq handling.
    """
    def __init__(self):
        self._seq = Seq()

00422     def _adddefaultproductions(self, productions, new=None):
        """
        adds default productions if not already present, used by
        _parse only

        each production should return the next expected token
        normaly a name like "uri" or "EOF"
        some have no expectation like S or COMMENT, so simply return
        the current value of self.__expected
        """
        def ATKEYWORD(expected, seq, token, tokenizer=None):
            "default impl for unexpected @rule"
            if expected != 'EOF':
                # TODO: parentStyleSheet=self
                rule = css.CSSUnknownRule()
                rule.cssText = self._tokensupto2(tokenizer, token)
                if rule.wellformed:
                    seq.append(rule, css.CSSRule.UNKNOWN_RULE,
                               line=token[2], col=token[3])
                return expected
            else:
                new['wellformed'] = False
                self._log.error(u'Expected EOF.', token=token)
                return expected

        def COMMENT(expected, seq, token, tokenizer=None):
            "default impl, adds CSSCommentRule if not token == EOF"
            if expected == 'EOF':
                new['wellformed'] = False
                self._log.error(u'Expected EOF but found comment.', token=token)
            seq.append(css.CSSComment([token]), 'COMMENT')
            return expected

        def S(expected, seq, token, tokenizer=None):
            "default impl, does nothing if not token == EOF"
            if expected == 'EOF':
                new['wellformed'] = False
                self._log.error(u'Expected EOF but found whitespace.', token=token)
            return expected

        def EOF(expected=None, seq=None, token=None, tokenizer=None):
            "default implementation for EOF token"
            return 'EOF'

        defaultproductions = {'ATKEYWORD': ATKEYWORD,
             'COMMENT': COMMENT,
             'S': S,
             'EOF': EOF # only available if fullsheet
             }
        defaultproductions.update(productions)
        return defaultproductions


00475 class Seq(object):
    """
    property seq of Base2 inheriting classes, holds a list of Item objects.

    used only by Selector for now

    is normally readonly, only writable during parsing
    """
00483     def __init__(self, readonly=True):
        """
        only way to write to a Seq is to initialize it with new items
        each itemtuple has (value, type, line) where line is optional
        """
        self._seq = []
        self._readonly = readonly

    def __delitem__(self, i):
        del self._seq[i]

    def __getitem__(self, i):
        return self._seq[i]

    def __setitem__(self, i, (val, typ, line, col)):
        self._seq[i] = Item(val, typ, line, col)

    def __iter__(self):
        return iter(self._seq)

    def __len__(self):
        return len(self._seq)

    def append(self, val, typ, line=None, col=None):
        "if not readonly add new Item()"
        if self._readonly:
            raise AttributeError('Seq is readonly.')
        else:
            self._seq.append(Item(val, typ, line, col))

    def appendItem(self, item):
        "if not readonly add item which must be an Item"
        if self._readonly:
            raise AttributeError('Seq is readonly.')
        else:
            self._seq.append(item)

00520     def replace(self, index= - 1, val=None, typ=None, line=None, col=None):
        """
        if not readonly replace Item at index with new Item or
        simply replace value or type
        """
        if self._readonly:
            raise AttributeError('Seq is readonly.')
        else:
            self._seq[index] = Item(val, typ, line, col)

    def rstrip(self):
        "trims S items from end of Seq"
        while self._seq and self._seq[ - 1].type == tokenize2.CSSProductions.S:
            # TODO: removed S before CSSComment /**/ /**/
            del self._seq[ - 1]

00536     def appendToVal(self, val=None, index= - 1):
        """
        if not readonly append to Item's value at index
        """
        if self._readonly:
            raise AttributeError('Seq is readonly.')
        else:
            old = self._seq[index]
            self._seq[index] = Item(old.value + val, old.type,
                                    old.line, old.col)

    def __repr__(self):
        "returns a repr same as a list of tuples of (value, type)"
        return u'cssutils.%s.%s([\n    %s], readonly=%r)' % (self.__module__,
                                          self.__class__.__name__,
            u',\n    '.join([u'%r' % item for item in self._seq]
            ), self._readonly)

    def __str__(self):
        vals = []
        for v in self:
            if isinstance(v.value, basestring):
                vals.append(v.value)
            elif type(v) == tuple:
                vals.append(v.value[1])
            else:
                vals.append(str(v))

        return "<cssutils.%s.%s object length=%r values=%r readonly=%r at 0x%x>" % (
                self.__module__, self.__class__.__name__, len(self),
                u''.join(vals), self._readonly, id(self))

00568 class Item(object):
    """
    an item in the seq list of classes (successor to tuple items in old seq)

    each item has attributes:

    type
        a sematic type like "element", "attribute"
    value
        the actual value which may be a string, number etc or an instance
        of e.g. a CSSComment
    *line*
        **NOT IMPLEMENTED YET, may contain the line in the source later**
    """
    def __init__(self, value, type, line=None, col=None):
        self.__value = value
        self.__type = type
        self.__line = line
        self.__col = col

    type = property(lambda self: self.__type)
    value = property(lambda self: self.__value)
    line = property(lambda self: self.__line)
    col = property(lambda self: self.__col)

    def __repr__(self):
        return "%s.%s(value=%r, type=%r, line=%r, col=%r)" % (
                self.__module__, self.__class__.__name__,
                self.__value, self.__type, self.__line, self.__col)


00599 class ListSeq(object):
    """
    (EXPERIMENTAL)
    A base class used for list classes like css.SelectorList or
    stylesheets.MediaList

    adds list like behaviour running on inhering class' property ``seq``

    - item in x => bool
    - len(x) => integer
    - get, set and del x[i]
    - for item in x
    - append(item)

    some methods must be overwritten in inheriting class
    """
    def __init__(self):
        self.seq = [] # does not need to use ``Seq`` as simple list only

    def __contains__(self, item):
        return item in self.seq

    def __delitem__(self, index):
        del self.seq[index]

    def __getitem__(self, index):
        return self.seq[index]

    def __iter__(self):
        def gen():
            for x in self.seq:
                yield x
        return gen()

    def __len__(self):
        return len(self.seq)

    def __setitem__(self, index, item):
        "must be overwritten"
        raise NotImplementedError

    def append(self, item):
        "must be overwritten"
        raise NotImplementedError


00645 class _Namespaces(object):
    """
    A dictionary like wrapper for @namespace rules used in a CSSStyleSheet.
    Works on effective namespaces, so e.g. if::

        @namespace p1 "uri";
        @namespace p2 "uri";

    only the second rule is effective and kept.

    namespaces
        a dictionary {prefix: namespaceURI} containing the effective namespaces
        only. These are the latest set in the CSSStyleSheet.
    parentStyleSheet
        the parent CSSStyleSheet
    """
    def __init__(self, parentStyleSheet, log=None, *args):
        "no initial values are set, only the relevant sheet is"
        self.parentStyleSheet = parentStyleSheet
        self._log = log

    def __repr__(self):
        return "%r" % self.namespaces

    def __contains__(self, prefix):
        return prefix in self.namespaces

00672     def __delitem__(self, prefix):
        """deletes CSSNamespaceRule(s) with rule.prefix == prefix

        prefix '' and None are handled the same
        """
        if not prefix:
            prefix = u''
        delrule = self.__findrule(prefix)
        for i, rule in enumerate(ifilter(lambda r: r.type == r.NAMESPACE_RULE,
                            self.parentStyleSheet.cssRules)):
            if rule == delrule:
                self.parentStyleSheet.deleteRule(i)
                return

        self._log.error('Prefix %r not found.' % prefix,
                        error=xml.dom.NamespaceErr)

    def __getitem__(self, prefix):
        try:
            return self.namespaces[prefix]
        except KeyError, e:
            self._log.error('Prefix %r not found.' % prefix,
                            error=xml.dom.NamespaceErr)

    def __iter__(self):
        return self.namespaces.__iter__()

    def __len__(self):
        return len(self.namespaces)

    def __setitem__(self, prefix, namespaceURI):
        "replaces prefix or sets new rule, may raise NoModificationAllowedErr"
        if not prefix:
            prefix = u'' # None or ''
        rule = self.__findrule(prefix)
        if not rule:
            self.parentStyleSheet.insertRule(css.CSSNamespaceRule(
                                                    prefix=prefix,
                                                    namespaceURI=namespaceURI),
                                  inOrder=True)
        else:
            if prefix in self.namespaces:
                rule.namespaceURI = namespaceURI # raises NoModificationAllowedErr
            if namespaceURI in self.namespaces.values():
                rule.prefix = prefix

    def __findrule(self, prefix):
        # returns namespace rule where prefix == key
        for rule in ifilter(lambda r: r.type == r.NAMESPACE_RULE,
                            reversed(self.parentStyleSheet.cssRules)):
            if rule.prefix == prefix:
                return rule

    @property
00726     def namespaces(self):
        """
        A property holding only effective @namespace rules in
        self.parentStyleSheets.
        """
        namespaces = {}
        for rule in ifilter(lambda r: r.type == r.NAMESPACE_RULE,
                            reversed(self.parentStyleSheet.cssRules)):
            if rule.namespaceURI not in namespaces.values():
                namespaces[rule.prefix] = rule.namespaceURI
        return namespaces

    def get(self, prefix, default):
        return self.namespaces.get(prefix, default)

    def items(self):
        return self.namespaces.items()

    def keys(self):
        return self.namespaces.keys()

    def values(self):
        return self.namespaces.values()

00750     def prefixForNamespaceURI(self, namespaceURI):
        """
        returns effective prefix for given namespaceURI or raises IndexError
        if this cannot be found"""
        for prefix, uri in self.namespaces.items():
            if uri == namespaceURI:
                return prefix
        raise IndexError(u'NamespaceURI %r not found.' % namespaceURI)

    def __str__(self):
        return u"<cssutils.util.%s object parentStyleSheet=%r at 0x%x>" % (
                self.__class__.__name__, str(self.parentStyleSheet), id(self))


00764 class _SimpleNamespaces(_Namespaces):
    """
    namespaces used in objects like Selector as long as they are not connected
    to a CSSStyleSheet
    """
00769     def __init__(self, log=None, *args):
        """init"""
        super(_SimpleNamespaces, self).__init__(parentStyleSheet=None, log=log)
        self.__namespaces = dict(*args)

    def __setitem__(self, prefix, namespaceURI):
        self.__namespaces[prefix] = namespaceURI

    namespaces = property(lambda self: self.__namespaces,
                          doc=u'Dict Wrapper for self.sheets @namespace rules.')

    def __str__(self):
        return u"<cssutils.util.%s object namespaces=%r at 0x%x>" % (
                self.__class__.__name__, self.namespaces, id(self))

    def __repr__(self):
        return u"cssutils.util.%s(%r)" % (self.__class__.__name__,
            self.namespaces)


00789 def _readUrl(url, fetcher=None, overrideEncoding=None, parentEncoding=None):
    """
    Read cssText from url and decode it using all relevant methods (HTTP
    header, BOM, @charset). Returns

    - encoding used to decode text (which is needed to set encoding of
      stylesheet properly)
    - type of encoding (how it was retrieved, see list below)
    - decodedCssText

    ``fetcher``
        see cssutils.registerFetchUrl for details
    ``overrideEncoding``
        If given this encoding is used and all other encoding information is
        ignored (HTTP, BOM etc)
    ``parentEncoding``
        Encoding of parent stylesheet (while e.g. reading @import references sheets)
        or document if available.

    Priority or encoding information
    --------------------------------
    **cssutils only**: 0. overrideEncoding

    1. An HTTP "charset" parameter in a "Content-Type" field (or similar parameters in other protocols)
    2. BOM and/or @charset (see below)
    3. <link charset=""> or other metadata from the linking mechanism (if any)
    4. charset of referring style sheet or document (if any)
    5. Assume UTF-8

    """
    enctype = None

    if not fetcher:
        fetcher = _defaultFetcher
    r = fetcher(url)
    if r and len(r) == 2 and r[1] is not None:
        httpEncoding, content = r

        if overrideEncoding:
            enctype = 0 # 0. override encoding
            encoding = overrideEncoding
        elif httpEncoding:
            enctype = 1 # 1. HTTP
            encoding = httpEncoding
        else:  
            if isinstance(content, unicode): 
                # no need to check content as unicode so no BOM
                explicit = False
            else:
                # check content
                contentEncoding, explicit = codec.detectencoding_str(content)

            if explicit:
                enctype = 2 # 2. BOM/@charset: explicitly
                encoding = contentEncoding
            elif parentEncoding:
                enctype = 4 # 4. parent stylesheet or document
                # may also be None in which case 5. is used in next step anyway
                encoding = parentEncoding
            else:
                enctype = 5 # 5. assume UTF-8
                encoding = 'utf-8'

        if isinstance(content, unicode):
            decodedCssText = content
        else:
            try:
                # encoding may still be wrong if encoding *is lying*!
                decodedCssText = codecs.lookup("css")[1](content, encoding=encoding)[0]
            except UnicodeDecodeError, e:
                log.warn(e, neverraise=True)
                decodedCssText = None

        return encoding, enctype, decodedCssText
    else:
        return None, None, None


Generated by  Doxygen 1.6.0   Back to index