Logo Search packages:      
Sourcecode: calibre version File versions  Download package

collection.py

#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import with_statement

__license__   = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'

import os, calendar
from threading import RLock
from datetime import datetime, timedelta

from lxml import etree
from lxml.builder import ElementMaker
from dateutil import parser

from calibre import browser

NS = 'http://calibre-ebook.com/recipe_collection'
E = ElementMaker(namespace=NS, nsmap={None:NS})

def iterate_over_builtin_recipe_files():
    exclude = ['craigslist', 'iht', 'outlook_india', 'toronto_sun']
    d = os.path.dirname
    base = os.path.join(d(d(d(d(d(d(os.path.abspath(__file__))))))), 'resources', 'recipes')
    for x in os.walk(base):
        for f in x[-1]:
            fbase, ext = os.path.splitext(f)
            if ext != '.recipe' or fbase in exclude:
                continue
            f = os.path.join(x[0], f)
            rid = os.path.splitext(os.path.relpath(f, base).replace(os.sep,
                '/'))[0]
            yield rid, os.path.join(x[0], f)


def serialize_recipe(urn, recipe_class):

    def attr(n, d):
        ans = getattr(recipe_class, n, d)
        if isinstance(ans, str):
            ans = ans.decode('utf-8', 'replace')
        return ans

    default_author = _('You') if urn.startswith('custom:') else _('Unknown')
    return E.recipe({
        'id'                 : str(urn),
        'title'              : attr('title', _('Unknown')),
        'author'             : attr('__author__', default_author),
        'language'           : attr('language', 'und'),
        'needs_subscription' : 'yes' if attr('needs_subscription', False) else 'no',
        'description'        : attr('description', '')
        })

def serialize_collection(mapping_of_recipe_classes):
    collection = E.recipe_collection()
    for urn, recipe_class in mapping_of_recipe_classes.items():
        recipe = serialize_recipe(urn, recipe_class)
        collection.append(recipe)
    collection.set('count', str(len(collection)))
    return etree.tostring(collection, encoding='utf-8', xml_declaration=True,
            pretty_print=True)

def serialize_builtin_recipes():
    from calibre.web.feeds.recipes import compile_recipe
    recipe_mapping = {}
    for rid, f in iterate_over_builtin_recipe_files():
        recipe_class = compile_recipe(open(f, 'rb').read())
        if recipe_class is not None:
            recipe_mapping['builtin:'+rid] = recipe_class

    return serialize_collection(recipe_mapping)

def get_builtin_recipe_collection():
    return etree.parse(P('builtin_recipes.xml')).getroot()

def get_custom_recipe_collection(db):
    from calibre.web.feeds.recipes import compile_recipe
    rmap = {}
    for id, title, recipe in db.get_custom_recipes():
        try:
            recipe_class = compile_recipe(recipe)
            if recipe_class is not None:
                rmap['custom:%d'%id] = recipe_class
        except:
            continue

    return etree.fromstring(serialize_collection(rmap))

def get_builtin_recipe_titles():
    return [r.get('title') for r in get_builtin_recipe_collection()]

def download_builtin_recipe(urn):
    br = browser()
    return br.open_novisit('http://status.calibre-ebook.com/recipe/'+urn).read()


def get_builtin_recipe_by_title(title, log=None, download_recipe=False):
    for x in get_builtin_recipe_collection():
        if x.get('title') == title:
            urn = x.get('id')[8:]
            if download_recipe:
                try:
                    if log is not None:
                        log('Trying to get latest version of recipe:', urn)
                    return download_builtin_recipe(urn)
                except:
                    if log is None:
                        import traceback
                        traceback.print_exc()
                    else:
                        log.exception(
                        'Failed to download recipe, using builtin version')
            return P('recipes/%s.recipe'%urn, data=True)

class SchedulerConfig(object):

    def __init__(self):
        from calibre.utils.config import config_dir
        from calibre.utils.lock import ExclusiveFile
        self.conf_path = os.path.join(config_dir, 'scheduler.xml')
        old_conf_path  = os.path.join(config_dir, 'scheduler.pickle')
        self.root = E.recipe_collection()
        self.lock = RLock()
        if os.access(self.conf_path, os.R_OK):
            with ExclusiveFile(self.conf_path) as f:
                self.root = etree.fromstring(f.read())
        elif os.path.exists(old_conf_path):
            self.migrate_old_conf(old_conf_path)

    def iter_recipes(self):
        for x in self.root:
            if x.tag == '{%s}scheduled_recipe'%NS:
                yield x

    def iter_accounts(self):
        for x in self.root:
            if x.tag == '{%s}account_info'%NS:
                yield x

    def iter_customization(self):
        for x in self.root:
            if x.tag == '{%s}recipe_customization'%NS:
                yield x

    def schedule_recipe(self, recipe, schedule_type, schedule, last_downloaded=None):
        with self.lock:
            for x in list(self.iter_recipes()):
                if x.get('id', False) == recipe.get('id'):
                    ld = x.get('last_downloaded', None)
                    if ld and last_downloaded is None:
                        try:
                            last_downloaded = parser.parse(ld)
                        except:
                            pass
                    self.root.remove(x)
                    break
            if last_downloaded is None:
                last_downloaded = datetime.fromordinal(1)
            sr = E.scheduled_recipe({
                'id' : recipe.get('id'),
                'title': recipe.get('title'),
                'last_downloaded':last_downloaded.isoformat(),
                }, self.serialize_schedule(schedule_type, schedule))
            self.root.append(sr)
            self.write_scheduler_file()

    def customize_recipe(self, urn, add_title_tag, custom_tags):
        with self.lock:
            for x in list(self.iter_customization()):
                if x.get('id') == urn:
                    self.root.remove(x)
            cs = E.recipe_customization({
                'id' : urn,
                'add_title_tag' : 'yes' if add_title_tag else 'no',
                'custom_tags' : ','.join(custom_tags),
                })
            self.root.append(cs)
            self.write_scheduler_file()

    def un_schedule_recipe(self, recipe_id):
        with self.lock:
            for x in list(self.iter_recipes()):
                if x.get('id', False) == recipe_id:
                    self.root.remove(x)
                    break
            self.write_scheduler_file()

    def update_last_downloaded(self, recipe_id):
        with self.lock:
            now = datetime.utcnow()
            for x in self.iter_recipes():
                if x.get('id', False) == recipe_id:
                    typ, sch, last_downloaded = self.un_serialize_schedule(x)
                    if typ == 'interval':
                        actual_interval = now - last_downloaded
                        nominal_interval = timedelta(days=sch)
                        if abs(actual_interval - nominal_interval) < \
                                timedelta(hours=1):
                            now = last_downloaded + nominal_interval
                    x.set('last_downloaded', now.isoformat())
                    break
            self.write_scheduler_file()

    def get_to_be_downloaded_recipes(self):
        ans = []
        with self.lock:
            for recipe in self.iter_recipes():
                if self.recipe_needs_to_be_downloaded(recipe):
                    ans.append(recipe.get('id'))
        return ans

    def write_scheduler_file(self):
        from calibre.utils.lock import ExclusiveFile
        self.root.text = '\n\n\t'
        for x in self.root:
            x.tail = '\n\n\t'
        if len(self.root) > 0:
            self.root[-1].tail = '\n\n'
        with ExclusiveFile(self.conf_path) as f:
            f.seek(0)
            f.truncate()
            f.write(etree.tostring(self.root, encoding='utf-8',
                xml_declaration=True, pretty_print=False))

    def serialize_schedule(self, typ, schedule):
        s = E.schedule({'type':typ})
        if typ == 'interval':
            if schedule < 0.1:
                schedule = 1/24.
            text = '%f'%schedule
        elif typ == 'day/time':
            text = '%d:%d:%d'%schedule
        s.text = text
        return s

    def un_serialize_schedule(self, recipe):
        for x in recipe.iterdescendants():
            if 'schedule' in x.tag:
                sch, typ = x.text, x.get('type')
                if typ == 'interval':
                    sch = float(sch)
                elif typ == 'day/time':
                    sch = list(map(int, sch.split(':')))
                return typ, sch, parser.parse(recipe.get('last_downloaded'))

    def recipe_needs_to_be_downloaded(self, recipe):
        try:
            typ, sch, ld = self.un_serialize_schedule(recipe)
        except:
            return False
        utcnow = datetime.utcnow()
        if typ == 'interval':
            return utcnow - ld > timedelta(sch)
        elif typ == 'day/time':
            now = datetime.now()
            offset = now - utcnow
            ld_local = ld + offset
            day, hour, minute = sch

            is_today = day < 0 or day > 6 or \
                    day == calendar.weekday(now.year, now.month, now.day)
            is_time = now.hour > hour or \
                    (now.hour == hour and now.minute >= minute)
            was_downloaded_already_today = ld_local.date() == now.date()
            return is_today and not was_downloaded_already_today and is_time
        return False

    def set_account_info(self, urn, un, pw):
        with self.lock:
            for x in list(self.iter_accounts()):
                if x.get('id', False) == urn:
                    self.root.remove(x)
                    break
            ac = E.account_info({'id':urn, 'username':un, 'password':pw})
            self.root.append(ac)
            self.write_scheduler_file()

    def get_account_info(self, urn):
        with self.lock:
            for x in self.iter_accounts():
                if x.get('id', False) == urn:
                    return x.get('username', ''), x.get('password', '')

    def get_customize_info(self, urn):
        add_title_tag = True
        custom_tags = []
        with self.lock:
            for x in self.iter_customization():
                if x.get('id', False) == urn:
                    add_title_tag = x.get('add_title_tag', 'yes') == 'yes'
                    custom_tags = [i.strip() for i in x.get('custom_tags',
                        '').split(',')]
                    break
        return add_title_tag, custom_tags

    def get_schedule_info(self, urn):
        with self.lock:
            for x in self.iter_recipes():
                if x.get('id', False) == urn:
                    ans = list(self.un_serialize_schedule(x))
                    return ans

    def migrate_old_conf(self, old_conf_path):
        from calibre.utils.config import DynamicConfig
        c = DynamicConfig('scheduler')
        for r in c.get('scheduled_recipes', []):
            try:
                self.add_old_recipe(r)
            except:
                continue
        for k in c.keys():
            if k.startswith('recipe_account_info'):
                try:
                    urn = k.replace('recipe_account_info_', '')
                    if urn.startswith('recipe_'):
                        urn = 'builtin:'+urn[7:]
                    else:
                        urn = 'custom:%d'%int(urn)
                    try:
                        username, password = c[k]
                    except:
                        username = password = ''
                    self.set_account_info(urn, unicode(username),
                            unicode(password))
                except:
                    continue
        del c
        self.write_scheduler_file()
        try:
            os.remove(old_conf_path)
        except:
            pass

    def add_old_recipe(self, r):
        urn = None
        if r['builtin'] and r['id'].startswith('recipe_'):
            urn = 'builtin:'+r['id'][7:]
        elif not r['builtin']:
            try:
                urn = 'custom:%d'%int(r['id'])
            except:
                return
        schedule = r['schedule']
        typ = 'interval'
        if schedule > 1e5:
            typ = 'day/time'
            raw = '%d'%int(schedule)
            day = int(raw[0]) - 1
            hour = int(raw[2:4]) - 1
            minute = int(raw[-2:]) - 1
            if day >= 7:
                day = -1
            schedule = [day, hour, minute]
        recipe = {'id':urn, 'title':r['title']}
        self.schedule_recipe(recipe, typ, schedule,
        last_downloaded=r['last_downloaded'])





Generated by  Doxygen 1.6.0   Back to index