''' Created on May 3, 2015 @author: ivan ''' import os from collections import deque import logging import copy import threading import traceback import shutil import urlparse from StringIO import StringIO from hachoir_metadata import extractMetadata from hachoir_parser import guessParser import hachoir_core.config as hachoir_config from hachoir_core.stream.input import InputIOStream logger = logging.getLogger('common') hachoir_config.quiet = True def enum(**enums): return type('Enum', (), enums) TerminalColor = enum(default='\033[39m', green='\033[32m', red='\033[31m', yellow='\033[33m') def get_duration(fn): # We need to provide just begining of file otherwise hachoir might try to read all file with open(fn, 'rb') as f: s = StringIO(f.read(1024 * 64)) p = guessParser(InputIOStream(s, filename=unicode(fn), tags=[])) m = extractMetadata(p) if m: return m.getItem('duration', 0) and m.getItem('duration', 0).value def debug_fn(fn): def _fn(*args, **kwargs): print "Entering %s, thread %s" % (fn.__name__, threading.current_thread().name) traceback.print_stack() ret = fn(*args, **kwargs) print "Leaving %s, thread %s" % (fn.__name__, threading.current_thread().name) return ret return _fn class Hasher(threading.Thread): def __init__(self, btfile, hash_cb): threading.Thread.__init__(self, name="Hasher") if btfile is None: raise ValueError('BTFile is None!') self._btfile = btfile self._hash_cb = hash_cb self.hash = None self.daemon = True self.start() def run(self): with self._btfile.create_cursor() as c: filehash = OpenSubtitles.hash_file(c, self._btfile.size) self.hash = filehash self._hash_cb(filehash) class OpenSubtitles(object): USER_AGENT = 'BTClient' def __init__(self, lang, user='', pwd=''): self._lang = lang self._token = None self._user = user self._pwd = pwd @staticmethod def hash_file(f, filesize): import struct longlongformat = ' 0: for i in xrange(self.size): if i + diff < self.size: self._cache[i] = self._cache[i + diff] else: self._cache[i] = None elif diff < 0: for i in xrange(self.size - 1, -1, -1): if i + diff >= 0: self._cache[i] = self._cache[i + diff] else: self._cache[i] = None self._cache_first = first self._event.clear() for i in xrange(self.size): if self._cache[i] is None and (self._cache_first + i) <= self._last_piece: to_request.append((self._cache_first + i, i)) for args in to_request: self._request_piece(*args) def add_piece(self, n, data): with self._lock: i = n - self._cache_first if i >= 0 and i < self.size: self._cache[i] = data if i == 0: self._event.set() def has_piece(self, n): with self._lock: i = n - self._cache_first if i >= 0 and i < self.size: return not (self._cache[i] is None) def _wait_piece(self, pc_no): while not self.has_piece(pc_no): self.fill_cache(pc_no) # self._event.clear() logger.debug('Waiting for piece %d' % pc_no) self._event.wait(self.TIMEOUT) def _get_piece(self, n): with self._lock: i = n - self._cache_first if i < 0 or i > self.size: raise ValueError('index of of scope of current cache') return self._cache[i] def get_piece(self, n): self._wait_piece(n) return self._get_piece(n) def read(self, offset, size): size = min(size, self._file_size - offset) if not size: return pc_no, ofs = self._map_offset(offset) data = self.get_piece(pc_no) pc_size = self._piece_size - ofs if pc_size > size: return data[ofs: ofs + size] else: pieces = [data[ofs:self._piece_size]] remains = size - pc_size new_head = pc_no + 1 while remains and self.has_piece(new_head): sz = min(remains, self._piece_size) data = self.get_piece(new_head) pieces.append(data[:sz]) remains -= sz if remains: new_head += 1 self.fill_cache(new_head) return ''.join(pieces) class BTCursor(object): def __init__(self, btfile): self._btfile = btfile self._pos = 0 self._cache = PieceCache(btfile) def clone(self): c = BTCursor(self._btfile) c._cache = self._cache.clone() return c def close(self): self._btfile.remove_cursor(self) def read(self, n=None): sz = self._btfile.size - self._pos if not n: n = sz else: n = min(n, sz) res = self._cache.read(self._pos, n) if res: self._pos += len(res) return res def seek(self, n): if n > self._btfile.size: n = self._btfile.size # raise ValueError('Seeking beyond file size') elif n < 0: raise ValueError('Seeking negative') self._pos = n def tell(self): return self._pos def update_piece(self, n, data): self._cache.add_piece(n, data) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() class AbstractFile(object): def __init__(self, path, base, size, piece_size): self._base = base self.size = size self.path = path self.piece_size = piece_size self.offset = 0 self._full_path = os.path.join(base, path) self._cursors = [] self._cursors_history = deque(maxlen=3) self._lock = threading.Lock() self.first_piece = 0 self.last_piece = self.first_piece + (max(size - 1, 0)) // piece_size self._rate = None self._piece_duration = None def add_cursor(self, c): with self._lock: self._cursors.append(c) def remove_cursor(self, c): with self._lock: self._cursors.remove(c) self._cursors_history.appendleft(c) def create_cursor(self, offset=None): c = None if offset is not None: with self._lock: for e in reversed(self._cursors): if abs(e.tell() - offset) < self.piece_size: c = e.clone() logger.debug('Cloning existing cursor') break if not c: with self._lock: for e in reversed(self._cursors_history): if abs(e.tell() - offset) < self.piece_size: c = e logger.debug('Reusing previous cursor') if not c: c = BTCursor(self) self.add_cursor(c) if offset: c.seek(offset) return c def map_piece(self, ofs): return self.first_piece + (ofs + self.offset) // self.piece_size, \ (ofs + self.offset) % self.piece_size def prioritize_piece(self, piece, idx): raise NotImplementedError() @property def full_path(self): return self._full_path def close(self): pass def remove(self): dirs = self.path.split(os.sep) if len(dirs) > 1: shutil.rmtree(os.path.join(self._base, dirs[0]), ignore_errors=True) else: os.unlink(self._full_path) def update_piece(self, n, data): for c in self._cursors: c.update_piece(n, data) @property def duration(self): if not hasattr(self, '_duration'): self._duration = get_duration(self._full_path) if os.path.exists(self._full_path) else 0 return self._duration or 0 @property def piece_duration_ms(self): if not self._piece_duration: if self.byte_rate: self._piece_duration = self.piece_size / self.byte_rate / 1000 return self._piece_duration @property def byte_rate(self): if not self._rate: d = self.duration if d: if hasattr(d, 'total_seconds'): total_seconds=d.total_seconds() else: total_seconds=(d.microseconds + (d.seconds + d.days * 24 * 3600) * 10**6) / 10**6 self._rate = self.size / total_seconds return self._rate def __str__(self): return self.path class Resolver(object): URL_PATTERN = None SPEED_LIMIT = None # kB/s THREADS = 4 def __init__(self, loader): self._client = loader def resolve(self, url): return url @staticmethod def url_to_file(uri): path = urlparse.urlsplit(uri)[2] if path.startswith('/'): path = path[1:] return path