# -*- coding: utf-8 -*- import re import time import urllib import os import tempfile import zipfile import xbmc from net import HTTP from cache import Cache class TvDb: """ API: scraper - скрапер search - поиск сериалов movie - профайл фильма """ def __init__(self, language='en'): self.api_key = '33DBB309BB2B0ADB' dbname='tvdb.%s.db' % language self.cache = Cache(dbname, 1.0) self.language = language self.http = HTTP() self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:10.0.2) Gecko/20100101 Firefox/10.0.2', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'ru-ru,ru;q=0.8,en-us;q=0.5,en;q=0.3', 'Cache-Control': 'no-cache', 'Referer': 'http://www.thetvdb.com/' } # API def scraper(self, search, year=None): try: if not isinstance(search, list): search = [search] tag = 'scraper:' + urllib.quote_plus(":".join(search).encode('utf8')) except: return None else: if year: tag += ':' + str(year) id = self.cache.get(tag, self._scraper, search, year) if not id: return None return self.movie(id) def search(self, search, year=None): return self._search(search, year) def movie(self, id): id = str(id) return self.cache.get('movie:' + id, self._movie, id) def _movie(self, id): try: dirname = tempfile.mkdtemp() except: dirname = xbmc.translatePath('special://temp') for subdir in ('xbmcup', 'plugin.video.torrenter'): dirname = os.path.join(dirname, subdir) if not os.path.exists(dirname): os.mkdir(dirname) url = 'http://www.thetvdb.com/api/' + self.api_key + '/series/' + id + '/all/' + self.language + '.zip' # print url response = self.http.fetch(url, headers=self.headers, download=os.path.join(dirname, 'movie.zip'), timeout=20) if response.error: print "ERRRRRROR! " + str(response.error) self._movie_clear(dirname) return False, None try: filezip = zipfile.ZipFile(os.path.join(dirname, 'movie.zip'), 'r') filezip.extractall(dirname) filezip.close() movie = file(os.path.join(dirname, self.language + '.xml'), 'rb').read().decode('utf8') except: self._movie_clear(dirname) return False, None self._movie_clear(dirname) body = re.compile(r'(.+?)', re.U | re.S).search(movie) if not body: return False, None body = body.group(1) res = { 'icon': None, 'thumbnail': None, 'properties': { 'fanart_image': None, }, 'info': { 'count': int(id) } } # режисеры и сценаристы for tag in ('Director', 'Writer'): people = {} people_list = [] [people_list.extend(x.split('|')) for x in re.compile(r'<' + tag + r'>([^<]+)', re.U | re.S).findall(movie)] [people.update({x: 1}) for x in [x.strip() for x in people_list] if x] if people: res['info'][tag.lower()] = u', '.join([x for x in people.keys() if x]) for tag, retag, typeof, targettype in ( ('plot', 'Overview', None, None), ('mpaa', 'ContentRating', None, None), ('premiered', 'FirstAired', None, None), ('studio', 'Network', None, None), ('title', 'SeriesName', None, None), ('runtime', 'Runtime', None, None), ('votes', 'RatingCount', None, None), ('rating', 'Rating', float, None), ('genre', 'Genre', list, unicode), ('cast', 'Actors', list, None) ): r = re.compile(r'<' + retag + r'>([^<]+)', re.U | re.S).search(body) if r: r = r.group(1).strip() if typeof == float: res['info'][tag] = float(r) elif typeof == list: if targettype == unicode: res['info'][tag] = u', '.join([x for x in [x.strip() for x in r.split(u'|')] if x]) else: res['info'][tag] = [x for x in [x.strip() for x in r.split(u'|')] if x] else: res['info'][tag] = r # год if 'premiered' in res['info']: res['info']['year'] = int(res['info']['premiered'].split('-')[0]) # постер r = re.compile(r'([^<]+)', re.U | re.S).search(body) if r: res['icon'] = 'http://thetvdb.com/banners/' + r.group(1).strip() res['thumbnail'] = 'http://thetvdb.com/banners/' + r.group(1).strip() # фанарт r = re.compile(r'([^<]+)', re.U | re.S).search(body) if r: res['properties']['fanart_image'] = 'http://thetvdb.com/banners/' + r.group(1).strip() timeout = True # если фильм свежий, то кладем в кэш НЕ на долго (могут быть обновления на сайте) if 'year' not in res['info'] or not res['properties']['fanart_image'] \ or int(res['info']['year']) > time.gmtime(time.time()).tm_year: timeout = 7 * 24 * 60 * 60 * 4 #4 week return timeout, res def _movie_clear(self, dirname): for filename in os.listdir(dirname): try: os.unlink(os.path.join(dirname, filename)) except: raise try: os.rmdir(dirname) except: raise def _search(self, search, year=None): i = -1 id = None for name in search: # print urllib.quote_plus(name.encode('utf-8')) url = 'http://www.thetvdb.com/api/GetSeries.php?language=' + self.language + '&seriesname=' + urllib.quote_plus( name.encode('utf-8')) #print url i += 1 response = self.http.fetch(url, headers=self.headers, timeout=20) #print response.body if response.error: #print "ERRRRRROR! "+str(response.error) return None res = [] rows = re.compile('(.+?)', re.U | re.S).findall(response.body.decode('utf8')) if rows: recmd = re.compile('([0-9]+)', re.U | re.S) for row in [x for x in rows if x.find(u'%s' % self.language.decode('utf8')) != -1]: r = recmd.search(row) if r: res.append(int(r.group(1))) # в некоторых случаях можно найти только по оригинальному названию, # но при этом русское описание есть if not res and self.language != 'en': for row in [x for x in rows if x.find(u'en') != -1]: r = recmd.search(row) if r: res.append(int(r.group(1))) if len(res) > 1: Data = [] for id in res: for row in rows: recmd = re.compile('([0-9]+)', re.U | re.S) r = recmd.search(row) if int(r.group(1)) == id: title = re.compile('(.+?)', re.U | re.S).search(row) Syear = re.compile('(.+?)', re.U | re.S).search(row) if not Syear: Syear = 0 else: Syear = Syear.group(1) Data.append((title.group(1), Syear, id)) index = get_best(Data, search, year) if index and index['rate'] > 70: id = str(index['id']) elif len(res) == 1: id = str(res[0]) if id: break return id def _scraper(self, search, year): timeout = True # если фильм свежий, то кладем в кэш НЕ на долго (могут быть обновления на сайте) if year and year > time.gmtime(time.time()).tm_year: timeout = 7 * 24 * 60 * 60 * 4 # 4week id = self._search(search, year) if id is None: return 7 * 24 * 60 * 60 * 4, None else: # print str((timeout, ids['data'][0])) return timeout, id def isAsciiString(mediaName): """ Returns True if all characters of the string are ASCII. """ for index, char in enumerate(mediaName): if ord(char) >= 128: return False return True def toInteger(maybeNumber): """ Returns the argument converted to an integer if it represents a number or None if the argument is None or does not represent a number. """ try: if maybeNumber is not None and str(maybeNumber).strip() != '': return int(maybeNumber) except: pass return None import difflib SCORE_PENALTY_ITEM_ORDER = 0 SCORE_PENALTY_YEAR = 40 SCORE_PENALTY_TITLE = 40 def computeTitlePenalty(mediaName, title, year=None): """ Given media name and a candidate title, returns the title result score penalty. @param mediaName Movie title parsed from the file system. @param title Movie title from the website. """ mediaName = mediaName.lower() title = title.lower() if mediaName != title: # First approximate the whole strings. diffRatio = difflib.SequenceMatcher(None, mediaName, title).ratio() penalty = int(SCORE_PENALTY_TITLE * (1 - diffRatio)) # print '++++++ DIFF("%s", "%s") = %g --> %d' % (mediaName.encode('utf8'), title.encode('utf8'), diffRatio, penalty) # If the penalty is more than 1/2 of max title penalty, check to see if # this title starts with media name. This means that media name does not # have the whole name of the movie - very common case. For example, media name # "Кавказская пленница" for a movie title "Кавказская пленница, или Новые приключения Шурика". if penalty >= 15: # This is so that we don't have to do the 'split' every time. # Compute the scores of the # First, check if the title starts with media name. mediaNameParts = mediaName.split() titleParts = title.split() penaltyYear = 100 if year: diffRatio = difflib.SequenceMatcher(None, mediaName, '%s (%s)' % (title, str(year))).ratio() penaltyYear = int(SCORE_PENALTY_TITLE * (1 - diffRatio)) if len(mediaNameParts) <= len(titleParts): i = 0 # Start with some small penalty, value of which depends on how # many words media name has relative to the title's word count. penaltyAlt = max(5, int(round((1.0 - (float(len(mediaNameParts)) / len(titleParts))) * 15 - 5))) penaltyPerPart = SCORE_PENALTY_TITLE / len(mediaNameParts) for mediaNamePart in mediaNameParts: partDiffRatio = difflib.SequenceMatcher(None, mediaNamePart, titleParts[i]).ratio() penaltyAlt = penaltyAlt + int(penaltyPerPart * (1 - partDiffRatio)) i = i + 1 penalty = min(penalty, penaltyAlt, penaltyYear) # print '++++++ DIFF("%s", "%s") = %g --> %d' % (mediaName.encode('utf8'), title.encode('utf8'), diffRatio, penalty) # Log.Debug('++++++ DIFF("%s", "%s") = %g --> %d' % (mediaName.encode('utf8'), title.encode('utf8'), diffRatio, penalty)) return penalty return 0 def scoreMediaTitleMatch(mediaName, mediaAltTitle, mediaYear, title, year, itemIndex): """ Compares page and media titles taking into consideration media item's year and title values. Returns score [0, 100]. Search item scores 100 when: - it's first on the list of results; AND - it equals to the media title (ignoring case) OR all media title words are found in the search item; AND - search item year equals to media year. For now, our title scoring is pretty simple - we check if individual words from media item's title are found in the title from search results. We should also take into consideration order of words, so that "One Two" would not have the same score as "Two One". Also, taking into consideration year difference. """ # add logging that works in unit tests. # Log.Debug('comparing item %d::: "%s-%s" with "%s-%s" (%s)...' % # (itemIndex, str(mediaName), str(mediaYear), str(title), str(year), str(altTitle))) # Max score is when both title and year match exactly. score = 100 # Item order penalty (the lower it is on the list or results, the larger the penalty). #score = score - (itemIndex * SCORE_PENALTY_ITEM_ORDER) # Compute year penalty: [equal, diff>=3] --> [0, MAX]. yearPenalty = SCORE_PENALTY_YEAR mediaYear = toInteger(mediaYear) year = toInteger(year) if mediaYear is not None and year is not None: yearDiff = abs(mediaYear - year) if not yearDiff: yearPenalty = 0 elif yearDiff == 1: yearPenalty = int(SCORE_PENALTY_YEAR / 2) elif yearDiff == 2: yearPenalty = int(SCORE_PENALTY_YEAR / 1) else: # If year is unknown, don't penalize the score too much. yearPenalty = int(SCORE_PENALTY_YEAR / 1) score = score - yearPenalty # Compute title penalty. titlePenalty = computeTitlePenalty(mediaName, title, mediaYear) altTitlePenalty, altTitlePenalty2 = 100, 100 if mediaAltTitle not in [None, '']: altTitlePenalty2 = computeTitlePenalty(mediaAltTitle, title, mediaYear) titlePenalty = min(titlePenalty, altTitlePenalty, altTitlePenalty2) score = score - titlePenalty # If the score is not high enough, add a few points to the first result - # let's give KinoPoisk some credit :-). if itemIndex == 0 and score <= 80: score = score + 5 # IMPORTANT: always return an int. score = int(score) # Log.Debug('***** title scored %d' % score) return score def get_best(Data, search, year): shows = [] itemIndex = -1 if len(search) == 2: mediaName, mediaAltTitle = search else: mediaName, mediaAltTitle = search[0], None # print str(shows)+str(mediaAltTitle.encode('utf-8'))+str(year) for show in Data: itemIndex = itemIndex + 1 Syear = None if isinstance(show[1], unicode): Syear = int(show[1].split('-')[0]) Stitle = show[0] id = show[2] rate = scoreMediaTitleMatch(mediaName, mediaAltTitle, year, Stitle, Syear, itemIndex) shows.append({'rate': rate, 'id': id}) shows = sorted(shows, key=lambda x: x['rate'], reverse=True) print '********************** TheTVDB ******************************' print str(shows) + str(mediaName.encode('utf-8')) if shows: return shows[0]