lol

why?

resources/scrapers/kinopoisk/HTTP.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+#
+# Russian metadata plugin for Plex, which uses http://www.kinopoisk.ru/ to get the tag data.
+# Плагин для обновления информации о фильмах использующий КиноПоиск (http://www.kinopoisk.ru/).
+# Copyright (C) 2012 Yevgeny Nyden
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+# @author zhenya (Yevgeny Nyden)
+# @revision 148
+
+import urllib
+import urllib2
+import base64
+import time
+
+import LOGGER as Log
+
+
+class HTTP:
+    def __init__(self):
+        pass
+
+
+    def fetch(self, request, **kwargs):
+        self.con, self.fd, self.progress, self.cookies, self.request = None, None, None, None, request
+
+        if not isinstance(self.request, HTTPRequest):
+            self.request = HTTPRequest(url=self.request, **kwargs)
+
+        self.response = HTTPResponse(self.request)
+
+        Log.Debug('XBMCup: HTTP: request: ' + str(self.request))
+        try:
+            self._opener()
+            self._fetch()
+        except Exception, e:
+            Log.Debug('XBMCup: HTTP: ' + str(e))
+            if isinstance(e, urllib2.HTTPError):
+                self.response.code = e.code
+            self.response.error = e
+        else:
+            self.response.code = 200
+
+        if self.fd:
+            self.fd.close()
+            self.fd = None
+
+        if self.con:
+            self.con.close()
+            self.con = None
+
+        if self.progress:
+            self.progress.close()
+            self.progress = None
+
+        self.response.time = time.time() - self.response.time
+
+        return self.response
+
+
+    def _opener(self):
+
+        build = [urllib2.HTTPHandler()]
+
+        if self.request.redirect:
+            build.append(urllib2.HTTPRedirectHandler())
+
+        if self.request.proxy_host and self.request.proxy_port:
+            build.append(urllib2.ProxyHandler(
+                {self.request.proxy_protocol: self.request.proxy_host + ':' + str(self.request.proxy_port)}))
+
+            if self.request.proxy_username:
+                proxy_auth_handler = urllib2.ProxyBasicAuthHandler()
+                proxy_auth_handler.add_password('realm', 'uri', self.request.proxy_username,
+                                                self.request.proxy_password)
+                build.append(proxy_auth_handler)
+
+        urllib2.install_opener(urllib2.build_opener(*build))
+
+
+    def _fetch(self):
+        params = {} if self.request.params is None else self.request.params
+
+        if self.request.method == 'POST':
+            if isinstance(params, dict) or isinstance(params, list):
+                params = urllib.urlencode(params)
+            req = urllib2.Request(self.request.url, params)
+        else:
+            req = urllib2.Request(self.request.url)
+
+        for key, value in self.request.headers.iteritems():
+            req.add_header(key, value)
+
+        if self.request.auth_username and self.request.auth_password:
+            req.add_header('Authorization', 'Basic %s' % base64.encodestring(
+                ':'.join([self.request.auth_username, self.request.auth_password])).strip())
+
+        # self.con = urllib2.urlopen(req, timeout=self.request.timeout)
+        self.con = urllib2.urlopen(req)
+        self.response.headers = self._headers(self.con.info())
+
+        self.response.body = self.con.read()
+
+        if self.request.cookies:
+            self.cookies.save(self.request.cookies)
+
+
+    def _headers(self, raw):
+        headers = {}
+        for line in raw.headers:
+            pair = line.split(':', 1)
+            if len(pair) == 2:
+                tag = pair[0].lower().strip()
+                value = pair[1].strip()
+                if tag and value:
+                    headers[tag] = value
+        return headers
+
+
+    def _progress(self, read, size, name):
+        res = []
+        if size < 0:
+            res.append(1)
+        else:
+            res.append(int(float(read) / (float(size) / 100.0)))
+        if name:
+            res.append(u'File: ' + name)
+        if size != -1:
+            res.append(u'Size: ' + self._human(size))
+        res.append(u'Load: ' + self._human(read))
+        return res
+
+    def _human(self, size):
+        human = None
+        for h, f in (('KB', 1024), ('MB', 1024 * 1024), ('GB', 1024 * 1024 * 1024), ('TB', 1024 * 1024 * 1024 * 1024)):
+            if size / f > 0:
+                human = h
+                factor = f
+            else:
+                break
+        if human is None:
+            return (u'%10.1f %s' % (size, u'byte')).replace(u'.0', u'')
+        else:
+            return u'%10.2f %s' % (float(size) / float(factor), human)
+
+
+class HTTPRequest:
+    def __init__(self, url, method='GET', headers=None, cookies=None, params=None, upload=None, download=None,
+                 progress=False, auth_username=None, auth_password=None, proxy_protocol='http', proxy_host=None,
+                 proxy_port=None, proxy_username=None, proxy_password='', timeout=20.0, redirect=True, gzip=False):
+        if headers is None:
+            headers = {}
+
+        self.url = url
+        self.method = method
+        self.headers = headers
+
+        self.cookies = cookies
+
+        self.params = params
+
+        self.upload = upload
+        self.download = download
+        self.progress = progress
+
+        self.auth_username = auth_username
+        self.auth_password = auth_password
+
+        self.proxy_protocol = proxy_protocol
+        self.proxy_host = proxy_host
+        self.proxy_port = proxy_port
+        self.proxy_username = proxy_username
+        self.proxy_password = proxy_password
+
+        self.timeout = timeout
+
+        self.redirect = redirect
+
+        self.gzip = gzip
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, ','.join('%s=%r' % i for i in self.__dict__.iteritems()))
+
+
+class HTTPResponse:
+    def __init__(self, request):
+        self.request = request
+        self.code = None
+        self.headers = {}
+        self.error = None
+        self.body = None
+        self.filename = None
+        self.time = time.time()
+
+    def __repr__(self):
+        args = ','.join('%s=%r' % i for i in self.__dict__.iteritems() if i[0] != 'body')
+        if self.body:
+            args += ',body=<data>'
+        else:
+            args += ',body=None'
+        return '%s(%s)' % (self.__class__.__name__, args)

resources/scrapers/kinopoisk/LOGGER.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+#
+try:
+    import xbmcaddon
+
+    __settings__ = xbmcaddon.Addon("plugin.video.torrenter")
+    debug = __settings__.getSetting("debug")
+except:
+    debug = 'true'
+
+
+def Log(msg, force=False):
+    try:
+        print "[torrenter log] " + msg
+    except UnicodeEncodeError:
+        print "[torrenter log] " + msg.encode("utf-8", "ignore")
+
+
+def Debug(msg, force=False):
+    if debug == 'true' or force:
+        try:
+            print "[torrenter] " + msg
+        except UnicodeEncodeError:
+            print "[torrenter] " + msg.encode("utf-8", "ignore")
+
+
+def Info(msg, force=False):
+    if debug == 'true' or force:
+        try:
+            print "[torrenter] " + msg
+        except UnicodeEncodeError:
+            print "[torrenter] " + msg.encode("utf-8", "ignore")
+
+
+def Warn(msg, force=False):
+    if debug == 'true' or force:
+        try:
+            print "[torrenter] " + msg
+        except UnicodeEncodeError:
+            print "[torrenter] " + msg.encode("utf-8", "ignore")

resources/scrapers/kinopoisk/__init__.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+#
+# Russian metadata plugin for Plex, which uses http://www.kinopoisk.ru/ to get the tag data.
+# Плагин для обновления информации о фильмах использующий КиноПоиск (http://www.kinopoisk.ru/).
+# Copyright (C) 2012  Yevgeny Nyden
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# @author zhenya (Yevgeny Nyden)
+# @version 1.52
+# @revision 148
+
+import common
+import pageparser
+import LOGGER
+import pluginsettings as S
+from translit import provide_unicode
+
+
+IS_DEBUG = True  # TODO - DON'T FORGET TO SET IT TO FALSE FOR A DISTRO.
+
+# Plugin preferences.
+# When changing default values here, also update the DefaultPrefs.json file.
+PREFS = common.Preferences(
+    ('kinopoisk_pref_max_posters', S.KINOPOISK_PREF_DEFAULT_MAX_POSTERS),
+    ('kinopoisk_pref_max_art', S.KINOPOISK_PREF_DEFAULT_MAX_ART),
+    ('kinopoisk_pref_get_all_actors', S.KINOPOISK_PREF_DEFAULT_GET_ALL_ACTORS),
+    ('kinopoisk_pref_imdb_support', S.KINOPOISK_PREF_DEFAULT_IMDB_SUPPORT),
+    (None, None),
+    ('kinopoisk_pref_imdb_rating', S.KINOPOISK_PREF_DEFAULT_IMDB_RATING),
+    ('kinopoisk_pref_kp_rating', S.KINOPOISK_PREF_DEFAULT_KP_RATING))
+
+
+def Start():
+    LOGGER.Info('***** START ***** %s' % common.USER_AGENT)
+    PREFS.readPluginPreferences()
+
+
+def ValidatePrefs():
+    LOGGER.Info('***** updating preferences...')
+    PREFS.readPluginPreferences()
+
+
+class KinoPoiskRuAgent():
+    name = 'KinoPoiskRu'
+    primary_provider = True
+    fallback_agent = False
+    accepts_from = ['com.plexapp.agents.localmedia']
+    contributes_to = None
+    parser = pageparser.PageParser(
+        LOGGER, common.HttpUtils(S.ENCODING_KINOPOISK_PAGE, pageparser.USER_AGENT), IS_DEBUG)
+
+
+    # #############################################################################
+    ############################# S E A R C H ####################################
+    ##############################################################################
+    def search(self, results, media, lang, manual=False):
+        """ Searches for matches on KinoPoisk using the title and year
+            passed via the media object. All matches are saved in a list of results
+            as MetadataSearchResult objects. For each results, we determine a
+            page id, title, year, and the score (how good we think the match
+            is on the scale of 1 - 100).
+        """
+        LOGGER.Debug('SEARCH START <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<')
+        try:
+            mediaName = media['name'].decode('utf-8')
+        except:
+            mediaName = media['name']
+        mediaYear = media['year']
+        LOGGER.Debug('searching for name="%s", year="%s"...' %
+                     (mediaName, str(mediaYear)))
+
+        # Look for matches on KinoPisk (result is returned as an array of tuples [kinoPoiskId, title, year, score]).
+        titleResults = KinoPoiskRuAgent.parser.fetchAndParseSearchResults(mediaName, mediaYear)
+        for titleResult in titleResults:
+            results.append((titleResult[0], titleResult[1], titleResult[2], lang, titleResult[3]))
+
+        # Sort results according to their score (Сортируем результаты).
+        LOGGER.Debug('SEARCH END <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<')
+        return results
+
+        #k=KinoPoiskRuAgent()
+
+        #print str(k.search([],{'name':'Django Unchained','year':'2012'},'English'))

resources/scrapers/kinopoisk/common.py

@@ -0,0 +1,382 @@
+# -*- coding: utf-8 -*-
+#
+# Russian metadata plugin for Plex, which uses http://www.kinopoisk.ru/ to get the tag data.
+# Плагин для обновления информации о фильмах использующий КиноПоиск (http://www.kinopoisk.ru/).
+# Copyright (C) 2012 Yevgeny Nyden
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+# @author zhenya (Yevgeny Nyden)
+# @revision 148
+
+import sys
+import math
+import difflib
+
+import translit
+import LOGGER as Log
+from HTTP import *
+
+
+USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.172 Safari/537.22'
+# USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.51.22 (KHTML, like Gecko) Version/5.1.1 Safari/534.51.22'
+ENCODING_PLEX = 'utf-8'
+
+SCORE_PENALTY_ITEM_ORDER = 2
+SCORE_PENALTY_YEAR = 30
+SCORE_PENALTY_TITLE = 40
+
+IMAGE_SCORE_MAX_NUMBER_OF_ITEMS = 5
+IMAGE_SCORE_ITEM_ORDER_BONUS_MAX = 25
+IMAGE_SCORE_RESOLUTION_BONUS_MAX = 25
+IMAGE_SCORE_RATIO_BONUS_MAX = 45
+IMAGE_SCORE_THUMB_BONUS = 5
+POSTER_SCORE_MIN_RESOLUTION_PX = 60 * 1000
+POSTER_SCORE_MAX_RESOLUTION_PX = 600 * 1000
+POSTER_SCORE_BEST_RATIO = 0.7
+ART_SCORE_BEST_RATIO = 1.5
+ART_SCORE_MIN_RESOLUTION_PX = 200 * 1000
+ART_SCORE_MAX_RESOLUTION_PX = 1000 * 1000
+
+
+class Preferences:
+    """ These instance variables are populated from plugin preferences.
+    """
+
+    def __init__(self,
+                 (maxPostersName, maxPostersDefault),
+                 (maxArtName, maxArtDefault),
+                 (getAllActorsName, getAllActorsDefault),
+                 (imdbSupportName, imdbSupportDefault),
+                 (cacheTimeName, cacheTimeDefault),
+                 (imdbRatingName, imdbRatingDefault),
+                 (additionalRatingName, additionalRatingDefault)):
+        self.maxPostersName = maxPostersName
+        self.maxPosters = maxPostersDefault
+        self.maxArtName = maxArtName
+        self.maxArt = maxArtDefault
+        self.getAllActorsName = getAllActorsName
+        self.getAllActors = getAllActorsDefault
+        self.imdbSupportName = imdbSupportName
+        self.imdbSupport = imdbSupportDefault
+        self.cacheTimeName = cacheTimeName
+        self.cacheTime = cacheTimeDefault
+        self.cacheTimeDefault = cacheTimeDefault
+        self.imdbRatingName = imdbRatingName
+        self.imdbRating = imdbRatingDefault
+        self.additionalRatingName = additionalRatingName
+        self.additionalRating = additionalRatingDefault
+
+    def readPluginPreferences(self):
+        # Setting image (poster and funart) preferences.
+        if self.maxPostersName is not None:
+            self.maxPosters = int(Prefs[self.maxPostersName])
+            Log.Debug('PREF: Max poster results is set to %d.' % self.maxPosters)
+        if self.maxArtName is not None:
+            self.maxArt = int(Prefs[self.maxArtName])
+            Log.Debug('PREF: Max art results is set to %d.' % self.maxArt)
+        if self.getAllActorsName is not None:
+            self.getAllActors = Prefs[self.getAllActorsName]
+            Log.Debug('PREF: Parse all actors is set to %s.' % str(self.getAllActors))
+
+        # Setting IMDB support.
+        if self.imdbSupportName is not None:
+            self.imdbSupport = Prefs[self.imdbSupportName]
+            Log.Debug('PREF: IMDB support is set to %s.' % str(self.imdbSupport))
+
+        # Setting cache expiration time.
+        if self.cacheTimeName is not None:
+            self.cacheTime = parseAndSetCacheTimeFromPrefs(self.cacheTimeName, self.cacheTimeDefault)
+
+        # Setting IMDB rating.
+        if self.imdbRatingName is not None:
+            self.imdbRating = Prefs[self.imdbRatingName]
+        Log.Debug('PREF: IMDB rating is set to %s.' % str(self.imdbRating))
+
+        # Setting kinopoisk.ru rating.
+        if self.additionalRatingName is not None:
+            self.additionalRating = Prefs[self.additionalRatingName]
+        Log.Debug('PREF: kinopoisk rating is set to %s.' % str(self.additionalRating))
+
+
+def getElementFromHttpRequest(url, encoding, userAgent=USER_AGENT):
+    """ Fetches a given URL and returns it as an element.
+        Функция преобразования html-кода в xml-код.
+    """
+    for i in range(3):
+        errorCount = 0
+        try:
+            req = HTTPRequest(url, headers={'User-agent': userAgent, 'Accept': 'text/html'})
+            response = HTTP().fetch(req)
+            return str(response.body).decode(encoding)
+        except:
+            errorCount += 1
+            Log.Debug('Error fetching URL: "%s".' % url)
+            time.sleep(1 + errorCount)
+    return None
+
+
+def requestImageJpeg(url, userAgent):
+    """ Requests an image given its URL and returns a request object.
+    """
+    try:
+        response = HTTP.Request(url, headers={
+            'User-agent': userAgent,
+            'Accept': 'image/jpeg'
+        })
+        return response
+    except:
+        Log.Debug('Error fetching URL: "%s".' % url)
+    return None
+
+
+def getWin1252ResponseFromHttpRequest(url):
+    """ Requests an image given its URL and returns a request object.
+    """
+    try:
+        response = HTTP.Request(url, headers={
+            'User-agent': USER_AGENT,
+            'Accept-Charset': 'ISO-8859-1;q=0.7,*;q=0.3',
+            'Accept-Language': 'en-US,en;q=0.8'
+        })
+        return response
+    except:
+        Log.Error('Error fetching URL: "%s".' % url)
+    return None
+
+
+def printSearchArrayResults(results):
+    """ Sends a list of media array results to debug log.
+    """
+    Log.Debug('Search produced %d results:' % len(results))
+    index = -1
+    for result in results:
+        index += 1
+        Log.Debug(' ... %d: id="%s", name="%s", year="%s", score="%d".' %
+                  (index, result[0], result[1], str(result[2]), result[3]))
+
+
+def printSearchResults(results):
+    """ Sends a list of media results to debug log.
+    """
+    Log.Debug('Search produced %d results:' % len(results))
+    index = 0
+    for result in results:
+        Log.Debug(' ... %d: id="%s", name="%s", year="%s", score="%d".' %
+                  (index, result.id, result.name, str(result.year), result.score))
+        index += 1
+
+
+def printImageSearchResults(thumbnailList):
+    Log.Debug('printing %d image results:' % len(thumbnailList))
+    index = 0
+    for result in thumbnailList:
+        Log.Debug(' ... %d: index=%s, score=%s, URL="%s".' %
+                  (index, result.index, result.score, result.url))
+        index += 1
+    return None
+
+
+def logException(msg):
+    excInfo = sys.exc_info()
+    Log.Exception('%s; exception: %s; cause: %s' % (msg, excInfo[0], excInfo[1]))
+
+
+def scoreMediaTitleMatch(mediaName, mediaYear, title, altTitle, year, itemIndex):
+    """ Compares page and media titles taking into consideration
+        media item's year and title values. Returns score [0, 100].
+        Search item scores 100 when:
+          - it's first on the list of results; AND
+          - it equals to the media title (ignoring case) OR all media title words are found in the search item; AND
+          - search item year equals to media year.
+
+        For now, our title scoring is pretty simple - we check if individual words
+        from media item's title are found in the title from search results.
+        We should also take into consideration order of words, so that "One Two" would not
+        have the same score as "Two One". Also, taking into consideration year difference.
+    """
+    # TODO(zhenya): add logging that works in unit tests.
+    #  Log.Debug('comparing item %d::: "%s-%s" with "%s-%s" (%s)...' %
+    #      (itemIndex, str(mediaName), str(mediaYear), str(title), str(year), str(altTitle)))
+    # Max score is when both title and year match exactly.
+    score = 100
+
+    # Item order penalty (the lower it is on the list or results, the larger the penalty).
+    score = score - (itemIndex * SCORE_PENALTY_ITEM_ORDER)
+
+    # Compute year penalty: [equal, diff>=3] --> [0, MAX].
+    yearPenalty = SCORE_PENALTY_YEAR
+    mediaYear = toInteger(mediaYear)
+    year = toInteger(year)
+    if mediaYear is not None and year is not None:
+        yearDiff = abs(mediaYear - year)
+        if not yearDiff:
+            yearPenalty = 0
+        elif yearDiff == 1:
+            yearPenalty = 15
+        elif yearDiff == 2:
+            yearPenalty = 25
+    score = score - yearPenalty
+    #print str(mediaYear)+' '+str(year)+' '+str(yearPenalty)
+
+    # Compute title penalty.
+    titlePenalty = computeTitlePenalty(mediaName, title)
+    altTitlePenalty = 100
+    if altTitle is not None:
+        altTitlePenalty = computeTitlePenalty(mediaName, altTitle)
+
+    # Get detranlitirated media name (in case filename is in latin characters),
+    # compare it's score with the original, and pick the min.
+    try:
+        altMediaName = translit.detranslify(mediaName)
+        detranslifiedTitlePenalty = computeTitlePenalty(altMediaName, title)
+        titlePenalty = min(detranslifiedTitlePenalty, titlePenalty)
+        if altTitle is not None:
+            detranslifiedAltTitlePenalty = computeTitlePenalty(altMediaName, altTitle)
+            altTitlePenalty = min(detranslifiedAltTitlePenalty, altTitlePenalty)
+    except:
+        pass
+
+    titlePenalty = min(titlePenalty, altTitlePenalty)
+    score = score - titlePenalty
+
+    # If the score is not high enough, add a few points to the first result -
+    # let's give KinoPoisk some credit :-).
+    if itemIndex == 0 and score <= 80:
+        score = score + 5
+
+    # IMPORTANT: always return an int.
+    score = int(score)
+    #  Log.Debug('***** title scored %d' % score)
+    return score
+
+
+def scoreThumbnailResult(thumb, isPoster):
+    """ Given a Thumbnail object that represents an poster or a funart result,
+        scores it, and stores the score on the passed object (thumb.score).
+    """
+    score = 0
+    if thumb.url is None:
+        thumb.score = 0
+        return
+
+    if thumb.index < IMAGE_SCORE_MAX_NUMBER_OF_ITEMS:
+        # Score bonus from index for items below 10 on the list.
+        bonus = IMAGE_SCORE_ITEM_ORDER_BONUS_MAX * \
+                ((IMAGE_SCORE_MAX_NUMBER_OF_ITEMS - thumb.index) / float(IMAGE_SCORE_MAX_NUMBER_OF_ITEMS))
+        score += bonus
+
+    if thumb.width is not None and thumb.height is not None:
+        # Get a resolution bonus if width*height is more than a certain min value.
+        if isPoster:
+            minPx = POSTER_SCORE_MIN_RESOLUTION_PX
+            maxPx = POSTER_SCORE_MAX_RESOLUTION_PX
+            bestRatio = POSTER_SCORE_BEST_RATIO
+        else:
+            minPx = ART_SCORE_MIN_RESOLUTION_PX
+            maxPx = ART_SCORE_MAX_RESOLUTION_PX
+            bestRatio = ART_SCORE_BEST_RATIO
+        pixelsCount = thumb.width * thumb.height
+        if pixelsCount > minPx:
+            if pixelsCount > maxPx:
+                pixelsCount = maxPx
+            bonus = float(IMAGE_SCORE_RESOLUTION_BONUS_MAX) * \
+                    float((pixelsCount - minPx)) / float((maxPx - minPx))
+            score += bonus
+
+        # Get an orientation (Portrait vs Landscape) bonus. (we prefer images that are have portrait orientation.
+        ratio = thumb.width / float(thumb.height)
+        ratioDiff = math.fabs(bestRatio - ratio)
+        if ratioDiff < 0.5:
+            bonus = IMAGE_SCORE_RATIO_BONUS_MAX * (0.5 - ratioDiff) * 2.0
+            score += bonus
+
+    # Get a bonus if image has a separate thumbnail URL.
+    if thumb.thumbUrl is not None and thumb.url != thumb.thumbUrl:
+        score += IMAGE_SCORE_THUMB_BONUS
+
+    thumb.score = int(score)
+
+
+def isAsciiString(mediaName):
+    """ Returns True if all characters of the string are ASCII.
+    """
+    for index, char in enumerate(mediaName):
+        if ord(char) >= 128:
+            return False
+    return True
+
+
+def toInteger(maybeNumber):
+    """ Returns the argument converted to an integer if it represents a number
+        or None if the argument is None or does not represent a number.
+    """
+    try:
+        if maybeNumber is not None and str(maybeNumber).strip() != '':
+            return int(maybeNumber)
+    except:
+        pass
+    return None
+
+
+def computeTitlePenalty(mediaName, title):
+    """ Given media name and a candidate title, returns the title result score penalty.
+        @param mediaName Movie title parsed from the file system.
+        @param title Movie title from the website.
+    """
+    mediaName = mediaName.lower()
+    title = title.lower()
+    if mediaName != title:
+        # First approximate the whole strings.
+        diffRatio = difflib.SequenceMatcher(None, mediaName, title).ratio()
+        penalty = int(SCORE_PENALTY_TITLE * (1 - diffRatio))
+
+        # If the penalty is more than 1/2 of max title penalty, check to see if
+        # this title starts with media name. This means that media name does not
+        # have the whole name of the movie - very common case. For example, media name
+        # "Кавказская пленница" for a movie title "Кавказская пленница, или Новые приключения Шурика".
+        if penalty >= 15:  # This is so that we don't have to do the 'split' every time.
+            # Compute the scores of the
+            # First, check if the title starts with media name.
+            mediaNameParts = mediaName.split()
+            titleParts = title.split()
+            if len(mediaNameParts) <= len(titleParts):
+                i = 0
+                # Start with some small penalty, value of which depends on how
+                # many words media name has relative to the title's word count.
+                penaltyAlt = max(5, int(round((1.0 - (float(len(mediaNameParts)) / len(titleParts))) * 15 - 5)))
+                penaltyPerPart = SCORE_PENALTY_TITLE / len(mediaNameParts)
+                for mediaNamePart in mediaNameParts:
+                    partDiffRatio = difflib.SequenceMatcher(None, mediaNamePart, titleParts[i]).ratio()
+                    penaltyAlt = penaltyAlt + int(penaltyPerPart * (1 - partDiffRatio))
+                    i = i + 1
+                penalty = min(penalty, penaltyAlt)
+            #    print '++++++ DIFF("%s", "%s") = %g --> %d' % (mediaName.encode('utf8'), title.encode('utf8'), diffRatio, penalty)
+            #    Log.Debug('++++++ DIFF("%s", "%s") = %g --> %d' % (mediaName.encode('utf8'), title.encode('utf8'), diffRatio, penalty))
+        return penalty
+    return 0
+
+
+class HttpUtils():
+    def __init__(self, encoding, userAgent):
+        self.encoding = encoding
+        self.userAgent = userAgent
+
+    def requestAndParseHtmlPage(self, url):
+        return getElementFromHttpRequest(url, self.encoding, self.userAgent)
+
+    def requestImageJpeg(self, url):
+        return requestImageJpeg(url, self.userAgent)

resources/scrapers/kinopoisk/pageparser.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+#
+# Russian metadata plugin for Plex, which uses http://www.kinopoisk.ru/ to get the tag data.
+# Плагин для обновления информации о фильмах использующий КиноПоиск (http://www.kinopoisk.ru/).
+# Copyright (C) 2013 Yevgeny Nyden
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+# @author zhenya (Yevgeny Nyden)
+# @version 1.52
+# @revision 148
+
+import re
+import urllib
+import operator
+
+import common
+import pluginsettings as S
+import translit
+
+
+# MATCHER_MOVIE_DURATION = re.compile('\s*(\d+).*?', re.UNICODE | re.DOTALL)
+#MATCHER_IMDB_RATING = re.compile('IMDb:\s*(\d+\.?\d*)\s*\(\s*([\s\d]+)\s*\)', re.UNICODE | re.DOTALL)
+#MATCHER_IMDB_RATING = re.compile('IMDb:\s*(\d+\.?\d*)\s?\((.*)\)', re.UNICODE)
+
+USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.172 Safari/537.22'
+'''
+MOVIE_THUMBNAIL_SMALL_WIDTH = 130
+MOVIE_THUMBNAIL_SMALL_HEIGHT = 168
+MOVIE_THUMBNAIL_BIG_WIDTH = 780
+MOVIE_THUMBNAIL_BIG_HEIGHT = 1024
+
+# Compiled regex matchers.
+MATCHER_WIDTH_FROM_STYLE = re.compile('.*width\s*:\s*(\d+)px.*', re.UNICODE)
+MATCHER_HEIGHT_FROM_STYLE = re.compile('.*height\s*:\s*(\d+)px.*', re.UNICODE)
+MATCHER_LEADING_NONALPHA = re.compile('^[\s\d\.\(\)]*', re.UNICODE | re.MULTILINE)
+
+
+# Русские месяца, пригодятся для определения дат.
+RU_MONTH = {
+  u'января': '01',
+  u'февраля': '02',
+  u'марта': '03',
+  u'апреля': '04',
+  u'мая': '05',
+  u'июня': '06',
+  u'июля': '07',
+  u'августа': '08',
+  u'сентября': '09',
+  u'октября': '10',
+  u'ноября': '11',
+  u'декабря': '12'
+}
+'''
+
+
+class PageParser:
+    def __init__(self, logger, httpUtils=common.HttpUtils(S.ENCODING_KINOPOISK_PAGE, USER_AGENT), isDebug=False):
+        self.log = logger
+        self.isDebug = isDebug
+        self.httpUtils = httpUtils
+
+    def fetchAndParseSearchResults(self, mediaName, mediaYear, mediaAltName=None):
+        """ Searches for movie titles on KinoPoisk.
+            @param mediaName Movie title parsed from a filename.
+            @param mediaName Movie year parsed from a filename.
+            @return Array of tuples: [kinoPoiskId, title, year, score]
+        """
+        self.log.Info('Quering kinopoisk...')
+        results = self.queryKinoPoisk(mediaName, mediaYear)
+
+        # Check media name is all ASCII characters, and if it is,
+        # issue another query to KinoPoisk using a translified media name;
+        # lastly, merge the scored results.
+        if common.isAsciiString(mediaName):
+            translifiedMediaName = translit.detranslify(mediaName)
+            moreResults = self.queryKinoPoisk(translifiedMediaName, mediaYear)
+            resultsMap = dict()
+            for result in results:
+                resultsMap[result[0]] = result
+            results = []  # Recreate and repopulate the results array removing duplicates.
+            for result in moreResults:
+                currId = result[0]
+                if currId in resultsMap.keys():
+                    origResult = resultsMap[currId]
+                    del resultsMap[currId]
+                    if result[3] >= origResult[3]:
+                        results.append(result)
+                    else:
+                        results.append(origResult)
+                else:
+                    results.append(result)
+            results.extend(resultsMap.viewvalues())
+
+        if mediaAltName:
+            moreResults = self.queryKinoPoisk(mediaAltName, mediaYear)
+            resultsMap = dict()
+            for result in results:
+                resultsMap[result[0]] = result
+            results = []  # Recreate and repopulate the results array removing duplicates.
+            for result in moreResults:
+                currId = result[0]
+                if currId in resultsMap.keys():
+                    origResult = resultsMap[currId]
+                    del resultsMap[currId]
+                    if result[3] >= origResult[3]:
+                        results.append(result)
+                    else:
+                        results.append(origResult)
+                else:
+                    results.append(result)
+            results.extend(resultsMap.viewvalues())
+
+        # Sort all results based on their score.
+        results.sort(key=operator.itemgetter(3))
+        results.reverse()
+        if self.isDebug:
+            self.log.Debug('Search produced %d results:' % len(results))
+            index = -1
+            for result in results:
+                index += 1
+                self.log.Debug(' ... %d: id="%s", name="%s", year="%s", score="%d".' %
+                               (index, result[0], result[1], str(result[2]), result[3]))
+        return results
+
+    def queryKinoPoisk(self, mediaName, mediaYear):
+        """ Ищет фильм на кинопоиске.
+            Returns title results as they are returned (no sorting is done here!).
+        """
+        results = []
+        try:
+            encodedName = urllib.quote(mediaName.encode(S.ENCODING_KINOPOISK_PAGE))
+        except:
+            encodedName = urllib.quote(mediaName.encode('utf-8'))
+        page = self.httpUtils.requestAndParseHtmlPage(S.KINOPOISK_SEARCH_SIMPLE % encodedName)
+        if page is None:
+            self.log.Warn(' ### nothing was found on kinopoisk for media name "%s"' % mediaName)
+            return results
+
+        # Страница получена, берем с нее перечень всех названий фильмов.
+        self.log.Debug('**** Finding %s (%s) got a KinoPoisk...' % (mediaName, mediaYear))
+
+        reobj = re.compile(r'<span><a href="http://m\.kinopoisk\.ru/movie/(\d+)/">(.+?)</a><br />(.+?)</span>')
+        result = reobj.findall(page)
+
+        # Inspect query results titles and score them.
+        self.log.Debug('found %d results (div info tags)' % len(result))
+        itemIndex = -1
+        for itemKinoPoiskId, itemTitleitemYear, itemAltTitle in result:
+            itemIndex = itemIndex + 1
+            itemAltTitle = itemAltTitle.replace('&nbsp;', '')
+            try:
+                itemTitle, itemYear = re.compile('^(.+?), (\d\d\d\d)$').findall(itemTitleitemYear.strip())[0]
+            except:
+                itemYear = None
+                itemTitle = itemTitleitemYear
+            itemScore = common.scoreMediaTitleMatch(mediaName, mediaYear, itemTitle, itemAltTitle, itemYear, itemIndex)
+            results.append([itemKinoPoiskId, itemTitle, itemYear, itemScore])
+
+        return results
+
+

resources/scrapers/kinopoisk/pluginsettings.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+#
+# Global definitions for KinoPoiskRu plugin.
+# Copyright (C) 2013 Yevgeny Nyden
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+# @author zhenya (Yevgeny Nyden)
+# @version 1.52
+# @revision 148
+
+# Default plugin preferences. When modifying, please also change
+# corresponding values in the ../DefaultPrefs.json file.
+KINOPOISK_PREF_DEFAULT_MAX_POSTERS = 1
+KINOPOISK_PREF_DEFAULT_MAX_ART = 2
+KINOPOISK_PREF_DEFAULT_GET_ALL_ACTORS = False
+KINOPOISK_PREF_DEFAULT_IMDB_SUPPORT = True
+KINOPOISK_PREF_DEFAULT_IMDB_RATING = False
+KINOPOISK_PREF_DEFAULT_KP_RATING = False
+
+ENCODING_KINOPOISK_PAGE = 'cp1251'
+
+# Разные страницы сайта.
+KINOPOISK_SITE_BASE = 'http://www.kinopoisk.ru/'
+KINOPOISK_RESOURCE_BASE = 'http://st.kinopoisk.ru/'
+KINOPOISK_TITLE_PAGE_URL = KINOPOISK_SITE_BASE + 'film/%s/'
+KINOPOISK_CAST_PAGE_URL = KINOPOISK_SITE_BASE + 'film/%s/cast/'
+KINOPOISK_STUDIO_PAGE_URL = KINOPOISK_SITE_BASE + 'film/%s/studio/'
+KINOPOISK_THUMBNAIL_BIG_URL = KINOPOISK_RESOURCE_BASE + 'images/film_big/%s.jpg'
+KINOPOISK_THUMBNAIL_SMALL_URL = KINOPOISK_RESOURCE_BASE + 'images/film/%s.jpg'
+KINOPOISK_POSTERS_URL = KINOPOISK_SITE_BASE + 'film/%s/posters/page/%d/'
+KINOPOISK_STILLS_URL = KINOPOISK_SITE_BASE + 'film/%s/stills/page/%d/'
+
+# Страница поиска.
+KINOPOISK_SEARCH = KINOPOISK_SITE_BASE + 'index.php?first=no&kp_query=%s'
+KINOPOISK_SEARCH_SIMPLE = 'http://m.kinopoisk.ru/search/%s/'

resources/scrapers/kinopoisk/translit.py

@@ -0,0 +1,288 @@
+# -*- coding: utf-8 -*-
+# -*- test-case-name: pytils.test.test_translit -*-
+# pytils - simple processing for russian strings
+# Copyright (C) 2006-2007  Yury Yurevich
+#
+# http://www.pyobject.ru/projects/pytils/
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation, version 2
+# of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+"""
+Simple transliteration
+"""
+
+# __id__ = __revision__ = "$Id: translit.py 102 2007-07-12 12:33:36Z the.pythy $"
+#__url__ = "$URL: https://pythy.googlecode.com/svn/tags/pytils/0_2_2/pytils/translit.py $"
+
+import re
+import sys
+
+
+TRANSTABLE = (
+    (u"'", u"'"),
+    (u'"', u'"'),
+    (u"‘", u"'"),
+    (u"’", u"'"),
+    (u"«", u'"'),
+    (u"»", u'"'),
+    (u"–", u"-"),
+    (u"…", u"..."),
+    (u"№", u"#"),
+    ## верхний регистр
+    # трехбуквенные замены
+    (u"Щ", u"Sch"),
+    # при замене русский->английский будет первая замена,
+    # т.е. Sch
+    # а вот если английский->русский, то вариант SCH и Sch --
+    # оба пройдут
+    (u"Щ", u"SCH"),
+    # двухбуквенные замены
+    (u"Ё", u"Yo"),
+    (u"Ё", u"YO"),
+    (u"Ж", u"Zh"),
+    (u"Ж", u"ZH"),
+    (u"Ц", u"Ts"),
+    (u"Ц", u"TS"),
+    (u"Ч", u"Ch"),
+    (u"Ч", u"CH"),
+    (u"Ш", u"Sh"),
+    (u"Ш", u"SH"),
+    (u"Ы", u"Yi"),
+    (u"Ы", u"YI"),
+    (u"Ю", u"Yu"),
+    (u"Ю", u"YU"),
+    (u"Я", u"Ya"),
+    (u"Я", u"YA"),
+    (u"ИЙ", u"IY"),
+    # однобуквенные замены
+    (u"А", u"A"),
+    (u"Б", u"B"),
+    (u"В", u"V"),
+    (u"Г", u"G"),
+    (u"Д", u"D"),
+    (u"Е", u"E"),
+    (u"З", u"Z"),
+    (u"И", u"I"),
+    (u"Й", u"J"),
+    (u"К", u"K"),
+    (u"Л", u"L"),
+    (u"М", u"M"),
+    (u"Н", u"N"),
+    (u"О", u"O"),
+    (u"П", u"P"),
+    (u"Р", u"R"),
+    (u"С", u"S"),
+    (u"Т", u"T"),
+    (u"У", u"U"),
+    (u"Ф", u"F"),
+    (u"Х", u"H"),
+    (u"Э", u"E"),
+    (u"Ъ", u"`"),
+    (u"Ы", u"Y"),
+    (u"Ь", u"'"),
+    ## нижний регистр
+    # трехбуквенные замены
+    (u"щ", u"sch"),
+    # двухбуквенные замены
+    (u"ё", u"yo"),
+    (u"ж", u"zh"),
+    (u"ц", u"ts"),
+    (u"ч", u"ch"),
+    (u"ш", u"sh"),
+    (u"ы", u"yi"),
+    (u"ю", u"yu"),
+    (u"я", u"ya"),
+    (u"я", u"ja"),
+    (u"ий", u"iy"),
+    # однобуквенные замены
+    (u"а", u"a"),
+    (u"б", u"b"),
+    (u"в", u"v"),
+    (u"г", u"g"),
+    (u"д", u"d"),
+    (u"е", u"e"),
+    (u"з", u"z"),
+    (u"и", u"i"),
+    (u"й", u"j"),
+    (u"к", u"k"),
+    (u"л", u"l"),
+    (u"м", u"m"),
+    (u"н", u"n"),
+    (u"о", u"o"),
+    (u"п", u"p"),
+    (u"р", u"r"),
+    (u"с", u"s"),
+    (u"т", u"t"),
+    (u"у", u"u"),
+    (u"ф", u"f"),
+    (u"х", u"h"),
+    (u"э", u"e"),
+    (u"ъ", u"`"),
+    (u"ь", u"'"),
+    # для полноты английского алфавит (в slugify)
+    # дополняем английскими буквами, которых
+    # не в парах
+    (u"c", u"c"),
+    (u"q", u"q"),
+    (u"y", u"y"),
+    (u"x", u"x"),
+    (u"w", u"w"),
+    (u"1", u"1"),
+    (u"2", u"2"),
+    (u"3", u"3"),
+    (u"4", u"4"),
+    (u"5", u"5"),
+    (u"6", u"6"),
+    (u"7", u"7"),
+    (u"8", u"8"),
+    (u"9", u"9"),
+    (u"0", u"0"),
+)  #: Translation table
+
+RU_ALPHABET = [x[0] for x in TRANSTABLE]  #: Russian alphabet that we can translate
+EN_ALPHABET = [x[1] for x in TRANSTABLE]  #: English alphabet that we can detransliterate
+ALPHABET = RU_ALPHABET + EN_ALPHABET  #: Alphabet that we can (de)transliterate
+
+
+def translify(in_string):
+    """
+    Translify russian text
+
+    @param in_string: input string
+    @type in_string: C{unicode}
+
+    @return: transliterated string
+    @rtype: C{str}
+
+    @raise TypeError: when in_string is not C{unicode}
+    @raise ValueError: when string doesn't transliterate completely
+    """
+    if not isinstance(in_string, unicode):
+        raise TypeError("Argument must be unicode, not %s" % type(in_string))
+
+    translit = in_string
+    for symb_in, symb_out in TRANSTABLE:
+        translit = translit.replace(symb_in, symb_out)
+
+    try:
+        translit = str(translit)
+    except UnicodeEncodeError:
+        raise ValueError("Unicode string doesn't transliterate completely, " + \
+                         "is it russian?")
+
+    return translit
+
+
+def detranslify(in_string):
+    """
+    Detranslify
+
+    @param in_string: input string
+    @type in_string: C{basestring}
+
+    @return: detransliterated string
+    @rtype: C{str}
+
+    @raise TypeError: when in_string neither C{str}, no C{unicode}
+    @raise ValueError: if in_string is C{str}, but it isn't ascii
+    """
+    if not isinstance(in_string, basestring):
+        raise TypeError("Argument must be basestring, not %s" % type(in_string))
+
+    # в unicode
+    try:
+        russian = unicode(in_string)
+    except UnicodeDecodeError:
+        raise ValueError("We expects when in_string is str type," + \
+                         "it is an ascii, but now it isn't. Use unicode " + \
+                         "in this case.")
+
+    for symb_out, symb_in in TRANSTABLE:
+        russian = russian.replace(symb_in, symb_out)
+
+    return russian
+
+
+def slugify(in_string):
+    """
+    Prepare string for slug (i.e. URL or file/dir name)
+
+    @param in_string: input string
+    @type in_string: C{basestring}
+
+    @return: slug-string
+    @rtype: C{str}
+
+    @raise TypeError: when in_string isn't C{unicode} or C{str}
+    @raise ValueError: if in_string is C{str}, but it isn't ascii
+    """
+    if not isinstance(in_string, basestring):
+        raise TypeError("Argument must be basestring, not %s" % type(in_string))
+    try:
+        u_in_string = unicode(in_string).lower()
+    except UnicodeDecodeError:
+        raise ValueError("We expects when in_string is str type," + \
+                         "it is an ascii, but now it isn't. Use unicode " + \
+                         "in this case.")
+    # convert & to "and"
+    u_in_string = re.sub('\&amp\;|\&', ' and ', u_in_string)
+    # replace spaces by hyphen
+    u_in_string = re.sub('[-\s]+', '-', u_in_string)
+    # remove symbols that not in alphabet
+    u_in_string = u''.join([symb for symb in u_in_string if symb in ALPHABET])
+    # translify it
+    out_string = translify(u_in_string)
+    # remove non-alpha
+    return re.sub('[^\w\s-]', '', out_string).strip().lower()
+
+
+def dirify(in_string):
+    """
+    Alias for L{slugify}
+    """
+    slugify(in_string)
+
+
+def provide_unicode(stext, encoding, default=u"неизвестно"):
+    """
+    Provide Unicode from text
+
+    @param stext: text
+    @type stext: C{str}
+
+    @param encoding: encoding if input text
+    @type encoding: C{str}
+
+    @return: C{unicode}
+    """
+    try:
+        utext = str(stext).decode(encoding)
+    except UnicodeDecodeError, err:
+        utext = default % {'error': err, 'value': u""}
+    return utext
+
+
+def provide_str(utext, encoding, default="unknown"):
+    """
+    Provide text from Unicode
+
+    @param utext: unicode text
+    @type utext: C{unicode}
+
+    @param encoding: encoding of output text
+    @type encoding: C{str}
+
+    @return: C{str}
+    """
+    try:
+        stext = unicode(utext).encode(encoding)
+    except UnicodeEncodeError, err:
+        stext = default % {'error': err, 'value': ""}
+    return stext