383 lines
15 KiB
Python
383 lines
15 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
#
|
|||
|
|
# Russian metadata plugin for Plex, which uses http://www.kinopoisk.ru/ to get the tag data.
|
|||
|
|
# Плагин для обновления информации о фильмах использующий КиноПоиск (http://www.kinopoisk.ru/).
|
|||
|
|
# Copyright (C) 2012 Yevgeny Nyden
|
|||
|
|
#
|
|||
|
|
# This program is free software; you can redistribute it and/or
|
|||
|
|
# modify it under the terms of the GNU General Public License
|
|||
|
|
# as published by the Free Software Foundation; either version 2
|
|||
|
|
# of the License, or (at your option) any later version.
|
|||
|
|
#
|
|||
|
|
# This program is distributed in the hope that it will be useful,
|
|||
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|||
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|||
|
|
# GNU General Public License for more details.
|
|||
|
|
#
|
|||
|
|
# You should have received a copy of the GNU General Public License
|
|||
|
|
# along with this program; if not, write to the Free Software
|
|||
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|||
|
|
# 02110-1301, USA.
|
|||
|
|
#
|
|||
|
|
# @author zhenya (Yevgeny Nyden)
|
|||
|
|
# @revision 148
|
|||
|
|
|
|||
|
|
import sys
|
|||
|
|
import math
|
|||
|
|
import difflib
|
|||
|
|
|
|||
|
|
import translit
|
|||
|
|
import LOGGER as Log
|
|||
|
|
from HTTP import *
|
|||
|
|
|
|||
|
|
|
|||
|
|
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.172 Safari/537.22'
|
|||
|
|
# USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.51.22 (KHTML, like Gecko) Version/5.1.1 Safari/534.51.22'
|
|||
|
|
ENCODING_PLEX = 'utf-8'
|
|||
|
|
|
|||
|
|
SCORE_PENALTY_ITEM_ORDER = 2
|
|||
|
|
SCORE_PENALTY_YEAR = 30
|
|||
|
|
SCORE_PENALTY_TITLE = 40
|
|||
|
|
|
|||
|
|
IMAGE_SCORE_MAX_NUMBER_OF_ITEMS = 5
|
|||
|
|
IMAGE_SCORE_ITEM_ORDER_BONUS_MAX = 25
|
|||
|
|
IMAGE_SCORE_RESOLUTION_BONUS_MAX = 25
|
|||
|
|
IMAGE_SCORE_RATIO_BONUS_MAX = 45
|
|||
|
|
IMAGE_SCORE_THUMB_BONUS = 5
|
|||
|
|
POSTER_SCORE_MIN_RESOLUTION_PX = 60 * 1000
|
|||
|
|
POSTER_SCORE_MAX_RESOLUTION_PX = 600 * 1000
|
|||
|
|
POSTER_SCORE_BEST_RATIO = 0.7
|
|||
|
|
ART_SCORE_BEST_RATIO = 1.5
|
|||
|
|
ART_SCORE_MIN_RESOLUTION_PX = 200 * 1000
|
|||
|
|
ART_SCORE_MAX_RESOLUTION_PX = 1000 * 1000
|
|||
|
|
|
|||
|
|
|
|||
|
|
class Preferences:
|
|||
|
|
""" These instance variables are populated from plugin preferences.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self,
|
|||
|
|
(maxPostersName, maxPostersDefault),
|
|||
|
|
(maxArtName, maxArtDefault),
|
|||
|
|
(getAllActorsName, getAllActorsDefault),
|
|||
|
|
(imdbSupportName, imdbSupportDefault),
|
|||
|
|
(cacheTimeName, cacheTimeDefault),
|
|||
|
|
(imdbRatingName, imdbRatingDefault),
|
|||
|
|
(additionalRatingName, additionalRatingDefault)):
|
|||
|
|
self.maxPostersName = maxPostersName
|
|||
|
|
self.maxPosters = maxPostersDefault
|
|||
|
|
self.maxArtName = maxArtName
|
|||
|
|
self.maxArt = maxArtDefault
|
|||
|
|
self.getAllActorsName = getAllActorsName
|
|||
|
|
self.getAllActors = getAllActorsDefault
|
|||
|
|
self.imdbSupportName = imdbSupportName
|
|||
|
|
self.imdbSupport = imdbSupportDefault
|
|||
|
|
self.cacheTimeName = cacheTimeName
|
|||
|
|
self.cacheTime = cacheTimeDefault
|
|||
|
|
self.cacheTimeDefault = cacheTimeDefault
|
|||
|
|
self.imdbRatingName = imdbRatingName
|
|||
|
|
self.imdbRating = imdbRatingDefault
|
|||
|
|
self.additionalRatingName = additionalRatingName
|
|||
|
|
self.additionalRating = additionalRatingDefault
|
|||
|
|
|
|||
|
|
def readPluginPreferences(self):
|
|||
|
|
# Setting image (poster and funart) preferences.
|
|||
|
|
if self.maxPostersName is not None:
|
|||
|
|
self.maxPosters = int(Prefs[self.maxPostersName])
|
|||
|
|
Log.Debug('PREF: Max poster results is set to %d.' % self.maxPosters)
|
|||
|
|
if self.maxArtName is not None:
|
|||
|
|
self.maxArt = int(Prefs[self.maxArtName])
|
|||
|
|
Log.Debug('PREF: Max art results is set to %d.' % self.maxArt)
|
|||
|
|
if self.getAllActorsName is not None:
|
|||
|
|
self.getAllActors = Prefs[self.getAllActorsName]
|
|||
|
|
Log.Debug('PREF: Parse all actors is set to %s.' % str(self.getAllActors))
|
|||
|
|
|
|||
|
|
# Setting IMDB support.
|
|||
|
|
if self.imdbSupportName is not None:
|
|||
|
|
self.imdbSupport = Prefs[self.imdbSupportName]
|
|||
|
|
Log.Debug('PREF: IMDB support is set to %s.' % str(self.imdbSupport))
|
|||
|
|
|
|||
|
|
# Setting cache expiration time.
|
|||
|
|
if self.cacheTimeName is not None:
|
|||
|
|
self.cacheTime = parseAndSetCacheTimeFromPrefs(self.cacheTimeName, self.cacheTimeDefault)
|
|||
|
|
|
|||
|
|
# Setting IMDB rating.
|
|||
|
|
if self.imdbRatingName is not None:
|
|||
|
|
self.imdbRating = Prefs[self.imdbRatingName]
|
|||
|
|
Log.Debug('PREF: IMDB rating is set to %s.' % str(self.imdbRating))
|
|||
|
|
|
|||
|
|
# Setting kinopoisk.ru rating.
|
|||
|
|
if self.additionalRatingName is not None:
|
|||
|
|
self.additionalRating = Prefs[self.additionalRatingName]
|
|||
|
|
Log.Debug('PREF: kinopoisk rating is set to %s.' % str(self.additionalRating))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def getElementFromHttpRequest(url, encoding, userAgent=USER_AGENT):
|
|||
|
|
""" Fetches a given URL and returns it as an element.
|
|||
|
|
Функция преобразования html-кода в xml-код.
|
|||
|
|
"""
|
|||
|
|
for i in range(3):
|
|||
|
|
errorCount = 0
|
|||
|
|
try:
|
|||
|
|
req = HTTPRequest(url, headers={'User-agent': userAgent, 'Accept': 'text/html'})
|
|||
|
|
response = HTTP().fetch(req)
|
|||
|
|
return str(response.body).decode(encoding)
|
|||
|
|
except:
|
|||
|
|
errorCount += 1
|
|||
|
|
Log.Debug('Error fetching URL: "%s".' % url)
|
|||
|
|
time.sleep(1 + errorCount)
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def requestImageJpeg(url, userAgent):
|
|||
|
|
""" Requests an image given its URL and returns a request object.
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
response = HTTP.Request(url, headers={
|
|||
|
|
'User-agent': userAgent,
|
|||
|
|
'Accept': 'image/jpeg'
|
|||
|
|
})
|
|||
|
|
return response
|
|||
|
|
except:
|
|||
|
|
Log.Debug('Error fetching URL: "%s".' % url)
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def getWin1252ResponseFromHttpRequest(url):
|
|||
|
|
""" Requests an image given its URL and returns a request object.
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
response = HTTP.Request(url, headers={
|
|||
|
|
'User-agent': USER_AGENT,
|
|||
|
|
'Accept-Charset': 'ISO-8859-1;q=0.7,*;q=0.3',
|
|||
|
|
'Accept-Language': 'en-US,en;q=0.8'
|
|||
|
|
})
|
|||
|
|
return response
|
|||
|
|
except:
|
|||
|
|
Log.Error('Error fetching URL: "%s".' % url)
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def printSearchArrayResults(results):
|
|||
|
|
""" Sends a list of media array results to debug log.
|
|||
|
|
"""
|
|||
|
|
Log.Debug('Search produced %d results:' % len(results))
|
|||
|
|
index = -1
|
|||
|
|
for result in results:
|
|||
|
|
index += 1
|
|||
|
|
Log.Debug(' ... %d: id="%s", name="%s", year="%s", score="%d".' %
|
|||
|
|
(index, result[0], result[1], str(result[2]), result[3]))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def printSearchResults(results):
|
|||
|
|
""" Sends a list of media results to debug log.
|
|||
|
|
"""
|
|||
|
|
Log.Debug('Search produced %d results:' % len(results))
|
|||
|
|
index = 0
|
|||
|
|
for result in results:
|
|||
|
|
Log.Debug(' ... %d: id="%s", name="%s", year="%s", score="%d".' %
|
|||
|
|
(index, result.id, result.name, str(result.year), result.score))
|
|||
|
|
index += 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def printImageSearchResults(thumbnailList):
|
|||
|
|
Log.Debug('printing %d image results:' % len(thumbnailList))
|
|||
|
|
index = 0
|
|||
|
|
for result in thumbnailList:
|
|||
|
|
Log.Debug(' ... %d: index=%s, score=%s, URL="%s".' %
|
|||
|
|
(index, result.index, result.score, result.url))
|
|||
|
|
index += 1
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def logException(msg):
|
|||
|
|
excInfo = sys.exc_info()
|
|||
|
|
Log.Exception('%s; exception: %s; cause: %s' % (msg, excInfo[0], excInfo[1]))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scoreMediaTitleMatch(mediaName, mediaYear, title, altTitle, year, itemIndex):
|
|||
|
|
""" Compares page and media titles taking into consideration
|
|||
|
|
media item's year and title values. Returns score [0, 100].
|
|||
|
|
Search item scores 100 when:
|
|||
|
|
- it's first on the list of results; AND
|
|||
|
|
- it equals to the media title (ignoring case) OR all media title words are found in the search item; AND
|
|||
|
|
- search item year equals to media year.
|
|||
|
|
|
|||
|
|
For now, our title scoring is pretty simple - we check if individual words
|
|||
|
|
from media item's title are found in the title from search results.
|
|||
|
|
We should also take into consideration order of words, so that "One Two" would not
|
|||
|
|
have the same score as "Two One". Also, taking into consideration year difference.
|
|||
|
|
"""
|
|||
|
|
# TODO(zhenya): add logging that works in unit tests.
|
|||
|
|
# Log.Debug('comparing item %d::: "%s-%s" with "%s-%s" (%s)...' %
|
|||
|
|
# (itemIndex, str(mediaName), str(mediaYear), str(title), str(year), str(altTitle)))
|
|||
|
|
# Max score is when both title and year match exactly.
|
|||
|
|
score = 100
|
|||
|
|
|
|||
|
|
# Item order penalty (the lower it is on the list or results, the larger the penalty).
|
|||
|
|
score = score - (itemIndex * SCORE_PENALTY_ITEM_ORDER)
|
|||
|
|
|
|||
|
|
# Compute year penalty: [equal, diff>=3] --> [0, MAX].
|
|||
|
|
yearPenalty = SCORE_PENALTY_YEAR
|
|||
|
|
mediaYear = toInteger(mediaYear)
|
|||
|
|
year = toInteger(year)
|
|||
|
|
if mediaYear is not None and year is not None:
|
|||
|
|
yearDiff = abs(mediaYear - year)
|
|||
|
|
if not yearDiff:
|
|||
|
|
yearPenalty = 0
|
|||
|
|
elif yearDiff == 1:
|
|||
|
|
yearPenalty = 15
|
|||
|
|
elif yearDiff == 2:
|
|||
|
|
yearPenalty = 25
|
|||
|
|
score = score - yearPenalty
|
|||
|
|
#print str(mediaYear)+' '+str(year)+' '+str(yearPenalty)
|
|||
|
|
|
|||
|
|
# Compute title penalty.
|
|||
|
|
titlePenalty = computeTitlePenalty(mediaName, title)
|
|||
|
|
altTitlePenalty = 100
|
|||
|
|
if altTitle is not None:
|
|||
|
|
altTitlePenalty = computeTitlePenalty(mediaName, altTitle)
|
|||
|
|
|
|||
|
|
# Get detranlitirated media name (in case filename is in latin characters),
|
|||
|
|
# compare it's score with the original, and pick the min.
|
|||
|
|
try:
|
|||
|
|
altMediaName = translit.detranslify(mediaName)
|
|||
|
|
detranslifiedTitlePenalty = computeTitlePenalty(altMediaName, title)
|
|||
|
|
titlePenalty = min(detranslifiedTitlePenalty, titlePenalty)
|
|||
|
|
if altTitle is not None:
|
|||
|
|
detranslifiedAltTitlePenalty = computeTitlePenalty(altMediaName, altTitle)
|
|||
|
|
altTitlePenalty = min(detranslifiedAltTitlePenalty, altTitlePenalty)
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
titlePenalty = min(titlePenalty, altTitlePenalty)
|
|||
|
|
score = score - titlePenalty
|
|||
|
|
|
|||
|
|
# If the score is not high enough, add a few points to the first result -
|
|||
|
|
# let's give KinoPoisk some credit :-).
|
|||
|
|
if itemIndex == 0 and score <= 80:
|
|||
|
|
score = score + 5
|
|||
|
|
|
|||
|
|
# IMPORTANT: always return an int.
|
|||
|
|
score = int(score)
|
|||
|
|
# Log.Debug('***** title scored %d' % score)
|
|||
|
|
return score
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scoreThumbnailResult(thumb, isPoster):
|
|||
|
|
""" Given a Thumbnail object that represents an poster or a funart result,
|
|||
|
|
scores it, and stores the score on the passed object (thumb.score).
|
|||
|
|
"""
|
|||
|
|
score = 0
|
|||
|
|
if thumb.url is None:
|
|||
|
|
thumb.score = 0
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
if thumb.index < IMAGE_SCORE_MAX_NUMBER_OF_ITEMS:
|
|||
|
|
# Score bonus from index for items below 10 on the list.
|
|||
|
|
bonus = IMAGE_SCORE_ITEM_ORDER_BONUS_MAX * \
|
|||
|
|
((IMAGE_SCORE_MAX_NUMBER_OF_ITEMS - thumb.index) / float(IMAGE_SCORE_MAX_NUMBER_OF_ITEMS))
|
|||
|
|
score += bonus
|
|||
|
|
|
|||
|
|
if thumb.width is not None and thumb.height is not None:
|
|||
|
|
# Get a resolution bonus if width*height is more than a certain min value.
|
|||
|
|
if isPoster:
|
|||
|
|
minPx = POSTER_SCORE_MIN_RESOLUTION_PX
|
|||
|
|
maxPx = POSTER_SCORE_MAX_RESOLUTION_PX
|
|||
|
|
bestRatio = POSTER_SCORE_BEST_RATIO
|
|||
|
|
else:
|
|||
|
|
minPx = ART_SCORE_MIN_RESOLUTION_PX
|
|||
|
|
maxPx = ART_SCORE_MAX_RESOLUTION_PX
|
|||
|
|
bestRatio = ART_SCORE_BEST_RATIO
|
|||
|
|
pixelsCount = thumb.width * thumb.height
|
|||
|
|
if pixelsCount > minPx:
|
|||
|
|
if pixelsCount > maxPx:
|
|||
|
|
pixelsCount = maxPx
|
|||
|
|
bonus = float(IMAGE_SCORE_RESOLUTION_BONUS_MAX) * \
|
|||
|
|
float((pixelsCount - minPx)) / float((maxPx - minPx))
|
|||
|
|
score += bonus
|
|||
|
|
|
|||
|
|
# Get an orientation (Portrait vs Landscape) bonus. (we prefer images that are have portrait orientation.
|
|||
|
|
ratio = thumb.width / float(thumb.height)
|
|||
|
|
ratioDiff = math.fabs(bestRatio - ratio)
|
|||
|
|
if ratioDiff < 0.5:
|
|||
|
|
bonus = IMAGE_SCORE_RATIO_BONUS_MAX * (0.5 - ratioDiff) * 2.0
|
|||
|
|
score += bonus
|
|||
|
|
|
|||
|
|
# Get a bonus if image has a separate thumbnail URL.
|
|||
|
|
if thumb.thumbUrl is not None and thumb.url != thumb.thumbUrl:
|
|||
|
|
score += IMAGE_SCORE_THUMB_BONUS
|
|||
|
|
|
|||
|
|
thumb.score = int(score)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def isAsciiString(mediaName):
|
|||
|
|
""" Returns True if all characters of the string are ASCII.
|
|||
|
|
"""
|
|||
|
|
for index, char in enumerate(mediaName):
|
|||
|
|
if ord(char) >= 128:
|
|||
|
|
return False
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
|
|||
|
|
def toInteger(maybeNumber):
|
|||
|
|
""" Returns the argument converted to an integer if it represents a number
|
|||
|
|
or None if the argument is None or does not represent a number.
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
if maybeNumber is not None and str(maybeNumber).strip() != '':
|
|||
|
|
return int(maybeNumber)
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def computeTitlePenalty(mediaName, title):
|
|||
|
|
""" Given media name and a candidate title, returns the title result score penalty.
|
|||
|
|
@param mediaName Movie title parsed from the file system.
|
|||
|
|
@param title Movie title from the website.
|
|||
|
|
"""
|
|||
|
|
mediaName = mediaName.lower()
|
|||
|
|
title = title.lower()
|
|||
|
|
if mediaName != title:
|
|||
|
|
# First approximate the whole strings.
|
|||
|
|
diffRatio = difflib.SequenceMatcher(None, mediaName, title).ratio()
|
|||
|
|
penalty = int(SCORE_PENALTY_TITLE * (1 - diffRatio))
|
|||
|
|
|
|||
|
|
# If the penalty is more than 1/2 of max title penalty, check to see if
|
|||
|
|
# this title starts with media name. This means that media name does not
|
|||
|
|
# have the whole name of the movie - very common case. For example, media name
|
|||
|
|
# "Кавказская пленница" for a movie title "Кавказская пленница, или Новые приключения Шурика".
|
|||
|
|
if penalty >= 15: # This is so that we don't have to do the 'split' every time.
|
|||
|
|
# Compute the scores of the
|
|||
|
|
# First, check if the title starts with media name.
|
|||
|
|
mediaNameParts = mediaName.split()
|
|||
|
|
titleParts = title.split()
|
|||
|
|
if len(mediaNameParts) <= len(titleParts):
|
|||
|
|
i = 0
|
|||
|
|
# Start with some small penalty, value of which depends on how
|
|||
|
|
# many words media name has relative to the title's word count.
|
|||
|
|
penaltyAlt = max(5, int(round((1.0 - (float(len(mediaNameParts)) / len(titleParts))) * 15 - 5)))
|
|||
|
|
penaltyPerPart = SCORE_PENALTY_TITLE / len(mediaNameParts)
|
|||
|
|
for mediaNamePart in mediaNameParts:
|
|||
|
|
partDiffRatio = difflib.SequenceMatcher(None, mediaNamePart, titleParts[i]).ratio()
|
|||
|
|
penaltyAlt = penaltyAlt + int(penaltyPerPart * (1 - partDiffRatio))
|
|||
|
|
i = i + 1
|
|||
|
|
penalty = min(penalty, penaltyAlt)
|
|||
|
|
# print '++++++ DIFF("%s", "%s") = %g --> %d' % (mediaName.encode('utf8'), title.encode('utf8'), diffRatio, penalty)
|
|||
|
|
# Log.Debug('++++++ DIFF("%s", "%s") = %g --> %d' % (mediaName.encode('utf8'), title.encode('utf8'), diffRatio, penalty))
|
|||
|
|
return penalty
|
|||
|
|
return 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
class HttpUtils():
|
|||
|
|
def __init__(self, encoding, userAgent):
|
|||
|
|
self.encoding = encoding
|
|||
|
|
self.userAgent = userAgent
|
|||
|
|
|
|||
|
|
def requestAndParseHtmlPage(self, url):
|
|||
|
|
return getElementFromHttpRequest(url, self.encoding, self.userAgent)
|
|||
|
|
|
|||
|
|
def requestImageJpeg(self, url):
|
|||
|
|
return requestImageJpeg(url, self.userAgent)
|