lol

why?

resources/scrapers/fuzzywuzzy/StringMatcher.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+StringMatcher.py
+
+ported from python-Levenshtein
+[https://github.com/miohtama/python-Levenshtein]
+"""
+
+from Levenshtein import *
+from warnings import warn
+
+class StringMatcher:
+    """A SequenceMatcher-like class built on the top of Levenshtein"""
+
+    def _reset_cache(self):
+        self._ratio = self._distance = None
+        self._opcodes = self._editops = self._matching_blocks = None
+
+    def __init__(self, isjunk=None, seq1='', seq2=''):
+        if isjunk:
+            warn("isjunk not NOT implemented, it will be ignored")
+        self._str1, self._str2 = seq1, seq2
+        self._reset_cache()
+
+    def set_seqs(self, seq1, seq2):
+        self._str1, self._str2 = seq1, seq2
+        self._reset_cache()
+
+    def set_seq1(self, seq1):
+        self._str1 = seq1
+        self._reset_cache()
+
+    def set_seq2(self, seq2):
+        self._str2 = seq2
+        self._reset_cache()
+
+    def get_opcodes(self):
+        if not self._opcodes:
+            if self._editops:
+                self._opcodes = opcodes(self._editops, self._str1, self._str2)
+            else:
+                self._opcodes = opcodes(self._str1, self._str2)
+        return self._opcodes
+
+    def get_editops(self):
+        if not self._editops:
+            if self._opcodes:
+                self._editops = editops(self._opcodes, self._str1, self._str2)
+            else:
+                self._editops = editops(self._str1, self._str2)
+        return self._editops
+
+    def get_matching_blocks(self):
+        if not self._matching_blocks:
+            self._matching_blocks = matching_blocks(self.get_opcodes(),
+                                                    self._str1, self._str2)
+        return self._matching_blocks
+
+    def ratio(self):
+        if not self._ratio:
+            self._ratio = ratio(self._str1, self._str2)
+        return self._ratio
+
+    def quick_ratio(self):
+        # This is usually quick enough :o)
+        if not self._ratio:
+            self._ratio = ratio(self._str1, self._str2)
+        return self._ratio
+
+    def real_quick_ratio(self):
+        len1, len2 = len(self._str1), len(self._str2)
+        return 2.0 * min(len1, len2) / (len1 + len2)
+
+    def distance(self):
+        if not self._distance:
+            self._distance = distance(self._str1, self._str2)
+        return self._distance

resources/scrapers/fuzzywuzzy/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+__version__ = '0.4.0'

resources/scrapers/fuzzywuzzy/fuzz.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+fuzz.py
+
+Copyright (c) 2011 Adam Cohen
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+from __future__ import unicode_literals
+import warnings
+
+try:
+    from .StringMatcher import StringMatcher as SequenceMatcher
+except ImportError:
+    #warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')
+    from difflib import SequenceMatcher
+
+from . import utils
+
+
+###########################
+# Basic Scoring Functions #
+###########################
+
+
+def ratio(s1, s2):
+
+    if s1 is None:
+        raise TypeError("s1 is None")
+    if s2 is None:
+        raise TypeError("s2 is None")
+    s1, s2 = utils.make_type_consistent(s1, s2)
+    if len(s1) == 0 or len(s2) == 0:
+        return 0
+
+    m = SequenceMatcher(None, s1, s2)
+    return utils.intr(100 * m.ratio())
+
+
+# todo: skip duplicate indexes for a little more speed
+def partial_ratio(s1, s2):
+
+    if s1 is None:
+        raise TypeError("s1 is None")
+    if s2 is None:
+        raise TypeError("s2 is None")
+    s1, s2 = utils.make_type_consistent(s1, s2)
+    if len(s1) == 0 or len(s2) == 0:
+        return 0
+
+    if len(s1) <= len(s2):
+        shorter = s1
+        longer = s2
+    else:
+        shorter = s2
+        longer = s1
+
+    m = SequenceMatcher(None, shorter, longer)
+    blocks = m.get_matching_blocks()
+
+    # each block represents a sequence of matching characters in a string
+    # of the form (idx_1, idx_2, len)
+    # the best partial match will block align with at least one of those blocks
+    #   e.g. shorter = "abcd", longer = XXXbcdeEEE
+    #   block = (1,3,3)
+    #   best score === ratio("abcd", "Xbcd")
+    scores = []
+    for block in blocks:
+        long_start = block[1] - block[0] if (block[1] - block[0]) > 0 else 0
+        long_end = long_start + len(shorter)
+        long_substr = longer[long_start:long_end]
+
+        m2 = SequenceMatcher(None, shorter, long_substr)
+        r = m2.ratio()
+        if r > .995:
+            return 100
+        else:
+            scores.append(r)
+
+    return int(100 * max(scores))
+
+
+##############################
+# Advanced Scoring Functions #
+##############################
+
+# Sorted Token
+#   find all alphanumeric tokens in the string
+#   sort those tokens and take ratio of resulting joined strings
+#   controls for unordered string elements
+def _token_sort(s1, s2, partial=True, force_ascii=True):
+
+    if s1 is None:
+        raise TypeError("s1 is None")
+    if s2 is None:
+        raise TypeError("s2 is None")
+
+    # pull tokens
+    tokens1 = utils.full_process(s1, force_ascii=force_ascii).split()
+    tokens2 = utils.full_process(s2, force_ascii=force_ascii).split()
+
+    # sort tokens and join
+    sorted1 = " ".join(sorted(tokens1))
+    sorted2 = " ".join(sorted(tokens2))
+
+    sorted1 = sorted1.strip()
+    sorted2 = sorted2.strip()
+
+    if partial:
+        return partial_ratio(sorted1, sorted2)
+    else:
+        return ratio(sorted1, sorted2)
+
+
+def token_sort_ratio(s1, s2, force_ascii=True):
+    return _token_sort(s1, s2, partial=False, force_ascii=force_ascii)
+
+
+def partial_token_sort_ratio(s1, s2, force_ascii=True):
+    return _token_sort(s1, s2, partial=True, force_ascii=force_ascii)
+
+
+# Token Set
+#   find all alphanumeric tokens in each string...treat them as a set
+#   construct two strings of the form
+#       <sorted_intersection><sorted_remainder>
+#   take ratios of those two strings
+#   controls for unordered partial matches
+def _token_set(s1, s2, partial=True, force_ascii=True):
+
+    if s1 is None:
+        raise TypeError("s1 is None")
+    if s2 is None:
+        raise TypeError("s2 is None")
+
+    p1 = utils.full_process(s1, force_ascii=force_ascii)
+    p2 = utils.full_process(s2, force_ascii=force_ascii)
+
+    if not utils.validate_string(p1):
+        return 0
+    if not utils.validate_string(p2):
+        return 0
+
+    # pull tokens
+    tokens1 = set(utils.full_process(p1).split())
+    tokens2 = set(utils.full_process(p2).split())
+
+    intersection = tokens1.intersection(tokens2)
+    diff1to2 = tokens1.difference(tokens2)
+    diff2to1 = tokens2.difference(tokens1)
+
+    sorted_sect = " ".join(sorted(intersection))
+    sorted_1to2 = " ".join(sorted(diff1to2))
+    sorted_2to1 = " ".join(sorted(diff2to1))
+
+    combined_1to2 = sorted_sect + " " + sorted_1to2
+    combined_2to1 = sorted_sect + " " + sorted_2to1
+
+    # strip
+    sorted_sect = sorted_sect.strip()
+    combined_1to2 = combined_1to2.strip()
+    combined_2to1 = combined_2to1.strip()
+
+    if partial:
+        ratio_func = partial_ratio
+    else:
+        ratio_func = ratio
+
+    pairwise = [
+        ratio_func(sorted_sect, combined_1to2),
+        ratio_func(sorted_sect, combined_2to1),
+        ratio_func(combined_1to2, combined_2to1)
+    ]
+    return max(pairwise)
+
+
+def token_set_ratio(s1, s2, force_ascii=True):
+    return _token_set(s1, s2, partial=False, force_ascii=force_ascii)
+
+
+def partial_token_set_ratio(s1, s2, force_ascii=True):
+    return _token_set(s1, s2, partial=True, force_ascii=force_ascii)
+
+
+# TODO: numerics
+
+###################
+# Combination API #
+###################
+
+# q is for quick
+def QRatio(s1, s2, force_ascii=True):
+
+    p1 = utils.full_process(s1, force_ascii=force_ascii)
+    p2 = utils.full_process(s2, force_ascii=force_ascii)
+
+    if not utils.validate_string(p1):
+        return 0
+    if not utils.validate_string(p2):
+        return 0
+
+    return ratio(p1, p2)
+
+
+def UQRatio(s1, s2):
+    return QRatio(s1, s2, force_ascii=False)
+
+
+# w is for weighted
+def WRatio(s1, s2, force_ascii=True):
+
+    p1 = utils.full_process(s1, force_ascii=force_ascii)
+    p2 = utils.full_process(s2, force_ascii=force_ascii)
+
+    if not utils.validate_string(p1):
+        return 0
+    if not utils.validate_string(p2):
+        return 0
+
+    # should we look at partials?
+    try_partial = True
+    unbase_scale = .95
+    partial_scale = .90
+
+    base = ratio(p1, p2)
+    len_ratio = float(max(len(p1), len(p2))) / min(len(p1), len(p2))
+
+    # if strings are similar length, don't use partials
+    if len_ratio < 1.5:
+        try_partial = False
+
+    # if one string is much much shorter than the other
+    if len_ratio > 8:
+        partial_scale = .6
+
+    if try_partial:
+        partial = partial_ratio(p1, p2) * partial_scale
+        ptsor = partial_token_sort_ratio(p1, p2, force_ascii=force_ascii) \
+            * unbase_scale * partial_scale
+        ptser = partial_token_set_ratio(p1, p2, force_ascii=force_ascii) \
+            * unbase_scale * partial_scale
+
+        return int(max(base, partial, ptsor, ptser))
+    else:
+        tsor = token_sort_ratio(p1, p2, force_ascii=force_ascii) * unbase_scale
+        tser = token_set_ratio(p1, p2, force_ascii=force_ascii) * unbase_scale
+
+        return int(max(base, tsor, tser))
+
+
+def UWRatio(s1, s2):
+    return WRatio(s1, s2, force_ascii=False)

resources/scrapers/fuzzywuzzy/process.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+process.py
+
+Copyright (c) 2011 Adam Cohen
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+import itertools
+
+from . import fuzz
+from . import utils
+
+
+def extract(query, choices, processor=None, scorer=None, limit=5):
+    """Find best matches in a list or dictionary of choices, return a
+    list of tuples containing the match and it's score. If a dictionery
+    is used, also returns the key for each match.
+
+    Arguments:
+        query       -- an object representing the thing we want to find
+        choices     -- a list or dictionary of objects we are attempting
+                       to extract values from. The dictionary should
+                       consist of {key: str} pairs.
+        scorer      -- f(OBJ, QUERY) --> INT. We will return the objects
+                       with the highest score by default, we use
+                       score.WRatio() and both OBJ and QUERY should be
+                       strings
+        processor   -- f(OBJ_A) --> OBJ_B, where the output is an input
+                       to scorer for example, "processor = lambda x:
+                       x[0]" would return the first element in a
+                       collection x (of, say, strings) this would then
+                       be used in the scoring collection by default, we
+                       use utils.full_process()
+
+    """
+    if choices is None:
+        return []
+
+    try:
+        if len(choices) == 0:
+            return []
+    except TypeError:
+        pass
+
+    # default, turn whatever the choice is into a workable string
+    if processor is None:
+        processor = lambda x: utils.full_process(x)
+
+    # default: wratio
+    if scorer is None:
+        scorer = fuzz.WRatio
+
+    sl = list()
+
+    if isinstance(choices, dict):
+        for key, choice in choices.items():
+            processed = processor(choice)
+            score = scorer(query, processed)
+            tuple = (choice, score, key)
+            sl.append(tuple)
+
+    else:
+        for choice in choices:
+            processed = processor(choice)
+            score = scorer(query, processed)
+            tuple = (choice, score)
+            sl.append(tuple)
+
+    sl.sort(key=lambda i: i[1], reverse=True)
+    return sl[:limit]
+
+
+def extractBests(query, choices, processor=None, scorer=None, score_cutoff=0, limit=5):
+    """Find best matches above a score in a list of choices, return a
+    list of tuples containing the match and it's score.
+
+    Convenience method which returns the choices with best scores, see
+    extract() for full arguments list
+
+    Optional parameter: score_cutoff.
+        If the choice has a score of less than or equal to score_cutoff
+        it will not be included on result list
+
+    """
+
+    best_list = extract(query, choices, processor, scorer, limit)
+    if len(best_list) > 0:
+        return list(itertools.takewhile(lambda x: x[1] >= score_cutoff, best_list))
+    else:
+        return []
+
+
+def extractOne(query, choices, processor=None, scorer=None, score_cutoff=0):
+    """Find the best match above a score in a list of choices, return a
+    tuple containing the match and it's score if it's above the treshold
+    or None.
+
+    Convenience method which returns the single best choice, see
+    extract() for full arguments list
+
+    Optional parameter: score_cutoff.
+        If the best choice has a score of less than or equal to
+        score_cutoff we will return none (intuition: not a good enough
+        match)
+
+    """
+
+    best_list = extract(query, choices, processor, scorer, limit=1)
+    if len(best_list) > 0:
+        best = best_list[0]
+        if best[1] >= score_cutoff:
+            return best
+        else:
+            return None
+    else:
+        return None

resources/scrapers/fuzzywuzzy/string_processing.py

@@ -0,0 +1,41 @@
+from __future__ import unicode_literals
+import re
+
+
+class StringProcessor(object):
+    """
+    This class defines method to process strings in the most
+    efficient way. Ideally all the methods below use unicode strings
+    for both input and output.
+    """
+
+    @classmethod
+    def replace_non_letters_non_numbers_with_whitespace(cls, a_string):
+        """
+        This function replaces any sequence of non letters and non
+        numbers with a single white space.
+        """
+        regex = re.compile(r"(?ui)\W")
+        return regex.sub(" ", a_string)
+
+    @classmethod
+    def strip(cls, a_string):
+        """
+        This function strips leading and trailing white space.
+        """
+
+        return a_string.strip()
+
+    @classmethod
+    def to_lower_case(cls, a_string):
+        """
+        This function returns the lower-cased version of the string given.
+        """
+        return a_string.lower()
+
+    @classmethod
+    def to_upper_case(cls, a_string):
+        """
+        This function returns the upper-cased version of the string given.
+        """
+        return a_string.upper()

resources/scrapers/fuzzywuzzy/utils.py

@@ -0,0 +1,76 @@
+from __future__ import unicode_literals
+import sys
+
+from fuzzywuzzy.string_processing import StringProcessor
+
+
+PY3 = sys.version_info[0] == 3
+
+
+def validate_string(s):
+    try:
+        if len(s) > 0:
+            return True
+        else:
+            return False
+    except:
+        return False
+
+bad_chars = str('')  # ascii dammit!
+for i in range(128, 256):
+    bad_chars += chr(i)
+if PY3:
+    translation_table = dict((ord(c), None) for c in bad_chars)
+
+
+def asciionly(s):
+    if PY3:
+        return s.translate(translation_table)
+    else:
+        return s.translate(None, bad_chars)
+
+
+def asciidammit(s):
+    if type(s) is str:
+        return asciionly(s)
+    elif type(s) is unicode:
+        return asciionly(s.encode('ascii', 'ignore'))
+    else:
+        return asciidammit(unicode(s))
+
+
+def make_type_consistent(s1, s2):
+    if isinstance(s1, str) and isinstance(s2, str):
+        return s1, s2
+
+    elif isinstance(s1, unicode) and isinstance(s2, unicode):
+        return s1, s2
+
+    else:
+        return unicode(s1), unicode(s2)
+
+
+def full_process(s, force_ascii=False):
+    """Process string by
+        -- removing all but letters and numbers
+        -- trim whitespace
+        -- force to lower case
+        if force_ascii == True, force convert to ascii"""
+
+    if s is None:
+        return ""
+
+    if force_ascii:
+        s = asciidammit(s)
+    # Keep only Letters and Numbres (see Unicode docs).
+    string_out = StringProcessor.replace_non_letters_non_numbers_with_whitespace(s)
+    # Force into lowercase.
+    string_out = StringProcessor.to_lower_case(string_out)
+    # Remove leading and trailing whitespaces.
+    string_out = StringProcessor.strip(string_out)
+    return string_out
+
+
+def intr(n):
+    '''Returns a correctly rounded integer'''
+    return int(round(n))