update to version 0.1.4

- Fixed: Support negative int
- Fixed: seek_back not move pos
- Added: Auto encoding

CHANGELOG.md

@@ -0,0 +1,71 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.4] - 2018-04-06
+
+### Added
+
+- `encoding` option can be `auto`, which will use `chardet` package to decide which encoding to use. If `chardet` is noe installed, will raise a warning and fallback to 'utf-8'. (Thanks to [@ltfychrise])
+- Add changelog.
+
+### Change
+
+- Reorganize test codes/files.
+
+### Fixed
+
+- Fix integer filed can't be negative bug. (Thanks to [@ltfychrise])
+- Fix `_seek_back` method not make `_pos` back bug. (Thanks to [@ltfychrise])
+
+## [0.1.3] - 2017-06-21
+
+### Added
+
+- Now `UnicodeDecodeError` is wrapped in `InvalidTorrentDataException`.
+
+### Fixed
+
+- Use `IOError` instead of `FileNotFoundError` in Python 2.
+
+### Changed
+
+- `InvalidTorrentFileException` rename to `InvalidTorrentDataException`.
+
+## [0.1.2] - 2017-06-21
+
+### Changed
+
+- Emm, I don't know, I just changed the version code...
+
+## [0.1.1] - 2017-06-20
+
+### Added
+
+- CLI add coding `--coding/-c` option for file string filed encoding.
+
+### Changed
+
+- `ed2k` and `filehash` field now use same structure as 'pieces'.
+
+## [0.1.0] - 2017-05-23
+
+### Added
+
+- Parse torrent from file and data into a dict.
+- CLI provided.
+- Simple tests.
+- Available on pip.
+
+[@ltfychrise]: https://github.com/ltfychrise
+[Unreleased]: https://github.com/7sDream/torrent_parser/compare/v0.1.3...HEAD
+[0.1.4]: https://github.com/7sDream/torrent_parser/compare/v0.1.3...v0.1.4
+[0.1.3]: https://github.com/7sDream/torrent_parser/compare/v0.1.2...v0.1.3
+[0.1.2]: https://github.com/7sDream/torrent_parser/compare/v0.1.1...v0.1.2
+[0.1.1]: https://github.com/7sDream/torrent_parser/compare/v0.1.0...v0.1.1
+[0.1.0]: https://github.com/7sDream/torrent_parser/tree/v0.1.0

MANIFEST.in

@@ -1,3 +1,3 @@
-include README.md LICENSE changelog.md
+include README.md LICENSE CHANGELOG.md
 include test.py
 include test.torrent

README.md

@@ -37,9 +37,13 @@ http://tracker.trackerfix.com:80/announce
 ## Test
 
 ```bash
-python -m unittest test
+python -m unittest tests
 ```
 
+## Changelog
+
+See [Changelog][CHANGELOG].
+
 ## LICENSE
 
 See [License][LICENSE].
@@ -48,3 +52,4 @@ See [License][LICENSE].
 [screenshots-normal]: http://rikka-10066868.image.myqcloud.com/1492616d-9f14-4fe2-9146-9a3ac06c6868.png
 [screenshots-indent]: http://rikka-10066868.image.myqcloud.com/eadc4184-6deb-42eb-bfd4-239da8f50c08.png
 [LICENSE]: https://github.com/7sDream/torrent_parser/blob/master/LICENSE
+[CHANGELOG]: https://github.com/7sDream/torrent_parser/blob/master/CHANGELOG.md

test.py

@@ -1,49 +0,0 @@
-from __future__ import unicode_literals
-
-import unittest
-import collections
-
-from torrent_parser import TorrentFileParser, parse_torrent_file
-
-
-class Test(unittest.TestCase):
-    TEST_FILENAME = 'test.torrent'
-
-    def test_parse_torrent_file_use_shortcut(self):
-        parse_torrent_file(self.TEST_FILENAME)
-
-    def test_parse_torrent_file_use_class(self):
-        with open(self.TEST_FILENAME, 'rb') as fp:
-            TorrentFileParser(fp).parse()
-
-    def test_parse_torrent_file_to_ordered_dict(self):
-        data = parse_torrent_file(self.TEST_FILENAME, True)
-        self.assertIsInstance(data, collections.OrderedDict)
-
-        with open(self.TEST_FILENAME, 'rb') as fp:
-            data = TorrentFileParser(fp, True).parse()
-        self.assertIsInstance(data, collections.OrderedDict)
-
-    def test_parse_correctness(self):
-        data = parse_torrent_file(self.TEST_FILENAME)
-        self.assertIn(['udp://p4p.arenabg.ch:1337/announce'],
-                      data['announce-list'])
-        self.assertEqual(data['comment'],
-                         'Torrent downloaded from https://rarbg.to')
-        self.assertEqual(data['creation date'], 1472762993)
-
-    def test_parse_two_times(self):
-        with open(self.TEST_FILENAME, 'rb') as fp:
-            parser = TorrentFileParser(fp)
-            data = parser.parse()
-            self.assertIn(['udp://p4p.arenabg.ch:1337/announce'],
-                          data['announce-list'])
-            self.assertEqual(data['comment'],
-                             'Torrent downloaded from https://rarbg.to')
-            self.assertEqual(data['creation date'], 1472762993)
-            data = parser.parse()
-            self.assertIn(['udp://p4p.arenabg.ch:1337/announce'],
-                          data['announce-list'])
-            self.assertEqual(data['comment'],
-                             'Torrent downloaded from https://rarbg.to')
-            self.assertEqual(data['creation date'], 1472762993)

test.torrent

@@ -1,2 +0,0 @@
-d8:announce41:http://tracker.trackerfix.com:80/announce13:announce-listll41:http://tracker.trackerfix.com:80/announceel30:udp://9.rarbg.me:2710/announceel30:udp://9.rarbg.to:2710/announceel43:udp://tracker.coppersurfer.tk:6969/announceel34:udp://glotorrents.pw:6969/announceel40:udp://tracker.trackerfix.com:80/announceel40:udp://inferno.demonoid.ooo:3392/announceel34:udp://p4p.arenabg.ch:1337/announceel30:udp://9.rarbg.me:2710/announceel30:udp://9.rarbg.to:2710/announceel38:udp://torrent.gresille.org:80/announceel35:http://retracker.krs-ix.ru/announceel34:http://mgtracker.org:2710/announceel30:http://thetracker.org/announceel33:http://explodie.org:6969/announceee7:comment40:Torrent downloaded from https://rarbg.to10:created by13:uTorrent/221013:creation datei1472762993e8:encoding5:UTF-84:infod5:filesld6:lengthi505957e4:pathl83:Streaming, Sharing, Stealing - Big Data and the Future of Entertainment (2016).epubeed6:lengthi36e4:pathl16:Come Join Us.txteee4:name92:Streaming, Sharing, Stealing - Big Data and the Future of Entertainment (2016) (Epub) Gooner12:piece lengthi65536e6:pieces160:™d\6Z)¡þ�Š¸l0 aÛèP
Z™6HÿV'†",fÄ{9ô3™üG2õ0½íOoj¹/v%YG†•VÞƒW&‘®YoÏ=-5ö�À¬-¡rCÆÉbñÂ,ìB,¹’�Äw¸Ç—f{PJ|—‰C/ä�+
-+	B¾+¹žÓ ò±!ЧæTåO0pSDšlüÇ92º¸3Ý8ÅAee

tests/__init__.py

@@ -0,0 +1 @@
+from .test_all import *

tests/test_all.py

@@ -0,0 +1,56 @@
+from __future__ import unicode_literals
+
+import unittest
+import collections
+
+from torrent_parser import TorrentFileParser, parse_torrent_file
+
+
+class Test(unittest.TestCase):
+    REAL_FILE = 'tests/testfiles/real.torrent'
+    NEG_FILE = 'tests/testfiles/neg.torrent'
+
+    def test_parse_torrent_file_use_shortcut(self):
+        parse_torrent_file(self.REAL_FILE)
+
+    def test_parse_torrent_file_use_class(self):
+        with open(self.REAL_FILE, 'rb') as fp:
+            TorrentFileParser(fp).parse()
+
+    def test_encoding_auto(self):
+        with open(self.REAL_FILE, 'rb') as fp:
+            TorrentFileParser(fp, encoding='auto').parse()
+
+    def test_parse_torrent_file_to_ordered_dict(self):
+        data = parse_torrent_file(self.REAL_FILE, True)
+        self.assertIsInstance(data, collections.OrderedDict)
+
+        with open(self.REAL_FILE, 'rb') as fp:
+            data = TorrentFileParser(fp, True).parse()
+        self.assertIsInstance(data, collections.OrderedDict)
+
+    def test_parse_correctness(self):
+        data = parse_torrent_file(self.REAL_FILE)
+        self.assertIn(['udp://tracker.publicbt.com:80/announce'],
+                      data['announce-list'])
+        self.assertEqual(data['creation date'], 1409254242)
+
+    def test_parse_two_times(self):
+        with open(self.REAL_FILE, 'rb') as fp:
+            parser = TorrentFileParser(fp)
+            data = parser.parse()
+            self.assertIn(['udp://tracker.publicbt.com:80/announce'],
+                          data['announce-list'])
+            self.assertEqual(data['creation date'], 1409254242)
+            data = parser.parse()
+            self.assertIn(['udp://tracker.publicbt.com:80/announce'],
+                          data['announce-list'])
+            self.assertEqual(data['creation date'], 1409254242)
+
+    def test_int_is_negative(self):
+        data = parse_torrent_file(self.NEG_FILE)
+        self.assertEqual(data['neg'], -1)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/testfiles/neg.torrent

@@ -0,0 +1 @@
+d3:negi-1ee

torrent_parser.py

@@ -7,9 +7,9 @@ A .torrent file parser for both Python 2 and 3
 Usage:
 
     data = parse_torrent_file(filename)
-    
+
     # or
-    
+
     with open(filename, 'rb') as f: # the binary mode 'b' is necessary
         data = TorrentFileParser(f).parse()
 """
@@ -21,21 +21,48 @@ import collections
 import io
 import json
 import sys
+import warnings
 
+try:
+    FileNotFoundError
+except NameError:
+    # Python 2 do not have FileNotFoundError, use IOError instead
+    # noinspection PyShadowingBuiltins
+    FileNotFoundError = IOError
+
+try:
+    # noinspection PyPackageRequirements
+    from chardet import detect as _detect
+except ImportError:
+    def _detect(_):
+        warnings.warn("No chardet module installed, encoding will be utf-8")
+        return {'encoding': 'utf-8', 'confidence': 1}
 
 __all__ = [
-    'InvalidTorrentFileException',
+    'InvalidTorrentDataException',
     'parse_torrent_file',
     'TorrentFileParser',
 ]
 
-__version__ = '0.1.0'
+__version__ = '0.1.4'
 
 
-class InvalidTorrentFileException(Exception):
+def detect(content):
+    return _detect(content)['encoding']
+
+
+class InvalidTorrentDataException(Exception):
     def __init__(self, pos, msg=None):
-        msg = msg or "Invalid torrent format when reading at pos " + str(pos)
+        msg = msg or "Invalid torrent format when read at pos {pos}"
-        super(InvalidTorrentFileException, self).__init__(msg)
+        msg = msg.format(pos=pos)
+        super(InvalidTorrentDataException, self).__init__(msg)
+
+
+class __EndCls(object):
+    pass
+
+
+_END = __EndCls()
 
 
 class TorrentFileParser(object):
@@ -62,9 +89,10 @@ class TorrentFileParser(object):
 
     def __init__(self, fp, use_ordered_dict=False, encoding='utf-8'):
         """
-        :param fp: a **binary** file-like object to parse, 
+        :param fp: a **binary** file-like object to parse,
           which means need 'b' mode when use built-in open function
-        :param encoding: file content encoding, default utf-8
+        :param encoding: file content encoding, default utf-8, use 'auto' to
+          enable charset auto detection ('chardet' package should be installed)
         :param use_ordered_dict: Use collections.OrderedDict as dict container
           default False, which mean use built-in dict
         """
@@ -88,7 +116,7 @@ class TorrentFileParser(object):
 
         try:
             c = self._read_byte(1, True)
-            raise InvalidTorrentFileException(
+            raise InvalidTorrentDataException(
                 0, 'Expect EOF, but get [{}] at pos {}'.format(c, self._pos)
             )
         except EOFError:  # expect EOF
@@ -97,7 +125,7 @@ class TorrentFileParser(object):
         if isinstance(data, dict):
             return data
 
-        raise InvalidTorrentFileException('Outermost element is not a dict')
+        raise InvalidTorrentDataException('Outermost element is not a dict')
 
     def _read_byte(self, count=1, raise_eof=False):
         assert count >= 0
@@ -105,7 +133,7 @@ class TorrentFileParser(object):
         if count != 0 and len(gotten) == 0:
             if raise_eof:
                 raise EOFError()
-            raise InvalidTorrentFileException(
+            raise InvalidTorrentDataException(
                 self._pos,
                 'Unexpected EOF when reading torrent file'
             )
@@ -114,6 +142,7 @@ class TorrentFileParser(object):
 
     def _seek_back(self, count):
         self._content.seek(-count, 1)
+        self._pos = self._pos - count
 
     def _restart(self):
         self._content.seek(0, 0)
@@ -121,12 +150,15 @@ class TorrentFileParser(object):
 
     def _dict_items_generator(self):
         while True:
-            try:
+            k = self._next_element()
-                k = self._next_element()
+            if k is _END:
-            except InvalidTorrentFileException:
                 return
             if k == 'pieces':
-                v = self._pieces()
+                v = self._next_hash()
+            elif k == 'ed2k':
+                v = self._next_hash(16, False)
+            elif k == 'filehash':
+                v = self._next_hash(20, False)
             else:
                 v = self._next_element()
             if k == 'encoding':
@@ -141,9 +173,8 @@ class TorrentFileParser(object):
 
     def _list_items_generator(self):
         while True:
-            try:
+            element = self._next_element()
-                element = self._next_element()
+            if element is _END:
-            except InvalidTorrentFileException:
                 return
             yield element
 
@@ -153,19 +184,31 @@ class TorrentFileParser(object):
     def _next_int(self, end=END_INDICATOR):
         value = 0
         char = self._read_byte(1)
+        neg = False
         while char != end:
-            # noinspection PyTypeChecker
+            if not neg and char == b'-':
-            if not b'0' <= char <= b'9':
+                neg = True
-                raise InvalidTorrentFileException(self._pos)
+            elif not b'0' <= char <= b'9':
-            value = value * 10 + int(char) - int(b'0')
+                raise InvalidTorrentDataException(self._pos - 1)
+            else:
+                value = value * 10 + int(char) - int(b'0')
             char = self._read_byte(1)
-        return value
+        return -value if neg else value
 
     def _next_string(self, decode=True):
         length = self._next_int(b':')
         raw = self._read_byte(length)
         if decode:
-            string = raw.decode(self._encoding)
+            encoding = self._encoding
+            if encoding == 'auto':
+                encoding = detect(raw)
+            try:
+                string = raw.decode(encoding)
+            except UnicodeDecodeError as e:
+                raise InvalidTorrentDataException(
+                    self._pos - length + e.start,
+                    "Fail to decode string at pos {pos} using " + e.encoding
+                )
             return string
         return raw
 
@@ -173,17 +216,25 @@ class TorrentFileParser(object):
     def __to_hex(v):
         return hex(ord(v) if isinstance(v, str) else v)[2:].rjust(2, str(0))
 
-    def _pieces(self):
+    def _next_hash(self, p_len=20, need_list=True):
         raw = self._next_string(decode=False)
-        if len(raw) % 20 != 0:
+        if len(raw) % p_len != 0:
-            raise InvalidTorrentFileException(self._pos)
+            raise InvalidTorrentDataException(
-        return [
+                self._pos - len(raw), "Hash bit length not match at pos {pos}"
+            )
+        res = [
             ''.join([self.__to_hex(c) for c in h])
-            for h in (raw[x:x+20] for x in range(0, len(raw), 20))
+            for h in (raw[x:x+p_len] for x in range(0, len(raw), p_len))
         ]
+        if len(res) == 0 and not need_list:
+            return ''
+        if len(res) == 1 and not need_list:
+            return res[0]
+        return res
 
-    def _next_end(self):
+    @staticmethod
-        raise InvalidTorrentFileException(self._pos)
+    def _next_end():
+        return _END
 
     def _next_type(self):
         for (element_type, indicator) in self.TYPES:
@@ -192,7 +243,7 @@ class TorrentFileParser(object):
             if indicator == char:
                 return element_type
             self._seek_back(indicator_length)
-        raise InvalidTorrentFileException(self._pos)
+        raise InvalidTorrentDataException(self._pos)
 
     def _type_to_func(self, t):
         return getattr(self, '_next_' + t)
@@ -206,7 +257,7 @@ class TorrentFileParser(object):
 def parse_torrent_file(filename, use_ordered_dict=False):
     """
     Shortcut function for parse torrent object use TorrentFileParser
-    
+
     :param string filename: torrent filename
     :param bool use_ordered_dict: see :any:`TorrentFileParser.__init__`
     :rtype: dict if ``use_ordered_dict`` is false,
@@ -229,6 +280,8 @@ def __main():
     parser.add_argument('--ascii', '-a', action='store_true', default=False,
                         help='ensure output json use ascii char, '
                              'escape other char use \\u')
+    parser.add_argument('--coding', '-c', default='utf-8',
+                        help='string encoding, default utf-8')
     parser.add_argument('--version', '-v', action='store_true', default=False,
                         help='print version and exit')
     args = parser.parse_args()
@@ -245,11 +298,11 @@ def __main():
         else:
             target_file = open(args.file, 'rb')
     except FileNotFoundError:
-        sys.stderr.write('Unable to find file {}\n'.format(args.file))
+        sys.stderr.write('File "{}" not exist\n'.format(args.file))
         exit(1)
 
     # noinspection PyUnboundLocalVariable
-    data = TorrentFileParser(target_file, not args.dict).parse()
+    data = TorrentFileParser(target_file, not args.dict, args.coding).parse()
 
     data = json.dumps(
         data, ensure_ascii=args.ascii,
@@ -258,5 +311,6 @@ def __main():
 
     print(data)
 
+
 if __name__ == '__main__':
     __main()