Bump new version: 0.2.0

- Fixed: Support negative int
- Fixed: seek_back not move pos
- Added: Auto encoding

CHANGELOG.md

@@ -0,0 +1,85 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.2.0] - 2018.5.25
+
+### Change
+
+- Just bump version to 0.2.0 to follow semver.
+
+## [0.1.5rc1] - 2018.4.28
+
+### Added
+
+- `TorrentFileCreator` class and `create_torrent_file` shortcut function for write back data to a torrent file
+
+## [0.1.4] - 2018-04-06
+
+### Added
+
+- `encoding` option can be `auto`, which will use `chardet` package to decide which encoding to use. If `chardet` is noe installed, will raise a warning and fallback to 'utf-8'. (Thanks to [@ltfychrise])
+- Add changelog.
+
+### Change
+
+- Reorganize test codes/files.
+
+### Fixed
+
+- Fix integer filed can't be negative bug. (Thanks to [@ltfychrise])
+- Fix `_seek_back` method not make `_pos` back bug. (Thanks to [@ltfychrise])
+
+## [0.1.3] - 2017-06-21
+
+### Added
+
+- Now `UnicodeDecodeError` is wrapped in `InvalidTorrentDataException`.
+
+### Fixed
+
+- Use `IOError` instead of `FileNotFoundError` in Python 2.
+
+### Changed
+
+- `InvalidTorrentFileException` rename to `InvalidTorrentDataException`.
+
+## [0.1.2] - 2017-06-21
+
+### Changed
+
+- Emm, I don't know, I just changed the version code...
+
+## [0.1.1] - 2017-06-20
+
+### Added
+
+- CLI add coding `--coding/-c` option for file string filed encoding.
+
+### Changed
+
+- `ed2k` and `filehash` field now use same structure as 'pieces'.
+
+## [0.1.0] - 2017-05-23
+
+### Added
+
+- Parse torrent from file and data into a dict.
+- CLI provided.
+- Simple tests.
+- Available on pip.
+
+[@ltfychrise]: https://github.com/ltfychrise
+[Unreleased]: https://github.com/7sDream/torrent_parser/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/7sDream/torrent_parser/compare/v0.1.5rc1...v0.2.0
+[0.1.5rc1]: https://github.com/7sDream/torrent_parser/compare/v0.1.4...v0.1.5rc1
+[0.1.4]: https://github.com/7sDream/torrent_parser/compare/v0.1.3...v0.1.4
+[0.1.3]: https://github.com/7sDream/torrent_parser/compare/v0.1.2...v0.1.3
+[0.1.2]: https://github.com/7sDream/torrent_parser/compare/v0.1.1...v0.1.2
+[0.1.1]: https://github.com/7sDream/torrent_parser/compare/v0.1.0...v0.1.1
+[0.1.0]: https://github.com/7sDream/torrent_parser/tree/v0.1.0

MANIFEST.in

@@ -1,3 +1,3 @@
-include README.md LICENSE changelog.md
+include README.md LICENSE CHANGELOG.md
 include test.py
 include test.torrent

README.md

@@ -1,4 +1,8 @@
-# Torrent file parser for Python
+# Torrent file parser and creator for Python
+
+A simple parser for `.torrent` file.
+
+Can also edit and write back to torrent format after version 0.2.0.
 
 ## Install
 
@@ -32,14 +36,20 @@ $ cat test.torrent | pytp
 >>> data = tp.parse_torrent_file('test.torrent')
 >>> print(data['announce'])
 http://tracker.trackerfix.com:80/announce
+>>> data['announce'] = 'http://127.0.0.1:12345'
+>>> tp.create_torrent_file('new.torrent', data)
 ```
 
 ## Test
 
 ```bash
-python -m unittest test
+python -m unittest tests
 ```
 
+## Changelog
+
+See [Changelog][CHANGELOG].
+
 ## LICENSE
 
 See [License][LICENSE].
@@ -48,3 +58,4 @@ See [License][LICENSE].
 [screenshots-normal]: http://rikka-10066868.image.myqcloud.com/1492616d-9f14-4fe2-9146-9a3ac06c6868.png
 [screenshots-indent]: http://rikka-10066868.image.myqcloud.com/eadc4184-6deb-42eb-bfd4-239da8f50c08.png
 [LICENSE]: https://github.com/7sDream/torrent_parser/blob/master/LICENSE
+[CHANGELOG]: https://github.com/7sDream/torrent_parser/blob/master/CHANGELOG.md

setup.py

@@ -17,7 +17,7 @@ setup(
     license='MIT',
     author='7sDream',
     author_email='7seconddream@gmail.com',
-    description='A .torrent file parser for both Python 2 and 3',
+    description='A .torrent file parser and creator for both Python 2 and 3',
     install_requires=[],
     entry_points={
         'console_scripts': ['pytp=torrent_parser:__main']

tests/__init__.py

@@ -0,0 +1,2 @@
+from .test_create import *
+from .test_parse import *

tests/test_create.py

@@ -0,0 +1,31 @@
+from __future__ import unicode_literals
+
+import collections
+import hashlib
+import io
+import os.path
+import unittest
+
+from torrent_parser import TorrentFileParser, TorrentFileCreator
+
+
+class TestCreate(unittest.TestCase):
+    TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), 'test_files')
+    REAL_FILE = os.path.join(TEST_FILES_DIR, 'real.torrent')
+
+    def test_simple_create(self):
+        data = collections.OrderedDict()
+        data['a'] = 1
+        data['b'] = 2
+        self.assertEqual(TorrentFileCreator(data).encode(), b'd1:ai1e1:bi2ee')
+
+    def test_same_output_if_no_edit(self):
+        with open(self.REAL_FILE, 'rb') as fp:
+            in_data = fp.read()
+            data = TorrentFileParser(io.BytesIO(in_data), True).parse()
+            out_data = TorrentFileCreator(data).encode()
+            m1 = hashlib.md5()
+            m1.update(in_data)
+            m2 = hashlib.md5()
+            m2.update(out_data)
+            self.assertEqual(m1.digest(), m2.digest())

tests/test_files/neg.torrent

@@ -0,0 +1 @@
+d3:negi-1ee

tests/test_parse.py

@@ -1,37 +1,44 @@
 from __future__ import unicode_literals
 
-import unittest
 import collections
+import os.path
+import unittest
 
 from torrent_parser import TorrentFileParser, parse_torrent_file
 
 
-class Test(unittest.TestCase):
+class TestParse(unittest.TestCase):
-    TEST_FILENAME = 'test.torrent'
+    TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), 'test_files')
+    REAL_FILE = os.path.join(TEST_FILES_DIR, 'real.torrent')
+    NEG_FILE = os.path.join(TEST_FILES_DIR, 'neg.torrent')
 
     def test_parse_torrent_file_use_shortcut(self):
-        parse_torrent_file(self.TEST_FILENAME)
+        parse_torrent_file(self.REAL_FILE)
 
     def test_parse_torrent_file_use_class(self):
-        with open(self.TEST_FILENAME, 'rb') as fp:
+        with open(self.REAL_FILE, 'rb') as fp:
             TorrentFileParser(fp).parse()
 
+    def test_encoding_auto(self):
+        with open(self.REAL_FILE, 'rb') as fp:
+            TorrentFileParser(fp, encoding='auto').parse()
+
     def test_parse_torrent_file_to_ordered_dict(self):
-        data = parse_torrent_file(self.TEST_FILENAME, True)
+        data = parse_torrent_file(self.REAL_FILE, True)
         self.assertIsInstance(data, collections.OrderedDict)
 
-        with open(self.TEST_FILENAME, 'rb') as fp:
+        with open(self.REAL_FILE, 'rb') as fp:
             data = TorrentFileParser(fp, True).parse()
         self.assertIsInstance(data, collections.OrderedDict)
 
     def test_parse_correctness(self):
-        data = parse_torrent_file(self.TEST_FILENAME)
+        data = parse_torrent_file(self.REAL_FILE)
         self.assertIn(['udp://tracker.publicbt.com:80/announce'],
                       data['announce-list'])
         self.assertEqual(data['creation date'], 1409254242)
 
     def test_parse_two_times(self):
-        with open(self.TEST_FILENAME, 'rb') as fp:
+        with open(self.REAL_FILE, 'rb') as fp:
             parser = TorrentFileParser(fp)
             data = parser.parse()
             self.assertIn(['udp://tracker.publicbt.com:80/announce'],
@@ -41,3 +48,11 @@ class Test(unittest.TestCase):
             self.assertIn(['udp://tracker.publicbt.com:80/announce'],
                           data['announce-list'])
             self.assertEqual(data['creation date'], 1409254242)
+
+    def test_int_is_negative(self):
+        data = parse_torrent_file(self.NEG_FILE)
+        self.assertEqual(data['neg'], -1)
+
+
+if __name__ == '__main__':
+    unittest.main()

torrent_parser.py

@@ -12,30 +12,82 @@ Usage:
 
     with open(filename, 'rb') as f: # the binary mode 'b' is necessary
         data = TorrentFileParser(f).parse()
+
+    # then you can edit the data
+
+    data['announce-list'].append(['http://127.0.0.1:8080'])
+
+    # and create a new torrent file from data
+
+    create_torrent_file('new.torrent', data)
+
+    # or
+
+    with open('new.torrent', 'wb') as f:
+        f.write(TorrentFileCreator(data).encode())
+
 """
 
 from __future__ import print_function, unicode_literals
 
 import argparse
+import binascii
 import collections
 import io
 import json
 import sys
+import warnings
 
+try:
+    FileNotFoundError
+except NameError:
+    # Python 2 do not have FileNotFoundError, use IOError instead
+    # noinspection PyShadowingBuiltins
+    FileNotFoundError = IOError
+
+try:
+    # noinspection PyPackageRequirements
+    from chardet import detect as _detect
+except ImportError:
+    def _detect(_):
+        warnings.warn("No chardet module installed, encoding will be utf-8")
+        return {'encoding': 'utf-8', 'confidence': 1}
+
+try:
+    # noinspection PyUnresolvedReferences
+    # For Python 2
+    str_type = unicode
+except NameError:
+    # For Python 3
+    str_type = str
 
 __all__ = [
-    'InvalidTorrentFileException',
+    'create_torrent_file',
     'parse_torrent_file',
+    'InvalidTorrentDataException',
+    'TorrentFileCreator',
     'TorrentFileParser',
 ]
 
-__version__ = '0.1.1'
+__version__ = '0.2.0'
 
 
-class InvalidTorrentFileException(Exception):
+def detect(content):
+    return _detect(content)['encoding']
+
+
+class InvalidTorrentDataException(Exception):
     def __init__(self, pos, msg=None):
-        msg = msg or "Invalid torrent format when reading at pos " + str(pos)
+        msg = msg or "Invalid torrent format when read at pos {pos}"
-        super(InvalidTorrentFileException, self).__init__(msg)
+        msg = msg.format(pos=pos)
+        super(InvalidTorrentDataException, self).__init__(msg)
+
+
+class __EndCls(object):
+    pass
+
+
+_END = __EndCls()
 
 
 class TorrentFileParser(object):
@@ -51,6 +103,14 @@ class TorrentFileParser(object):
     INT_INDICATOR = b'i'
     END_INDICATOR = b'e'
     STRING_INDICATOR = b''
+    STRING_DELIMITER = b':'
+
+    RAW_FIELD_PARAMS = {
+        # field length need_list
+        'pieces': (20, True),
+        'ed2k': (16, False),
+        'filehash': (20, False),
+    }
 
     TYPES = [
         (TYPE_LIST, LIST_INDICATOR),
@@ -64,7 +124,8 @@ class TorrentFileParser(object):
         """
         :param fp: a **binary** file-like object to parse,
           which means need 'b' mode when use built-in open function
-        :param encoding: file content encoding, default utf-8
+        :param encoding: file content encoding, default utf-8, use 'auto' to
+          enable charset auto detection ('chardet' package should be installed)
         :param use_ordered_dict: Use collections.OrderedDict as dict container
           default False, which mean use built-in dict
         """
@@ -88,7 +149,7 @@ class TorrentFileParser(object):
 
         try:
             c = self._read_byte(1, True)
-            raise InvalidTorrentFileException(
+            raise InvalidTorrentDataException(
                 0, 'Expect EOF, but get [{}] at pos {}'.format(c, self._pos)
             )
         except EOFError:  # expect EOF
@@ -97,7 +158,7 @@ class TorrentFileParser(object):
         if isinstance(data, dict):
             return data
 
-        raise InvalidTorrentFileException('Outermost element is not a dict')
+        raise InvalidTorrentDataException('Outermost element is not a dict')
 
     def _read_byte(self, count=1, raise_eof=False):
         assert count >= 0
@@ -105,7 +166,7 @@ class TorrentFileParser(object):
         if count != 0 and len(gotten) == 0:
             if raise_eof:
                 raise EOFError()
-            raise InvalidTorrentFileException(
+            raise InvalidTorrentDataException(
                 self._pos,
                 'Unexpected EOF when reading torrent file'
             )
@@ -114,6 +175,7 @@ class TorrentFileParser(object):
 
     def _seek_back(self, count):
         self._content.seek(-count, 1)
+        self._pos = self._pos - count
 
     def _restart(self):
         self._content.seek(0, 0)
@@ -121,16 +183,12 @@ class TorrentFileParser(object):
 
     def _dict_items_generator(self):
         while True:
-            try:
             k = self._next_element()
-            except InvalidTorrentFileException:
+            if k is _END:
                 return
-            if k == 'pieces':
+            if k in self.RAW_FIELD_PARAMS:
-                v = self._next_hash()
+                length, need_list = self.RAW_FIELD_PARAMS[k]
-            elif k == 'ed2k':
+                v = self._next_hash(length, need_list)
-                v = self._next_hash(16, False)
-            elif k == 'filehash':
-                v = self._next_hash(20, False)
             else:
                 v = self._next_element()
             if k == 'encoding':
@@ -145,9 +203,8 @@ class TorrentFileParser(object):
 
     def _list_items_generator(self):
         while True:
-            try:
             element = self._next_element()
-            except InvalidTorrentFileException:
+            if element is _END:
                 return
             yield element
 
@@ -157,33 +214,43 @@ class TorrentFileParser(object):
     def _next_int(self, end=END_INDICATOR):
         value = 0
         char = self._read_byte(1)
+        neg = False
         while char != end:
-            # noinspection PyTypeChecker
+            if not neg and char == b'-':
-            if not b'0' <= char <= b'9':
+                neg = True
-                raise InvalidTorrentFileException(self._pos)
+            elif not b'0' <= char <= b'9':
+                raise InvalidTorrentDataException(self._pos - 1)
+            else:
                 value = value * 10 + int(char) - int(b'0')
             char = self._read_byte(1)
-        return value
+        return -value if neg else value
 
     def _next_string(self, decode=True):
-        length = self._next_int(b':')
+        length = self._next_int(self.STRING_DELIMITER)
         raw = self._read_byte(length)
         if decode:
-            string = raw.decode(self._encoding)
+            encoding = self._encoding
+            if encoding == 'auto':
+                encoding = detect(raw)
+            try:
+                string = raw.decode(encoding)
+            except UnicodeDecodeError as e:
+                raise InvalidTorrentDataException(
+                    self._pos - length + e.start,
+                    "Fail to decode string at pos {pos} using " + e.encoding
+                )
             return string
         return raw
 
-    @staticmethod
+    def _next_hash(self, p_len, need_list):
-    def __to_hex(v):
-        return hex(ord(v) if isinstance(v, str) else v)[2:].rjust(2, str(0))
-
-    def _next_hash(self, p_len=20, need_list=True):
         raw = self._next_string(decode=False)
         if len(raw) % p_len != 0:
-            raise InvalidTorrentFileException(self._pos)
+            raise InvalidTorrentDataException(
+                self._pos - len(raw), "Hash bit length not match at pos {pos}"
+            )
         res = [
-            ''.join([self.__to_hex(c) for c in h])
+            binascii.hexlify(chunk).decode('ascii')
-            for h in (raw[x:x+p_len] for x in range(0, len(raw), p_len))
+            for chunk in (raw[x:x+p_len] for x in range(0, len(raw), p_len))
         ]
         if len(res) == 0 and not need_list:
             return ''
@@ -191,8 +258,9 @@ class TorrentFileParser(object):
             return res[0]
         return res
 
-    def _next_end(self):
+    @staticmethod
-        raise InvalidTorrentFileException(self._pos)
+    def _next_end():
+        return _END
 
     def _next_type(self):
         for (element_type, indicator) in self.TYPES:
@@ -201,7 +269,7 @@ class TorrentFileParser(object):
             if indicator == char:
                 return element_type
             self._seek_back(indicator_length)
-        raise InvalidTorrentFileException(self._pos)
+        raise InvalidTorrentDataException(self._pos)
 
     def _type_to_func(self, t):
         return getattr(self, '_next_' + t)
@@ -212,9 +280,122 @@ class TorrentFileParser(object):
         return element
 
 
+class TorrentFileCreator(object):
+
+    TYPES = {
+        (dict,): TorrentFileParser.TYPE_DICT,
+        (list,): TorrentFileParser.TYPE_LIST,
+        (int,): TorrentFileParser.TYPE_INT,
+        (str_type,): TorrentFileParser.TYPE_STRING,
+    }
+
+    def __init__(self, data, encoding='utf-8'):
+        """
+        :param data: torrent data, must be a dict or OrderedDict
+        :param encoding: string field output encoding
+        """
+        if not isinstance(data, dict):
+            raise InvalidTorrentDataException(
+                None,
+                "Top level structure should be a dict"
+            )
+        self._data = data
+        self._encoding = encoding
+
+    def encode(self):
+        """
+        Encode data to bytes that conform to torrent file format
+        """
+        return b''.join(self._output_element(self._data))
+
+    def encode_to_readable(self):
+        """
+        Encode data to a file-like(BytesIO) object which contains the result of
+        `TorrentFileCreator.encode()`
+        """
+        return io.BytesIO(self.encode())
+
+    def _output_string(self, data):
+        if isinstance(data, str_type):
+            data = data.encode(self._encoding)
+        yield str(len(data)).encode('ascii')
+        yield TorrentFileParser.STRING_DELIMITER
+        yield data
+
+    @staticmethod
+    def _output_int(data):
+        yield TorrentFileParser.INT_INDICATOR
+        yield str(data).encode('ascii')
+        yield TorrentFileParser.END_INDICATOR
+
+    def _output_decode_hash(self, data):
+        if isinstance(data, str_type):
+            data = [data]
+        result = []
+        for hash_line in data:
+            if not isinstance(hash_line, str_type):
+                raise InvalidTorrentDataException(
+                    None,
+                    "Hash must be " + str_type.__name__ + " not " +
+                    type(hash_line).__name__,
+                )
+            if len(hash_line) % 2 != 0:
+                raise InvalidTorrentDataException(
+                    None,
+                    "Hash(" + hash_line + ") length(" + str(len(hash_line)) +
+                    ") is a not even number",
+                )
+            try:
+                raw = binascii.unhexlify(hash_line)
+            except binascii.Error as e:
+                raise InvalidTorrentDataException(
+                    None, str(e),
+                )
+            result.append(raw)
+        for x in self._output_string(b''.join(result)):
+            yield x
+
+    def _output_dict(self, data):
+        yield TorrentFileParser.DICT_INDICATOR
+        for k, v in data.items():
+            if not isinstance(k, str_type):
+                raise InvalidTorrentDataException(
+                    None, "Dict key must be " + str_type.__name__,
+                )
+            for x in self._output_element(k):
+                yield x
+            if k in TorrentFileParser.RAW_FIELD_PARAMS:
+                for x in self._output_decode_hash(v):
+                    yield x
+            else:
+                for x in self._output_element(v):
+                    yield x
+        yield TorrentFileParser.END_INDICATOR
+
+    def _output_list(self, data):
+        yield TorrentFileParser.LIST_INDICATOR
+        for v in data:
+            for x in self._output_element(v):
+                yield x
+        yield TorrentFileParser.END_INDICATOR
+
+    def _type_to_func(self, t):
+        return getattr(self, '_output_' + t)
+
+    def _output_element(self, data):
+        for types, t in self.TYPES.items():
+            if isinstance(data, types):
+                # noinspection PyCallingNonCallable
+                return self._type_to_func(t)(data)
+        raise InvalidTorrentDataException(
+            None,
+            "Invalid type for torrent file: " + type(data).__name__,
+        )
+
+
 def parse_torrent_file(filename, use_ordered_dict=False):
     """
-    Shortcut function for parse torrent object use TorrentFileParser
+    Shortcut function for parse torrent object using TorrentFileParser
 
     :param string filename: torrent filename
     :param bool use_ordered_dict: see :any:`TorrentFileParser.__init__`
@@ -225,6 +406,18 @@ def parse_torrent_file(filename, use_ordered_dict=False):
         return TorrentFileParser(f, use_ordered_dict).parse()
 
 
+def create_torrent_file(filename, data, encoding='utf-8'):
+    """
+    Shortcut function for create a torrent file using TorrentFileCreator
+
+    :param filename: output torrent filename
+    :param data: torrent data, must be a dict or OrderedDict
+    :param encoding: string field output encoding
+    """
+    with open(filename, 'wb') as f:
+        f.write(TorrentFileCreator(data, encoding).encode())
+
+
 def __main():
     parser = argparse.ArgumentParser()
     parser.add_argument('file', nargs='?', default='',
@@ -256,7 +449,7 @@ def __main():
         else:
             target_file = open(args.file, 'rb')
     except FileNotFoundError:
-        sys.stderr.write('Unable to find file {}\n'.format(args.file))
+        sys.stderr.write('File "{}" not exist\n'.format(args.file))
         exit(1)
 
     # noinspection PyUnboundLocalVariable
@@ -269,5 +462,6 @@ def __main():
 
     print(data)
 
+
 if __name__ == '__main__':
     __main()