Implementation of the Mainline DHT (BEP #5 and the security extension BEP #42)

README.md

@@ -8,7 +8,8 @@ except for the python standard library.
 The implementation is spread over several files, each implementing
 a single component.
 
-  krpc.py    - implements the basic UDP Kademila-RPC protocol layer
+  - krpc.py    - implements the basic UDP Kademila-RPC protocol layer
+  - dht.py     - contains the code for accessing the Mainline DHT using KRPC
 
 
 KRPC Implementation
@@ -24,3 +25,39 @@ The KRPCPeer only exposes three methods:
       This method sends a query to a remote host specified by a (host, pool) tuple.
       The name and arguments to call on the remote host is given as well.
       An async result holder is returned, that allows to wait for a reply.
+
+DHT Implementation
+------------------
+
+The DHT class offers the 4 DHT methods described in BEP #5 - each takes the
+remote host in the form of a (host, port) tuple as the first argument. The
+other arguments are the same as described in the specification. They all return
+an async result holder with the unprocessed data from the remote host:
+  - ping(target_connection, sender_id)
+  - find_node(target_connection, sender_id, search_id)
+  - get_peers(target_connection, sender_id, info_hash)
+  - announce_peer(target_connection, sender_id, info_hash, port, token, implied_port = None)
+
+In addition, some additional helper functions are made available - these
+functions take care of updating the routing table and are blocking calls with
+a user specified timeout:
+  - dht_ping(connection, timeout = 1)
+      Returns the complete result dictionary of the call.
+  - dht_find_node(search_id, timeout = 120)
+      Searches iteratively for nodes with the given id
+      and yields the connection tuple if found.
+  - dht_get_peers(info_hash, timeout = 120)
+      Searches iteratively for nodes with the given info_hash
+      and yields the connection tuple if found.
+  - dht_announce_peer(info_hash)
+      Registers the availabilty of the info_hash on this node
+      to all peers that supplied a token while searching for it.
+
+The final two functions are used to start and shutdown the local DHT Peer:
+  - __init__(listen_connection, bootstrap_connection = ('router.bittorrent.com', 6881),
+             setup = {'report_t': 10, 'check_t': 30, 'check_N': 10, 'discover_t': 180})
+      The constructor needs to know what address and port to listen on and which node to use
+      as a bootstrap node. The run interval and some other parameters of the maintainance
+      threads can be configured as well.
+  - shutdown()
+      Start shutdown of the local DHT peer and all associated maintainance threads.

crc32c.py

@@ -0,0 +1,98 @@
+"""
+The MIT License
+
+Copyright (c) 2014 Fred Stober
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+"""
+
+# generated using pycrc (www.tty1.net/pycrc)
+crc32c_table = (
+	0x00000000L, 0xf26b8303L, 0xe13b70f7L, 0x1350f3f4L,
+	0xc79a971fL, 0x35f1141cL, 0x26a1e7e8L, 0xd4ca64ebL,
+	0x8ad958cfL, 0x78b2dbccL, 0x6be22838L, 0x9989ab3bL,
+	0x4d43cfd0L, 0xbf284cd3L, 0xac78bf27L, 0x5e133c24L,
+	0x105ec76fL, 0xe235446cL, 0xf165b798L, 0x030e349bL,
+	0xd7c45070L, 0x25afd373L, 0x36ff2087L, 0xc494a384L,
+	0x9a879fa0L, 0x68ec1ca3L, 0x7bbcef57L, 0x89d76c54L,
+	0x5d1d08bfL, 0xaf768bbcL, 0xbc267848L, 0x4e4dfb4bL,
+	0x20bd8edeL, 0xd2d60dddL, 0xc186fe29L, 0x33ed7d2aL,
+	0xe72719c1L, 0x154c9ac2L, 0x061c6936L, 0xf477ea35L,
+	0xaa64d611L, 0x580f5512L, 0x4b5fa6e6L, 0xb93425e5L,
+	0x6dfe410eL, 0x9f95c20dL, 0x8cc531f9L, 0x7eaeb2faL,
+	0x30e349b1L, 0xc288cab2L, 0xd1d83946L, 0x23b3ba45L,
+	0xf779deaeL, 0x05125dadL, 0x1642ae59L, 0xe4292d5aL,
+	0xba3a117eL, 0x4851927dL, 0x5b016189L, 0xa96ae28aL,
+	0x7da08661L, 0x8fcb0562L, 0x9c9bf696L, 0x6ef07595L,
+	0x417b1dbcL, 0xb3109ebfL, 0xa0406d4bL, 0x522bee48L,
+	0x86e18aa3L, 0x748a09a0L, 0x67dafa54L, 0x95b17957L,
+	0xcba24573L, 0x39c9c670L, 0x2a993584L, 0xd8f2b687L,
+	0x0c38d26cL, 0xfe53516fL, 0xed03a29bL, 0x1f682198L,
+	0x5125dad3L, 0xa34e59d0L, 0xb01eaa24L, 0x42752927L,
+	0x96bf4dccL, 0x64d4cecfL, 0x77843d3bL, 0x85efbe38L,
+	0xdbfc821cL, 0x2997011fL, 0x3ac7f2ebL, 0xc8ac71e8L,
+	0x1c661503L, 0xee0d9600L, 0xfd5d65f4L, 0x0f36e6f7L,
+	0x61c69362L, 0x93ad1061L, 0x80fde395L, 0x72966096L,
+	0xa65c047dL, 0x5437877eL, 0x4767748aL, 0xb50cf789L,
+	0xeb1fcbadL, 0x197448aeL, 0x0a24bb5aL, 0xf84f3859L,
+	0x2c855cb2L, 0xdeeedfb1L, 0xcdbe2c45L, 0x3fd5af46L,
+	0x7198540dL, 0x83f3d70eL, 0x90a324faL, 0x62c8a7f9L,
+	0xb602c312L, 0x44694011L, 0x5739b3e5L, 0xa55230e6L,
+	0xfb410cc2L, 0x092a8fc1L, 0x1a7a7c35L, 0xe811ff36L,
+	0x3cdb9bddL, 0xceb018deL, 0xdde0eb2aL, 0x2f8b6829L,
+	0x82f63b78L, 0x709db87bL, 0x63cd4b8fL, 0x91a6c88cL,
+	0x456cac67L, 0xb7072f64L, 0xa457dc90L, 0x563c5f93L,
+	0x082f63b7L, 0xfa44e0b4L, 0xe9141340L, 0x1b7f9043L,
+	0xcfb5f4a8L, 0x3dde77abL, 0x2e8e845fL, 0xdce5075cL,
+	0x92a8fc17L, 0x60c37f14L, 0x73938ce0L, 0x81f80fe3L,
+	0x55326b08L, 0xa759e80bL, 0xb4091bffL, 0x466298fcL,
+	0x1871a4d8L, 0xea1a27dbL, 0xf94ad42fL, 0x0b21572cL,
+	0xdfeb33c7L, 0x2d80b0c4L, 0x3ed04330L, 0xccbbc033L,
+	0xa24bb5a6L, 0x502036a5L, 0x4370c551L, 0xb11b4652L,
+	0x65d122b9L, 0x97baa1baL, 0x84ea524eL, 0x7681d14dL,
+	0x2892ed69L, 0xdaf96e6aL, 0xc9a99d9eL, 0x3bc21e9dL,
+	0xef087a76L, 0x1d63f975L, 0x0e330a81L, 0xfc588982L,
+	0xb21572c9L, 0x407ef1caL, 0x532e023eL, 0xa145813dL,
+	0x758fe5d6L, 0x87e466d5L, 0x94b49521L, 0x66df1622L,
+	0x38cc2a06L, 0xcaa7a905L, 0xd9f75af1L, 0x2b9cd9f2L,
+	0xff56bd19L, 0x0d3d3e1aL, 0x1e6dcdeeL, 0xec064eedL,
+	0xc38d26c4L, 0x31e6a5c7L, 0x22b65633L, 0xd0ddd530L,
+	0x0417b1dbL, 0xf67c32d8L, 0xe52cc12cL, 0x1747422fL,
+	0x49547e0bL, 0xbb3ffd08L, 0xa86f0efcL, 0x5a048dffL,
+	0x8ecee914L, 0x7ca56a17L, 0x6ff599e3L, 0x9d9e1ae0L,
+	0xd3d3e1abL, 0x21b862a8L, 0x32e8915cL, 0xc083125fL,
+	0x144976b4L, 0xe622f5b7L, 0xf5720643L, 0x07198540L,
+	0x590ab964L, 0xab613a67L, 0xb831c993L, 0x4a5a4a90L,
+	0x9e902e7bL, 0x6cfbad78L, 0x7fab5e8cL, 0x8dc0dd8fL,
+	0xe330a81aL, 0x115b2b19L, 0x020bd8edL, 0xf0605beeL,
+	0x24aa3f05L, 0xd6c1bc06L, 0xc5914ff2L, 0x37faccf1L,
+	0x69e9f0d5L, 0x9b8273d6L, 0x88d28022L, 0x7ab90321L,
+	0xae7367caL, 0x5c18e4c9L, 0x4f48173dL, 0xbd23943eL,
+	0xf36e6f75L, 0x0105ec76L, 0x12551f82L, 0xe03e9c81L,
+	0x34f4f86aL, 0xc69f7b69L, 0xd5cf889dL, 0x27a40b9eL,
+	0x79b737baL, 0x8bdcb4b9L, 0x988c474dL, 0x6ae7c44eL,
+	0xbe2da0a5L, 0x4c4623a6L, 0x5f16d052L, 0xad7d5351L,
+)
+
+def crc32c(data):
+	""" return CRC32C checksum """
+	crc = 0xffffffffL
+	for byte in map(ord, data):
+		crc = (crc32c_table[(crc ^ byte) & 0xff] ^ (crc >> 8)) & 0xffffffffL
+	return (crc & 0xffffffffL) ^ 0xffffffffL

dht.py

@@ -0,0 +1,434 @@
+"""
+The MIT License
+
+Copyright (c) 2014-2015 Fred Stober
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+"""
+
+import os, time, socket, hashlib, hmac, threading, logging
+from bencode import bencode, bdecode
+from utils import encode_int, encode_ip, encode_connection, encode_nodes, AsyncTimeout
+from utils import decode_int, decode_ip, decode_connection, decode_nodes, start_thread
+from krpc import KRPCPeer, KRPCError
+
+# BEP #0042 - prefix is based on ip and last byte of the node id - 21 most significant bits must match
+def bep42_prefix(ip, rand_char, rand_rest = '\x00'): # rand_rest determines the last (random) 3 bits
+	from crc32c import crc32c
+	ip = decode_int(encode_ip(ip))
+	value = crc32c(encode_int((ip & 0x030f3fff) | ((ord(rand_char) & 0x7) << 29)))
+	return (value & 0xfffff800) | ((ord(rand_rest) << 8) & 0x00000700)
+
+def valid_id(node_id, connection):
+	vprefix = bep42_prefix(connection[0], node_id[-1])
+	return (((vprefix ^ decode_int(node_id[:4])) & 0xfffff800) == 0)
+
+def strxor(a, b):
+	assert(len(a) == len(b))
+	return int(a.encode('hex'), 16) ^ int(b.encode('hex'), 16)
+
+
+class DHT_Node(object):
+	def __init__(self, connection, id, version = None):
+		self.connection = (socket.gethostbyname(connection[0]), connection[1])
+		self.set_id(id)
+		self.version = version
+		self.tokens = {} # tokens to gain write access to self.values
+		self.values = {}
+		self.attempt = 0
+		self.pending = 0
+		self.last_ping = 0
+
+	def set_id(self, id):
+		self.id = id
+		self.id_cmp = int(id.encode('hex'), 16)
+
+	def __repr__(self):
+		return '%s  %15s  %5d  %20s  %5s  %.2f' % (self.id.encode('hex'), self.connection[0], self.connection[1],
+			repr(self.version), valid_id(self.id, self.connection), time.time() - self.last_ping)
+
+
+# Trivial node list implementation
+class DHT_Router(object):
+	def __init__(self, name):
+		self._log = logging.getLogger(self.__class__.__name__ + '.%s' % name)
+		# This is our routing table.
+		self._nodes = {}
+		self._nodes_lock = threading.Lock()
+		self._nodes_protected = set()
+		self._connections_bad = set()
+
+	def protect_nodes(self, node_id_list):
+		self._log.info('protect %s' % repr(sorted(node_id_list)))
+		with self._nodes_lock:
+			self._nodes_protected.update(node_id_list)
+
+	def good_node(self, node):
+		with self._nodes_lock:
+			node.attempt = 0
+
+	def remove_node(self, node, force = False):
+		with self._nodes_lock:
+			node.attempt += 1
+			if node.id in self._nodes:
+				if force or ((node.id not in self._nodes_protected) and (node.attempt > 2)):
+					if not force:
+						self._connections_bad.add(node.connection)
+					self._nodes[node.id] = filter(lambda n: n.connection != node.connection, self._nodes[node.id])
+					if not self._nodes[node.id]:
+						self._nodes.pop(node.id)
+
+	def register_node(self, node_connection, node_id, node_version = None):
+		with self._nodes_lock:
+			if node_connection in self._connections_bad:
+				if self._log.isEnabledFor(logging.DEBUG):
+					self._log.debug('rejected bad connection %s' % repr(node_connection))
+				return
+			for node in self._nodes.get(node_id, []):
+				if node.connection == node_connection:
+					if not node.version:
+						node.version = node_version
+					return node
+			if self._log.isEnabledFor(logging.DEBUG):
+				self._log.debug('added connection %s' % repr(node_connection))
+			node = DHT_Node(node_connection, node_id, node_version)
+			self._nodes.setdefault(node_id, []).append(node)
+			return node
+
+	# Return nodes matching a filter expression
+	def get_nodes(self, N = None, expression = lambda n: True, sorter = None):
+		if len(self._nodes) == 0:
+			raise RuntimeError('No nodes in routing table!')
+		result = []
+		with self._nodes_lock:
+			for id, node_list in self._nodes.items():
+				result.extend(filter(expression, node_list))
+		result.sort(key = sorter)
+		if N == None:
+			return result
+		return result[:N]
+
+	def redeem_connections(self, fraction = 0.05):
+		remove = int(fraction * len(self._connections_bad))
+		with self._nodes_lock:
+			while self._connections_bad and (remove > 0):
+				self._connections_bad.pop()
+				remove -= 1
+
+	def show_status(self):
+		with self._nodes_lock:
+			self._log.info('Routing table contains %d nodes (%d blacklisted, %s protected)' %\
+				(len(self._nodes), len(self._connections_bad), len(self._nodes_protected)))
+			if self._log.isEnabledFor(logging.DEBUG):
+				for node in self.get_nodes():
+					self._log.debug('\t%r' % node)
+
+
+class DHT(object):
+	def __init__(self, listen_connection, bootstrap_connection = ('router.bittorrent.com', 6881), user_setup = {}):
+		""" Start DHT peer on given (host, port) and bootstrap connection to the DHT """
+		setup = {'report_t': 10, 'check_t': 30, 'check_N': 10, 'discover_t': 180, 'redeem_t': 300}
+		setup.update(user_setup)
+		self._log = logging.getLogger(self.__class__.__name__ + '.%s.%d' % listen_connection)
+		listen_connection = (socket.gethostbyname(listen_connection[0]), listen_connection[1])
+		# Generate key for token generation
+		self._token_key = os.urandom(20)
+		# Start KRPC server process and Routing table
+		self._krpc = KRPCPeer(listen_connection, self._handle_query, cleanup_interval = 1)
+		self._nodes = DHT_Router('%s.%d' % listen_connection)
+		self._node = DHT_Node(listen_connection, os.urandom(20))
+		self._node_lock = threading.RLock()
+		# Start bootstrap process
+		try:
+			tmp = self.ping(bootstrap_connection, sender_id = self._node.id).get_result(timeout = 5)
+		except Exception:
+			tmp = {'ip': encode_connection(listen_connection), 'r': {'id': self._node.id}}
+		self._node.connection = decode_connection(tmp['ip'])
+		self._bootstrap_node = self._nodes.register_node(bootstrap_connection, tmp['r']['id'])
+		# BEP #0042 Enable security extension
+		self._node.set_id(encode_int(bep42_prefix(self._node.connection[0], self._node.id[-1], self._node.id[0]))[:3] + self._node.id[3:])
+		assert(valid_id(self._node.id, self._node.connection))
+		self._nodes.protect_nodes([self._node.id])
+
+		# Start maintainance threads
+		self._shutdown_event = threading.Event()
+		# Report status of routing table
+		self._thread_report = start_thread(self._maintainance_task, self._nodes.show_status,
+			interval = setup['report_t'])
+		# Periodically ping nodes in the routing table
+		def _check_nodes(N):
+			check_nodes = list(self._nodes.get_nodes(N, expression = lambda n: (time.time() - n.last_ping > 15*60)))
+			if not check_nodes:
+				return
+			self._log.info('Starting cleanup of known nodes')
+			node_result_list = []
+			for node in check_nodes:
+				node.last_ping = time.time()
+				node_result_list.append((node, node.id, self.ping(node.connection, self._node.id)))
+			t_end = time.time() + 5
+			for (node, node_id, async_result) in node_result_list:
+				result = self._eval_dht_response(node, async_result, timeout = max(0, t_end - time.time()))
+				if node.id != result.get('id'):
+					self._nodes.remove_node(node, force = True)
+		self._thread_check = start_thread(self._maintainance_task, _check_nodes,
+			interval = setup['check_t'], N = setup['check_N'])
+		# Redeem random nodes from the blacklist
+		def _redeem():
+			self._log.info('Starting redemption of blacklisted nodes')
+			self._nodes.redeem_connections()
+		self._thread_redeem = start_thread(self._maintainance_task, _redeem, interval = setup['redeem_t'])
+		# Try to discover a random node to populate routing table
+		def _discover_nodes():
+			self._log.info('Starting discovery of random node')
+			for idx, entry in enumerate(self.dht_find_node(os.urandom(20))):
+				if idx > 10:
+					break
+		self._thread_discovery = start_thread(self._maintainance_task, _discover_nodes,
+			interval = setup['discover_t'])
+
+
+	def get_external_ip(self):
+		return self._node.connection
+
+	def shutdown(self):
+		""" This function allows to cleanly shutdown the DHT. """
+		self._log.info('shutting down DHT')
+		self._shutdown_event.set() # Trigger shutdown of maintainance threads
+		while True in map(threading.Thread.is_alive, [self._thread_report, self._thread_check,
+				self._thread_redeem, self._thread_discovery]):
+			time.sleep(0.1)
+		self._krpc.shutdown() # Stop listening for incoming connections
+
+	# Maintainance task
+	def _maintainance_task(self, function, interval, **kwargs):
+		while interval > 0:
+			try:
+				function(**kwargs)
+			except Exception:
+				self._log.exception('Exception in DHT maintenance thread')
+			if self._shutdown_event.wait(interval):
+				return
+
+	# Handle remote queries
+	_reply_handler = {}
+	def _handle_query(self, send_krpc_reply, rec, source_connection):
+		if self._log.isEnabledFor(logging.DEBUG):
+			self._log.debug('handling query from %r: %r' % (source_connection, rec))
+		kwargs = rec['a']
+		if 'id' in kwargs:
+			self._nodes.register_node(source_connection, kwargs['id'], rec.get('v'))
+		query = rec['q']
+		if query in self._reply_handler:
+			send_dht_reply = lambda **kwargs: send_krpc_reply(kwargs,
+				# BEP #0042 - require ip field in answer
+				{'ip': encode_connection(source_connection)})
+			send_dht_reply.connection = source_connection
+			self._reply_handler[query](self, send_dht_reply, **kwargs)
+		else:
+			self._log.error('Unknown request in query %r' % rec)
+
+	# Evaluate async KRPC result and notify the routing table about failures
+	def _eval_dht_response(self, node, async_result, timeout):
+		try:
+			result = async_result.get_result(timeout)
+			node.version = result.get('v', node.version)
+			self._nodes.good_node(node)
+			return result['r']
+		except AsyncTimeout: # The node did not reply
+			if self._log.isEnabledFor(logging.DEBUG):
+				self._log.debug('KRPC timeout %r' % node)
+		except KRPCError: # Some other error occured
+			self._log.exception('KRPC Error %r' % node)
+		self._nodes.remove_node(node)
+		async_result.discard_result()
+		return {}
+
+	# Iterate KRPC function on closest nodes - query_fun(connection, id, search_value)
+	def _iter_krpc_search(self, query_fun, process_fun, search_value, timeout, retries):
+		id_cmp = int(search_value.encode('hex'), 16)
+		(returned, used_connections, discovered_nodes) = (set(), {}, set())
+		while True:
+			blacklist_connections = filter(lambda c: used_connections[c] > retries, used_connections)
+			discovered_nodes = set(filter(lambda n: n and (n.connection not in blacklist_connections), discovered_nodes))
+			close_nodes = set(self._nodes.get_nodes(N = 20,
+				expression = lambda n: n.connection not in blacklist_connections,
+				sorter = lambda n: n.id_cmp ^ id_cmp))
+
+			if not close_nodes.union(discovered_nodes):
+				break
+
+			node_result_list = []
+			for node in close_nodes.union(discovered_nodes): # submit all queries at the same time
+				if node.pending > 3:
+					continue
+				if self._log.isEnabledFor(logging.DEBUG):
+					self._log.debug('asking %s' % repr(node))
+				async_result = query_fun(node.connection, self._node.id, search_value)
+				with self._node_lock:
+					node.pending += 1
+				node_result_list.append((node, async_result))
+				used_connections[node.connection] = used_connections.get(node.connection, 0) + 1
+
+			t_end = time.time() + timeout
+			for (node, async_result) in node_result_list: # sequentially retrieve results
+				result = self._eval_dht_response(node, async_result, timeout = max(0, t_end - time.time()))
+				with self._node_lock:
+					node.pending -= 1
+				for node_id, node_connection in decode_nodes(result.get('nodes', '')):
+					discovered_nodes.add(self._nodes.register_node(node_connection, node_id))
+				for tmp in process_fun(node, result):
+					if tmp not in returned:
+						returned.add(tmp)
+						yield tmp
+
+	# syncronous query / async reply implementation of BEP #0005 (DHT Protocol) #
+	#############################################################################
+	# Each KRPC method XYZ is implemented using 3 functions:
+	#   dht_XYZ(...) - wrapper to process the result of the KRPC function
+	#       XYZ(...) - direct call of the KRPC method - returns AsyncResult
+	#      _XYZ(...) - handler to process incoming KRPC calls
+
+	# ping methods
+	#   (sync method)
+	def dht_ping(self, connection, timeout = 1):
+		try:
+			result = self.ping(connection, self._node.id).get_result(timeout)
+			if result.get('r', {}).get('id'):
+				self._nodes.register_node(connection, result['r']['id'], result.get('v'))
+			return result.get('r', {})
+		except (AsyncTimeout, KRPCError):
+			pass
+	#   (verbatim, async KRPC method)
+	def ping(self, target_connection, sender_id):
+		return self._krpc.send_krpc_query(target_connection, 'ping', id = sender_id)
+	#   (reply method)
+	def _ping(self, send_krpc_reply, id, **kwargs):
+		send_krpc_reply(id = self._node.id)
+	_reply_handler['ping'] = _ping
+
+	# find_node methods
+	#   (sync method, iterating on close nodes)
+	def dht_find_node(self, search_id):
+		def process_find_node(node, result):
+			for node_id, node_connection in decode_nodes(result.get('nodes', '')):
+				if node_id == search_id:
+					yield node_connection
+		return self._iter_krpc_search(self.find_node, process_find_node, search_id, timeout = 5, retries = 2)
+	#   (verbatim, async KRPC method)
+	def find_node(self, target_connection, sender_id, search_id):
+		return self._krpc.send_krpc_query(target_connection, 'find_node', id = sender_id, target = search_id)
+	#   (reply method)
+	def _find_node(self, send_krpc_reply, id, target, **kwargs):
+		id_cmp = int(id.encode('hex'), 16)
+		send_krpc_reply(id = self._node.id, nodes = encode_nodes(self._nodes.get_nodes(N = 20,
+			expression = lambda n: valid_id(n.id, n.connection),
+			sorter = lambda n: n.id_cmp ^ id_cmp)))
+	_reply_handler['find_node'] = _find_node
+
+	# get_peers methods
+	#   (sync method, iterating on close nodes)
+	def dht_get_peers(self, info_hash):
+		def process_get_peers(node, result):
+			if result.get('token'):
+				node.tokens[info_hash] = result['token'] # store token for subsequent announce_peer
+			for node_connection in map(decode_connection, result.get('values', '')):
+				yield node_connection
+		return self._iter_krpc_search(self.get_peers, process_get_peers, info_hash, timeout = 5, retries = 2)
+	#   (verbatim, async KRPC method)
+	def get_peers(self, target_connection, sender_id, info_hash):
+		return self._krpc.send_krpc_query(target_connection, 'get_peers', id = sender_id, info_hash = info_hash)
+	#   (reply method)
+	def _get_peers(self, send_krpc_reply, id, info_hash, **kwargs):
+		token = hmac.new(self._token_key, send_krpc_reply.connection[0], hashlib.sha1).digest()
+		id_cmp = int(id.encode('hex'), 16)
+		reply_args = {'nodes': encode_nodes(self._nodes.get_nodes(N = 8,
+			expression = lambda n: valid_id(n.id, n.connection),
+			sorter = lambda n: n.id_cmp ^ id_cmp))}
+		if self._node.values.get(info_hash):
+			reply_args['values'] = map(encode_connection, self._node.values[info_hash])
+		send_krpc_reply(id = self._node.id, token = token, **reply_args)
+	_reply_handler['get_peers'] = _get_peers
+
+	# announce_peer methods
+	#   (sync method, announcing to all nodes giving tokens)
+	def dht_announce_peer(self, info_hash):
+		for node in self._nodes.get_nodes(expression = lambda n: info_hash in n.tokens):
+			yield self.announce_peer(node.connection, self._node.id, info_hash, self._node.connection[1],
+				node.tokens[info_hash], implied_port = 1)
+	#   (verbatim, async KRPC method)
+	def announce_peer(self, target_connection, sender_id, info_hash, port, token, implied_port = None):
+		req = {'id': sender_id, 'info_hash': info_hash, 'port': port, 'token': token}
+		if implied_port != None: # (optional) "1": port not reliable - remote should use source port
+			req['implied_port'] = implied_port
+		return self._krpc.send_krpc_query(target_connection, 'announce_peer', **req)
+	#   (reply method)
+	def _announce_peer(self, send_krpc_reply, id, info_hash, port, token, implied_port = None, **kwargs):
+		local_token = hmac.new(self._token_key, send_krpc_reply.connection[0], hashlib.sha1).digest()
+		if (local_token == token) and valid_id(id, send_krpc_reply.connection): # Validate token and ID
+			if implied_port:
+				port = send_krpc_reply.connection[1]
+			self._node.values.setdefault(info_hash, []).append((send_krpc_reply.connection[0], port))
+			send_krpc_reply(id = self._node.id)
+	_reply_handler['announce_peer'] = _announce_peer
+
+
+if __name__ == '__main__':
+	logging.basicConfig()
+#	logging.getLogger().setLevel(logging.INFO)
+#	logging.getLogger('DHT').setLevel(logging.INFO)
+	logging.getLogger('DHT_Router').setLevel(logging.DEBUG)
+#	logging.getLogger('KRPCPeer').setLevel(logging.INFO)
+
+	# Create a DHT node
+	setup = {'report_t': 5, 'check_t': 2, 'check_N': 10, 'discover_t': 3}
+	bootstrap_connection = ('localhost', 10001)
+#	bootstrap_connection = ('router.bittorrent.com', 6881)
+	dht1 = DHT(('0.0.0.0', 10001), bootstrap_connection, setup)
+	dht2 = DHT(('0.0.0.0', 10002), bootstrap_connection, setup)
+	dht3 = DHT(('0.0.0.0', 10003), bootstrap_connection, setup)
+	dht4 = DHT(('0.0.0.0', 10004), ('localhost', 10003), setup)
+	dht5 = DHT(('0.0.0.0', 10005), ('localhost', 10003), setup)
+	dht6 = DHT(('0.0.0.0', 10006), ('localhost', 10005), setup)
+
+	print '\nping\n' + '=' * 20 # Ping bootstrap node
+	print dht1.dht_ping(bootstrap_connection)
+	print dht6.dht_ping(bootstrap_connection)
+
+	print '\nfind_node\n' + '=' * 20 # Search myself
+	for node in dht3.dht_find_node(dht1._node.id):
+		print '->', node
+
+	print '\nget_peers\n' + '=' * 20 # Search Ubuntu 14.04 info hash
+	info_hash = 'cb84ccc10f296df72d6c40ba7a07c178a4323a14'.decode('hex')
+	for peer in dht5.dht_get_peers(info_hash):
+		print '->', peer
+
+	print '\nannounce_peer\n' + '=' * 20 # Announce availability of info hash at dht5
+	print dht5.dht_announce_peer(info_hash)
+
+	print '\nget_peers\n' + '=' * 20
+	for peer in dht3.dht_get_peers(info_hash):
+		print '->', peer
+
+	print 'done...'
+	time.sleep(5*60)
+	dht1.shutdown()
+	dht6.shutdown()
+	print 'shutdown complete'
+	time.sleep(60*60)