diff --git a/Dockerfile b/Dockerfile index d521d18cd..ccbf8e965 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ENV HOME /root RUN \ apt-get update -y; \ apt-get -y install msgpack-python python-gevent python-pip python-dev; \ - pip install msgpack-python --upgrade; \ + pip install -r requirements.txt --upgrade; \ apt-get clean -y; \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/README.md b/README.md index 362ff0608..c4b6d5e8d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,9 @@ Decentralized websites using Bitcoin crypto and the BitTorrent network - https:/ * Password-less [BIP32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) based authorization: Your account is protected by the same cryptography as your Bitcoin wallet * Built-in SQL server with P2P data synchronization: Allows easier site development and faster page load times - * Anonymity: Full Tor network support with .onion hidden services instead of IPv4 addresses + * Anonymity: + * Full Tor network support with .onion hidden services instead of IPv4 addresses + * Full I2P network support with I2P Destinations instead of IPv4 addresses * TLS encrypted connections * Automatic uPnP port opening * Plugin for multiuser (openproxy) support @@ -109,14 +111,14 @@ It downloads the latest version of ZeroNet then starts it automatically. * `virtualenv env` * `source env/bin/activate` -* `pip install msgpack-python gevent` +* `pip install -r requirements.txt` * `python zeronet.py` * Open http://127.0.0.1:43110/ in your browser ## Current limitations * No torrent-like file splitting for big file support -* ~~No more anonymous than Bittorrent~~ (built-in full Tor support added) +* ~~No more anonymous than Bittorrent~~ (built-in full Tor and I2P support added) * File transactions are not compressed ~~or encrypted yet~~ (TLS encryption added) * No private sites diff --git a/Vagrantfile b/Vagrantfile index 6c4da8945..10a11c58a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -40,6 +40,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provision "shell", inline: "sudo apt-get install msgpack-python python-gevent python-pip python-dev -y" config.vm.provision "shell", - inline: "sudo pip install msgpack-python --upgrade" + inline: "sudo pip install -r requirements.txt --upgrade" end diff --git a/plugins/AnnounceZero/AnnounceZeroPlugin.py b/plugins/AnnounceZero/AnnounceZeroPlugin.py index 14dba61e5..1b4737a79 100644 --- a/plugins/AnnounceZero/AnnounceZeroPlugin.py +++ b/plugins/AnnounceZero/AnnounceZeroPlugin.py @@ -28,11 +28,18 @@ def processPeerRes(site, peers): peer_onion, peer_port = helper.unpackOnionAddress(packed_address) if site.addPeer(peer_onion, peer_port): added += 1 + # I2P Destinations + found_dest = 0 + for packed_address in peers["i2p"]: + found_dest += 1 + peer_dest, peer_port = helper.unpackI2PAddress(packed_address) + if site.addPeer(peer_dest, peer_port): + added += 1 if added: site.worker_manager.onPeers() site.updateWebsocket(peers_added=added) - site.log.debug("Found %s ip4, %s onion peers, new: %s" % (found_ip4, found_onion, added)) + site.log.debug("Found %s ip4, %s onion, %s I2P peers, new: %s" % (found_ip4, found_onion, found_dest, added)) @PluginManager.registerTo("Site") @@ -48,6 +55,8 @@ def announceTracker(self, tracker_protocol, tracker_address, fileserver_port=0, need_types = ["ip4"] if self.connection_server and self.connection_server.tor_manager and self.connection_server.tor_manager.enabled: need_types.append("onion") + if self.connection_server and self.connection_server.i2p_manager and self.connection_server.i2p_manager.enabled: + need_types.append("i2p") if mode == "start" or mode == "more": # Single: Announce only this site sites = [self] @@ -62,12 +71,15 @@ def announceTracker(self, tracker_protocol, tracker_address, fileserver_port=0, # Create request request = { - "hashes": [], "onions": [], "port": fileserver_port, "need_types": need_types, "need_num": 20, "add": add_types + "hashes": [], "onions": [], "i2pdests": [], "port": fileserver_port, "need_types": need_types, "need_num": 20, "add": add_types } for site in sites: if "onion" in add_types: onion = self.connection_server.tor_manager.getOnion(site.address) request["onions"].append(onion) + if "i2p" in add_types: + dest = self.connection_server.i2p_manager.getDest(site.address) + request["i2pdests"].append(dest.base64()) request["hashes"].append(hashlib.sha256(site.address).digest()) # Tracker can remove sites that we don't announce @@ -112,6 +124,23 @@ def announceTracker(self, tracker_protocol, tracker_address, fileserver_port=0, time_full_announced[tracker_address] = 0 return False + # Check if we need to sign prove the I2P Destinations + if "i2p_sign_this" in res: + self.log.debug("Signing %s for %s to add %s I2P dests" % (res["i2p_sign_this"], tracker_address, len(sites))) + request["i2p_signs"] = {} + request["i2p_sign_this"] = res["i2p_sign_this"] + request["need_num"] = 0 + for site in sites: + dest = self.connection_server.i2p_manager.getPrivateDest(site.address) + sign = dest.sign(res["i2p_sign_this"]) + request["i2p_signs"][dest.base64()] = sign + res = tracker.request("announce", request) + if not res or "i2p_sign_this" in res: + self.log.debug("Announce I2P Destination to %s failed: %s" % (tracker_address, res)) + if full_announce: + time_full_announced[tracker_address] = 0 + return False + if full_announce: tracker.remove() # Close connection, we don't need it in next 5 minute diff --git a/plugins/Sidebar/SidebarPlugin.py b/plugins/Sidebar/SidebarPlugin.py index 76effe847..7479c64eb 100644 --- a/plugins/Sidebar/SidebarPlugin.py +++ b/plugins/Sidebar/SidebarPlugin.py @@ -62,13 +62,15 @@ def sidebarRenderPeerStats(self, body, site): connected = len([peer for peer in site.peers.values() if peer.connection and peer.connection.connected]) connectable = len([peer_id for peer_id in site.peers.keys() if not peer_id.endswith(":0")]) onion = len([peer_id for peer_id in site.peers.keys() if ".onion" in peer_id]) + i2p = len([peer_id for peer_id in site.peers.keys() if ".i2p" in peer_id]) peers_total = len(site.peers) if peers_total: percent_connected = float(connected) / peers_total percent_connectable = float(connectable) / peers_total percent_onion = float(onion) / peers_total + percent_i2p = float(i2p) / peers_total else: - percent_connectable = percent_connected = percent_onion = 0 + percent_connectable = percent_connected = percent_onion = percent_i2p = 0 body.append("""
  • @@ -76,12 +78,14 @@ def sidebarRenderPeerStats(self, body, site):
  • +
  • diff --git a/plugins/Stats/StatsPlugin.py b/plugins/Stats/StatsPlugin.py index d21b2b809..a601e4b26 100644 --- a/plugins/Stats/StatsPlugin.py +++ b/plugins/Stats/StatsPlugin.py @@ -126,6 +126,11 @@ def actionStats(self): for site_address, onion in main.file_server.tor_manager.site_onions.items(): yield "- %-34s: %s
    " % (site_address, onion) + # I2P Destinations + yield "

    I2P Destinations (status: %s):
    " % main.file_server.i2p_manager.status + for site_address, dest in main.file_server.i2p_manager.site_dests.items(): + yield "- %-34s: %s
    " % (site_address, dest.base32()) + # Db yield "

    Db:
    " for db in sys.modules["Db.Db"].opened_dbs: diff --git a/requirements.txt b/requirements.txt index eef988d04..a27eea8d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ gevent>=1.1.0 +i2p.socket>=0.3.1 msgpack-python>=0.4.4 diff --git a/src/Config.py b/src/Config.py index 1d4e2d7ff..3ceb1fe74 100644 --- a/src/Config.py +++ b/src/Config.py @@ -36,6 +36,9 @@ def createArguments(self): "udp://tracker.coppersurfer.tk:6969", "udp://tracker.leechers-paradise.org:6969", "udp://9.rarbg.com:2710", + "http://w7tpbzncbcocrqtwwm3nezhnnsw4ozadvi2hmvzdhrqzfxfum7wa.b32.i2p/a", # opentracker.dg2.i2p + "http://vmow3h54yljn7zvzbqepdddt5fmygijujycod2q6yznpy2rrzuwa.b32.i2p/announce", # opentracker.psi.i2p + "http://avviiexdngd32ccoy4kuckvc3mkf53ycvzbz6vz75vzhv4tbpk5a.b32.i2p/tracker/a", # psi.i2p/tracker "http://tracker.aletorrenty.pl:2710/announce", "http://explodie.org:6969/announce", "http://tracker1.wasabii.com.tw:6969/announce" @@ -171,6 +174,9 @@ def createArguments(self): self.parser.add_argument('--tor_controller', help='Tor controller address', metavar='ip:port', default='127.0.0.1:9051') self.parser.add_argument('--tor_proxy', help='Tor proxy address', metavar='ip:port', default='127.0.0.1:9050') + self.parser.add_argument('--i2p', help='enable: Use only for I2P peers, always: Use I2P for every connection', choices=["disable", "enable", "always"], default='enable') + self.parser.add_argument('--i2p_sam', help='I2P SAM API address', metavar='ip:port', default='127.0.0.1:7656') + self.parser.add_argument('--version', action='version', version='ZeroNet %s r%s' % (self.version, self.rev)) return self.parser diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index c9455d15d..be7bfa18a 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -89,6 +89,10 @@ def connect(self): if not self.server.tor_manager or not self.server.tor_manager.enabled: raise Exception("Can't connect to onion addresses, no Tor controller present") self.sock = self.server.tor_manager.createSocket(self.ip, self.port) + elif self.ip.endswith(".i2p"): + if not self.server.i2p_manager or not self.server.i2p_manager.enabled: + raise Exception("Can't connect to I2P addresses, no SAM API present") + self.sock = self.server.i2p_manager.createSocket(self.ip, self.port) else: self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.connect((self.ip, int(self.port))) @@ -164,24 +168,32 @@ def messageLoop(self): # My handshake info def getHandshakeInfo(self): - # No TLS for onion connections - if self.ip.endswith(".onion"): + # No TLS for onion or I2P connections + if self.ip.endswith(".onion") or self.ip.endswith(".i2p"): crypt_supported = [] else: crypt_supported = CryptConnection.manager.crypt_supported - # No peer id for onion connections - if self.ip.endswith(".onion") or self.ip == "127.0.0.1": + # No peer id for onion or I2P connections + if self.ip.endswith(".onion") or self.ip.endswith(".i2p") or self.ip == "127.0.0.1": peer_id = "" else: peer_id = self.server.peer_id - # Setup peer lock from requested onion address - if self.handshake and self.handshake.get("target_ip", "").endswith(".onion"): - target_onion = self.handshake.get("target_ip").replace(".onion", "") # My onion address - onion_sites = {v: k for k, v in self.server.tor_manager.site_onions.items()} # Inverse, Onion: Site address - self.site_lock = onion_sites.get(target_onion) - if not self.site_lock: - self.server.log.error("Unknown target onion address: %s" % target_onion) - self.site_lock = "unknown" + # Setup peer lock from requested onion address or I2P Destination + if self.handshake: + if self.handshake.get("target_ip", "").endswith(".onion"): + target_onion = self.handshake.get("target_ip").replace(".onion", "") # My onion address + onion_sites = {v: k for k, v in self.server.tor_manager.site_onions.items()} # Inverse, Onion: Site address + self.site_lock = onion_sites.get(target_onion) + if not self.site_lock: + self.server.log.error("Unknown target onion address: %s" % target_onion) + self.site_lock = "unknown" + elif self.handshake.get("target_ip", "").endswith(".i2p"): + target_dest = self.handshake.get("target_ip").replace(".i2p", "") # My I2P Destination + dest_sites = {v.base64(): k for k, v in self.server.i2p_manager.site_dests.items()} # Inverse, I2P Destination: Site address + self.site_lock = dest_sites.get(target_dest) + if not self.site_lock: + self.server.log.error("Unknown target I2P Destination: %s" % target_dest) + self.site_lock = "unknown" handshake = { "version": config.version, @@ -195,15 +207,21 @@ def getHandshakeInfo(self): "crypt": self.crypt } if self.site_lock: - handshake["onion"] = self.server.tor_manager.getOnion(self.site_lock) + if self.ip.endswith(".onion"): + handshake["onion"] = self.server.tor_manager.getOnion(self.site_lock) + elif self.ip.endswith(".i2p"): + handshake["i2p"] = self.server.i2p_manager.getDest(self.site_lock).base64() elif self.ip.endswith(".onion"): handshake["onion"] = self.server.tor_manager.getOnion("global") + elif self.ip.endswith(".i2p"): + handshake["i2p"] = self.server.i2p_manager.getDest("global").base64() return handshake def setHandshake(self, handshake): self.handshake = handshake - if handshake.get("port_opened", None) is False and "onion" not in handshake: # Not connectable + if handshake.get("port_opened", None) is False and "onion" not in handshake and \ + "i2p" not in handshake: # Not connectable self.port = 0 else: self.port = handshake["fileserver_port"] # Set peer fileserver port @@ -212,9 +230,13 @@ def setHandshake(self, handshake): self.ip = handshake["onion"] + ".onion" self.updateName() + if handshake.get("i2p") and not self.ip.endswith(".i2p"): # Set incoming connection's I2P Destination + self.ip = handshake["i2p"] + ".i2p" + self.updateName() + # Check if we can encrypt the connection if handshake.get("crypt_supported") and handshake["peer_id"] not in self.server.broken_ssl_peer_ids: - if self.ip.endswith(".onion"): + if self.ip.endswith(".onion") or self.ip.endswith(".i2p"): crypt = None elif handshake.get("crypt"): # Recommended crypt by server crypt = handshake["crypt"] diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 91d3e4e1d..01275442d 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -12,6 +12,7 @@ from Config import config from Crypt import CryptConnection from Crypt import CryptHash +from I2P import I2PManager from Tor import TorManager @@ -28,6 +29,11 @@ def __init__(self, ip=None, port=None, request_handler=None): else: self.tor_manager = None + if config.i2p != "disabled": + self.i2p_manager = I2PManager(self.handleIncomingConnection) + else: + self.i2p_manager = None + self.connections = [] # Connections self.whitelist = ("127.0.0.1",) # No flood protection on this ips self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood @@ -96,7 +102,8 @@ def handleIncomingConnection(self, sock, addr): connection.handleIncomingConnection(sock) def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None): - if ip.endswith(".onion") and self.tor_manager.start_onions and site: # Site-unique connection for Tor + if ((ip.endswith(".onion") and self.tor_manager.start_onions) or \ + (ip.endswith(".i2p") and self.i2p_manager.start_dests)) and site: # Site-unique connection for Tor or I2P key = ip + site.address else: key = ip @@ -116,7 +123,8 @@ def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None if connection.ip == ip: if peer_id and connection.handshake.get("peer_id") != peer_id: # Does not match continue - if ip.endswith(".onion") and self.tor_manager.start_onions and connection.site_lock != site.address: + if ((ip.endswith(".onion") and self.tor_manager.start_onions) or \ + (ip.endswith(".i2p") and self.i2p_manager.start_dests)) and connection.site_lock != site.address: # For different site continue if not connection.connected and create: @@ -130,7 +138,8 @@ def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None if port == 0: raise Exception("This peer is not connectable") try: - if ip.endswith(".onion") and self.tor_manager.start_onions and site: # Lock connection to site + if ((ip.endswith(".onion") and self.tor_manager.start_onions) or \ + (ip.endswith(".i2p") and self.i2p_manager.start_dests)) and site: # Lock connection to site connection = Connection(self, ip, port, site_lock=site.address) else: connection = Connection(self, ip, port) diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index 42d5261e1..521ab6d69 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -257,6 +257,13 @@ def actionPex(self, params): if site.addPeer(*address): added += 1 + # Add sent peers to site + for packed_address in params.get("peers_i2p", []): + address = helper.unpackI2PAddress(packed_address) + got_peer_keys.append("%s:%s" % address) + if site.addPeer(*address): + added += 1 + # Send back peers that is not in the sent list and connectable (not port 0) packed_peers = helper.packPeers(site.getConnectablePeers(params["need"], got_peer_keys)) @@ -265,7 +272,7 @@ def actionPex(self, params): if config.verbose: self.log.debug( "Added %s peers to %s using pex, sending back %s" % - (added, site, len(packed_peers["ip4"]) + len(packed_peers["onion"])) + (added, site, len(packed_peers["ip4"]) + len(packed_peers["onion"]) + len(packed_peers["i2p"])) ) back = {} @@ -273,6 +280,8 @@ def actionPex(self, params): back["peers"] = packed_peers["ip4"] if packed_peers["onion"]: back["peers_onion"] = packed_peers["onion"] + if packed_peers["i2p"]: + back["peers_i2p"] = packed_peers["i2p"] self.response(back) @@ -317,14 +326,20 @@ def actionFindHashIds(self, params): back_ip4 = {} back_onion = {} + back_i2p = {} for hash_id, peers in found.iteritems(): back_onion[hash_id] = [helper.packOnionAddress(peer.ip, peer.port) for peer in peers if peer.ip.endswith("onion")] - back_ip4[hash_id] = [helper.packAddress(peer.ip, peer.port) for peer in peers if not peer.ip.endswith("onion")] + back_i2p[hash_id] = [helper.packI2PAddress(peer.ip, peer.port) for peer in peers if peer.ip.endswith("i2p")] + back_ip4[hash_id] = [helper.packAddress(peer.ip, peer.port) for peer in peers if not (peer.ip.endswith("onion") or peer.ip.endswith("i2p"))] # Check my hashfield + # TODO Is it implied that a site address can only be on Tor, I2P or clearnet at once? if self.server.tor_manager and self.server.tor_manager.site_onions.get(site.address): # Running onion my_ip = helper.packOnionAddress(self.server.tor_manager.site_onions[site.address], self.server.port) my_back = back_onion + elif self.server.i2p_manager and self.server.i2p_manager.site_dests.get(site.address): # Running I2P dest + my_ip = helper.packI2PAddress(self.server.i2p_manager.site_dests[site.address], self.server.port) + my_back = back_i2p elif config.ip_external: # External ip defined my_ip = helper.packAddress(config.ip_external, self.server.port) my_back = back_ip4 @@ -340,10 +355,10 @@ def actionFindHashIds(self, params): if config.verbose: self.log.debug( - "Found: IP4: %s, Onion: %s for %s hashids" % - (len(back_ip4), len(back_onion), len(params["hash_ids"])) + "Found: IP4: %s, Onion: %s, I2P: %s for %s hashids" % + (len(back_ip4), len(back_onion), len(back_i2p), len(params["hash_ids"])) ) - self.response({"peers": back_ip4, "peers_onion": back_onion}) + self.response({"peers": back_ip4, "peers_onion": back_onion, "peers_i2p": back_i2p}) def actionSetHashfield(self, params): site = self.sites.get(params["site"]) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 8faf9c632..b54ec281c 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -191,6 +191,7 @@ def checkSites(self, check_files=False, force_port_check=False): self.openport() if self.port_opened is False: self.tor_manager.startOnions() + self.i2p_manager.startDests() if not sites_checking: for address, site in self.sites.items(): # Check sites integrity diff --git a/src/I2P/I2PManager.py b/src/I2P/I2PManager.py new file mode 100644 index 000000000..511e481f5 --- /dev/null +++ b/src/I2P/I2PManager.py @@ -0,0 +1,176 @@ +import logging + +from gevent.coros import RLock +from gevent.server import StreamServer +from gevent.pool import Pool +from httplib import HTTPConnection +import urllib2 + +from i2p import socket +from i2p.datatypes import Destination + +from Config import config +from Site import SiteManager +from Debug import Debug + + +class I2PHTTPConnection(HTTPConnection): + def __init__(self, i2p_manager, site_address, *args, **kwargs): + HTTPConnection.__init__(self, *args, **kwargs) + self.i2p_manager = i2p_manager + self.site_address = site_address + self._create_connection = self._create_i2p_connection + + def _create_i2p_connection(self, address, timeout=60, + source_address=None): + return self.i2p_manager.createSocket(self.site_address, *address) + +class I2PHTTPHandler(urllib2.HTTPHandler): + def __init__(self, i2p_manager, site_address, *args, **kwargs): + urllib2.HTTPHandler.__init__(self, *args, **kwargs) + self.i2p_manager = i2p_manager + self.site_address = site_address + + def http_open(self, req): + return self.do_open(self._createI2PHTTPConnection, req) + + def _createI2PHTTPConnection(self, *args, **kwargs): + return I2PHTTPConnection(self.i2p_manager, self.site_address, *args, **kwargs) + +class I2PManager: + def __init__(self, fileserver_handler=None): + self.dest_conns = {} # Destination: SAM connection + self.dest_servs = {} # Destination: StreamServer + self.site_dests = {} # Site address: Destination + self.log = logging.getLogger("I2PManager") + self.start_dests = None + self.lock = RLock() + + if config.i2p == "disable": + self.enabled = False + self.start_dests = False + self.status = "Disabled" + else: + self.enabled = True + self.status = "Waiting" + + if fileserver_handler: + self.fileserver_handler = fileserver_handler + else: + self.fileserver_handler = lambda self, sock, addr: None + + self.sam_ip, self.sam_port = config.i2p_sam.split(":") + self.sam_port = int(self.sam_port) + + # Test SAM port + if config.i2p != "disable": + try: + assert self.connect(), "No connection" + self.log.debug("I2P SAM port %s check ok" % config.i2p_sam) + except Exception, err: + self.log.debug("I2P SAM port %s check error: %s" % (config.i2p_sam, err)) + self.enabled = False + + def connect(self): + if not self.enabled: + return False + self.site_dests = {} + self.dest_conns = {} + self.dest_servs = {} + + self.log.debug("Connecting to %s:%s" % (self.sam_ip, self.sam_port)) + with self.lock: + try: + socket.checkAPIConnection((self.sam_ip, self.sam_port)) + self.status = u"Connected" + return True + except Exception, err: + self.status = u"Error (%s)" % err + self.log.error("I2P SAM connect error: %s" % Debug.formatException(err)) + self.enabled = False + return False + + def disconnect(self): + for server in self.dest_servs: + server.stop() + self.dest_conns = {} + self.dest_servs = {} + + def startDests(self): + if self.enabled: + self.log.debug("Start Destinations") + self.start_dests = True + + def addDest(self, site_address=None): + sock = socket.socket(socket.AF_I2P, socket.SOCK_STREAM, + samaddr=(self.sam_ip, self.sam_port)) + try: + sock.setblocking(0) + sock.bind(None, site_address) # Transient Destination, tied to site address + sock.listen() + server = StreamServer( + sock, self.fileserver_handler, spawn=Pool(1000) + ) + server.start() + dest = sock.getsockname() + self.dest_conns[dest] = sock + self.dest_servs[dest] = server + self.status = u"OK (%s Destinations running)" % len(self.dest_conns) + SiteManager.peer_blacklist.append((dest.base64()+".i2p", 0)) + return dest + except Exception, err: + self.status = u"SESSION CREATE error (%s)" % err + self.log.error("I2P SESSION CREATE error: %s" % Debug.formatException(err)) + return False + + def delDest(self, dest): + if dest in self.dest_servs: + self.dest_servs[dest].stop() + del self.dest_conns[dest] + del self.dest_servs[dest] + self.status = "OK (%s Destinations running)" % len(self.dest_conns) + return True + else: + self.status = u"Tried to delete non-existent Destination" + self.log.error("I2P error: Tried to delete non-existent") + self.disconnect() + return False + + def getDest(self, site_address): + with self.lock: + if not self.enabled: + return None + if self.start_dests: # Different Destination for every site + dest = self.site_dests.get(site_address) + else: # Same Destination for every site + dest = self.site_dests.get("global") + site_address = "global" + if not dest: + self.site_dests[site_address] = self.addDest(site_address) + dest = self.site_dests[site_address] + self.log.debug("Created new Destination for %s: %s" % (site_address, dest)) + return dest + + def getPrivateDest(self, addr): + dest = addr if isinstance(addr, Destination) else getDest(addr) + return self.dest_conns[dest].getPrivateDest() + + def createSocket(self, site_address, dest, port): + if not self.enabled: + return False + if dest.endswith(".i2p") and not dest.endswith(".b32.i2p"): + dest = Destination(raw=dest[:-4], b64=True) + self.log.debug("Creating new socket to %s:%s" % + (dest.base32() if isinstance(dest, Destination) else dest, port)) + sock = socket.socket(socket.AF_I2P, socket.SOCK_STREAM, + samaddr=(self.sam_ip, self.sam_port)) + sock.connect((dest, int(port)), site_address) + return sock + + def lookup(self, name): + return socket.lookup(name, (self.sam_ip, self.sam_port)) + + def urlopen(self, site_address, url, timeout): + handler = I2PHTTPHandler(self, site_address) + opener = urllib2.build_opener(handler) + return opener.open(url, timeout=50) diff --git a/src/I2P/__init__.py b/src/I2P/__init__.py new file mode 100644 index 000000000..2a1e80918 --- /dev/null +++ b/src/I2P/__init__.py @@ -0,0 +1 @@ +from I2PManager import I2PManager diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 3cf4694b6..c5ff788d9 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -105,6 +105,8 @@ def __repr__(self): def packMyAddress(self): if self.ip.endswith(".onion"): return helper.packOnionAddress(self.ip, self.port) + elif self.ip.endswith(".i2p"): + return helper.packI2PAddress(self.ip, self.port) else: return helper.packAddress(self.ip, self.port) @@ -241,6 +243,8 @@ def pex(self, site=None, need_num=5): request = {"site": site.address, "peers": packed_peers["ip4"], "need": need_num} if packed_peers["onion"]: request["peers_onion"] = packed_peers["onion"] + if packed_peers["i2p"]: + request["peers_i2p"] = packed_peers["i2p"] res = self.request("pex", request) if not res or "error" in res: return False @@ -255,6 +259,11 @@ def pex(self, site=None, need_num=5): address = helper.unpackOnionAddress(peer) if site.addPeer(*address): added += 1 + # I2P + for peer in res.get("peers_i2p", []): + address = helper.unpackI2PAddress(peer) + if site.addPeer(*address): + added += 1 if added: self.log("Added peers using pex: %s" % added) @@ -292,6 +301,11 @@ def findHashIds(self, hash_ids): if not hash in back: back[hash] = [] back[hash] += map(helper.unpackOnionAddress, onion_peers) + # Unpack I2P dest + for hash, i2p_peers in res.get("peers_i2p", {}).items()[0:30]: + if not hash in back: + back[hash] = [] + back[hash] += map(helper.unpackI2PAddress, i2p_peers) return back diff --git a/src/Site/Site.py b/src/Site/Site.py index 18acf56e8..604c8c69d 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -377,11 +377,17 @@ def publisher(self, inner_path, peers, published, limit, event_done=None, diffs= # Find out my ip and port tor_manager = self.connection_server.tor_manager + i2p_manager = self.connection_server.i2p_manager if tor_manager and tor_manager.enabled and tor_manager.start_onions: my_ip = tor_manager.getOnion(self.address) if my_ip: my_ip += ".onion" my_port = config.fileserver_port + elif i2p_manager and i2p_manager.enabled and i2p_manager.start_dests: + my_ip = i2p_manager.getDest(self.address) + if my_ip: + my_ip += ".i2p" + my_port = 0 else: my_ip = config.ip_external if self.connection_server.port_opened: @@ -666,6 +672,11 @@ def announcePex(self, query_num=2, need_num=5): # Gather peers from tracker # Return: Complete time or False on error def announceTracker(self, tracker_protocol, tracker_address, fileserver_port=0, add_types=[], my_peer_id="", mode="start"): + is_i2p = ".i2p" in tracker_address + i2p_manager = self.connection_server.i2p_manager + if is_i2p and not (i2p_manager and i2p_manager.enabled): + return False + s = time.time() if "ip4" not in add_types: fileserver_port = 0 @@ -692,12 +703,18 @@ def announceTracker(self, tracker_protocol, tracker_address, fileserver_port=0, 'uploaded': 0, 'downloaded': 0, 'left': 0, 'compact': 1, 'numwant': 30, 'event': 'started' } + if is_i2p: + params['ip'] = i2p_manager.getDest(self.address).base64() req = None try: url = "http://" + tracker_address + "?" + urllib.urlencode(params) + timeout = 60 if is_i2p else 30 # Load url - with gevent.Timeout(30, False): # Make sure of timeout - req = urllib2.urlopen(url, timeout=25) + with gevent.Timeout(timeout, False): # Make sure of timeout + if is_i2p: + req = i2p_manager.urlopen(self.address, url, timeout=50) + else: + req = urllib2.urlopen(url, timeout=25) response = req.read() req.fp._sock.recv = None # Hacky avoidance of memory leak for older python versions req.close() @@ -708,13 +725,31 @@ def announceTracker(self, tracker_protocol, tracker_address, fileserver_port=0, # Decode peers peer_data = bencode.decode(response)["peers"] response = None - peer_count = len(peer_data) / 6 peers = [] - for peer_offset in xrange(peer_count): - off = 6 * peer_offset - peer = peer_data[off:off + 6] - addr, port = struct.unpack('!LH', peer) - peers.append({"addr": socket.inet_ntoa(struct.pack('!L', addr)), "port": port}) + if isinstance(peer_data, str): + # Compact response + peer_length = 32 if is_i2p else 6 + peer_count = len(peer_data) / peer_length + for peer_offset in xrange(peer_count): + off = peer_length * peer_offset + peer = peer_data[off:off + peer_length] + if is_i2p: + # TODO measure whether non-compact is faster than compact+lookup + try: + dest = i2p_manager.lookup(peer+".b32.i2p") + peers.append({"addr": dest.base64()+".i2p", "port": 6881}) + except Exception: + pass + else: + addr, port = struct.unpack('!LH', peer) + peers.append({"addr": socket.inet_ntoa(struct.pack('!L', addr)), "port": port}) + else: + # Non-compact response + for peer in peer_data: + if is_i2p: + peers.append({"addr": peer["ip"]+".i2p", "port": peer["port"]}) + else: + peers.append({"addr": peer["ip"], "port": peer["port"]}) except Exception, err: self.log.debug("Http tracker %s error: %s" % (url, err)) if req: @@ -749,6 +784,8 @@ def announce(self, force=False, mode="start", pex=True): trackers = [tracker for tracker in trackers if not tracker.startswith("udp://")] if self.connection_server and not self.connection_server.tor_manager.enabled: trackers = [tracker for tracker in trackers if ".onion" not in tracker] + if self.connection_server and not self.connection_server.i2p_manager.enabled: + trackers = [tracker for tracker in trackers if ".i2p" not in tracker] if mode == "update" or mode == "more": # Only announce on one tracker, increment the queried tracker id self.last_tracker_id += 1 @@ -766,6 +803,8 @@ def announce(self, force=False, mode="start", pex=True): add_types.append("ip4") if self.connection_server.tor_manager.enabled and self.connection_server.tor_manager.start_onions: add_types.append("onion") + if self.connection_server.i2p_manager.enabled and self.connection_server.i2p_manager.start_dests: + add_types.append("i2p") else: my_peer_id = "" diff --git a/src/Test/TestI2P.py b/src/Test/TestI2P.py new file mode 100644 index 000000000..30603dfaa --- /dev/null +++ b/src/Test/TestI2P.py @@ -0,0 +1,135 @@ +import pytest +import time + +from File import FileServer + +# stats.i2p +TEST_B64 = 'Okd5sN9hFWx-sr0HH8EFaxkeIMi6PC5eGTcjM1KB7uQ0ffCUJ2nVKzcsKZFHQc7pLONjOs2LmG5H-2SheVH504EfLZnoB7vxoamhOMENnDABkIRGGoRisc5AcJXQ759LraLRdiGSR0WTHQ0O1TU0hAz7vAv3SOaDp9OwNDr9u902qFzzTKjUTG5vMTayjTkLo2kOwi6NVchDeEj9M7mjj5ySgySbD48QpzBgcqw1R27oIoHQmjgbtbmV2sBL-2Tpyh3lRe1Vip0-K0Sf4D-Zv78MzSh8ibdxNcZACmZiVODpgMj2ejWJHxAEz41RsfBpazPV0d38Mfg4wzaS95R5hBBo6SdAM4h5vcZ5ESRiheLxJbW0vBpLRd4mNvtKOrcEtyCvtvsP3FpA-6IKVswyZpHgr3wn6ndDHiVCiLAQZws4MsIUE1nkfxKpKtAnFZtPrrB8eh7QO9CkH2JBhj7bG0ED6mV5~X5iqi52UpsZ8gnjZTgyG5pOF8RcFrk86kHxAAAA' + +@pytest.mark.usefixtures("resetSettings") +@pytest.mark.usefixtures("resetTempSettings") +class TestI2P: + def testAddDest(self, i2p_manager): + # Add + dest = i2p_manager.addDest() + assert dest + assert dest in i2p_manager.dest_conns + + # Delete + assert i2p_manager.delDest(dest) + assert dest not in i2p_manager.dest_conns + + def testSignDest(self, i2p_manager): + dest = i2p_manager.addDest() + + # Sign + sign = i2p_manager.getPrivateDest(dest).sign("hello") + assert len(sign) == dest.signature_size() + + # Verify + assert dest.verify("hello", sign) + assert not dest.verify("not hello", sign) + + # Delete + i2p_manager.delDest(dest) + + @pytest.mark.skipif(not pytest.config.getvalue("slow"), reason="--slow not requested (takes around ~ 1min)") + def testConnection(self, i2p_manager, file_server, site, site_temp): + file_server.i2p_manager.start_dests = True + dest = file_server.i2p_manager.getDest(site.address) + assert dest + print "Connecting to", dest.base32() + for retry in range(5): # Wait for Destination creation + time.sleep(10) + try: + connection = file_server.getConnection(dest.base64()+".i2p", 1544) + if connection: + break + except Exception, err: + continue + assert connection.handshake + assert not connection.handshake["peer_id"] # No peer_id for I2P connections + + # Return the same connection without site specified + assert file_server.getConnection(dest.base64()+".i2p", 1544) == connection + # No reuse for different site + assert file_server.getConnection(dest.base64()+".i2p", 1544, site=site) != connection + assert file_server.getConnection(dest.base64()+".i2p", 1544, site=site) == file_server.getConnection(dest.base64()+".i2p", 1544, site=site) + site_temp.address = "1OTHERSITE" + assert file_server.getConnection(dest.base64()+".i2p", 1544, site=site) != file_server.getConnection(dest.base64()+".i2p", 1544, site=site_temp) + + # Only allow to query from the locked site + file_server.sites[site.address] = site + connection_locked = file_server.getConnection(dest.base64()+".i2p", 1544, site=site) + assert "body" in connection_locked.request("getFile", {"site": site.address, "inner_path": "content.json", "location": 0}) + assert connection_locked.request("getFile", {"site": "1OTHERSITE", "inner_path": "content.json", "location": 0})["error"] == "Invalid site" + + def testPex(self, file_server, site, site_temp): + # Register site to currently running fileserver + site.connection_server = file_server + file_server.sites[site.address] = site + # Create a new file server to emulate new peer connecting to our peer + file_server_temp = FileServer("127.0.0.1", 1545) + site_temp.connection_server = file_server_temp + file_server_temp.sites[site_temp.address] = site_temp + # We will request peers from this + peer_source = site_temp.addPeer("127.0.0.1", 1544) + + # Get ip4 peers from source site + assert peer_source.pex(need_num=10) == 1 # Need >5 to return also return non-connected peers + assert len(site_temp.peers) == 2 # Me, and the other peer + site.addPeer("1.2.3.4", 1555) # Add peer to source site + assert peer_source.pex(need_num=10) == 1 + assert len(site_temp.peers) == 3 + assert "1.2.3.4:1555" in site_temp.peers + + # Get I2P peers from source site + site.addPeer(TEST_B64+".i2p", 1555) + assert TEST_B64+".i2p:1555" not in site_temp.peers + assert peer_source.pex(need_num=10) == 1 # Need >5 to return also return non-connected peers + assert TEST_B64+".i2p:1555" in site_temp.peers + + def testFindHash(self, i2p_manager, file_server, site, site_temp): + file_server.ip_incoming = {} # Reset flood protection + file_server.sites[site.address] = site + assert file_server.i2p_manager == None + file_server.i2p_manager = i2p_manager + + client = FileServer("127.0.0.1", 1545) + client.sites[site_temp.address] = site_temp + site_temp.connection_server = client + + # Add file_server as peer to client + peer_file_server = site_temp.addPeer("127.0.0.1", 1544) + + assert peer_file_server.findHashIds([1234]) == {} + + # Add fake peer with requred hash + fake_peer_1 = site.addPeer(TEST_B64+".i2p", 1544) + fake_peer_1.hashfield.append(1234) + fake_peer_2 = site.addPeer("1.2.3.5", 1545) + fake_peer_2.hashfield.append(1234) + fake_peer_2.hashfield.append(1235) + fake_peer_3 = site.addPeer("1.2.3.6", 1546) + fake_peer_3.hashfield.append(1235) + fake_peer_3.hashfield.append(1236) + + assert peer_file_server.findHashIds([1234, 1235]) == { + 1234: [('1.2.3.5', 1545), (TEST_B64+".i2p", 1544)], + 1235: [('1.2.3.6', 1546), ('1.2.3.5', 1545)] + } + + # Test my address adding + site.content_manager.hashfield.append(1234) + my_i2p_address = i2p_manager.getDest(site_temp.address).base64()+".i2p" + + res = peer_file_server.findHashIds([1234, 1235]) + assert res[1234] == [('1.2.3.5', 1545), (TEST_B64+".i2p", 1544), (my_i2p_address, 1544)] + assert res[1235] == [('1.2.3.6', 1546), ('1.2.3.5', 1545)] + + # Reset + file_server.i2p_manager = None + + def testSiteDest(self, i2p_manager): + assert i2p_manager.getDest("address1") != i2p_manager.getDest("address2") + assert i2p_manager.getDest("address1") == i2p_manager.getDest("address1") diff --git a/src/Test/conftest.py b/src/Test/conftest.py index 380b407fb..12c3b1e03 100644 --- a/src/Test/conftest.py +++ b/src/Test/conftest.py @@ -37,6 +37,7 @@ def pytest_addoption(parser): config.debug_socket = True # Use test data for unittests config.verbose = True # Use test data for unittests config.tor = "disabled" # Don't start Tor client +config.i2p = "disable" # Don't start I2P client config.trackers = [] os.chdir(os.path.abspath(os.path.dirname(__file__) + "/../..")) # Set working dir @@ -58,6 +59,7 @@ def pytest_addoption(parser): from Crypt import CryptConnection from Ui import UiWebsocket from Tor import TorManager +from I2P import I2PManager from Content import ContentDb from util import RateLimit @@ -226,3 +228,15 @@ def tor_manager(): except Exception, err: raise pytest.skip("Test requires Tor with ControlPort: %s, %s" % (config.tor_controller, err)) return tor_manager + + +@pytest.fixture(scope="session") +def i2p_manager(): + try: + i2p_manager = I2PManager() + i2p_manager.enabled = True + assert i2p_manager.connect(), "No connection" + i2p_manager.startDests() + except Exception, err: + raise pytest.skip("Test requires I2P with SAM port: %s, %s" % (config.i2p_sam, err)) + return i2p_manager diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index 857b9e572..28c48f618 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -39,7 +39,8 @@ def start(self): # Add open fileserver port message or closed port error to homepage at first request after start self.site.page_requested = True # Dont add connection notification anymore file_server = sys.modules["main"].file_server - if file_server.port_opened is None or file_server.tor_manager.start_onions is None: + if file_server.port_opened is None or file_server.tor_manager.start_onions is None or \ + file_server.i2p_manager.start_dests is None: self.site.page_requested = False # Not ready yet, check next time elif file_server.port_opened is True: self.site.notifications.append([ @@ -66,15 +67,43 @@ def start(self): """, 0 ]) - elif file_server.port_opened is False and file_server.tor_manager.start_onions: + elif config.i2p == "always" and file_server.i2p_manager.start_dests: self.site.notifications.append([ "done", """ - Successfully started Tor onion hidden services.
    - For faster connections open %s port on your router. - """ % config.fileserver_port, + I2P mode active, every connection using I2P route.
    + Successfully started I2P Destinations. + """, 10000 ]) + elif config.i2p == "always" and file_server.i2p_manager.start_dests is not False: + self.site.notifications.append([ + "error", + """ + I2P mode active, every connection using I2P route.
    + Unable to start I2P Destinations, please check your config. + """, + 0 + ]) + elif file_server.port_opened is False: + if file_server.tor_manager.start_onions: + self.site.notifications.append([ + "done", + """ + Successfully started Tor onion hidden services.
    + For faster connections open %s port on your router. + """ % config.fileserver_port, + 10000 + ]) + if file_server.i2p_manager.start_dests: + self.site.notifications.append([ + "done", + """ + Successfully started I2P Destinations.
    + For faster connections open %s port on your router. + """ % config.fileserver_port, + 10000 + ]) else: self.site.notifications.append([ "error", @@ -231,6 +260,8 @@ def formatServerInfo(self): "fileserver_port": config.fileserver_port, "tor_enabled": sys.modules["main"].file_server.tor_manager.enabled, "tor_status": sys.modules["main"].file_server.tor_manager.status, + "i2p_enabled": sys.modules["main"].file_server.i2p_manager.enabled, + "i2p_status": sys.modules["main"].file_server.i2p_manager.status, "ui_ip": config.ui_ip, "ui_port": config.ui_port, "version": config.version, @@ -364,7 +395,8 @@ def cbSitePublish(self, to, site, thread, notification=True, callback=True): self.response(to, "ok") else: if len(site.peers) == 0: - if sys.modules["main"].file_server.port_opened or sys.modules["main"].file_server.tor_manager.start_onions: + if sys.modules["main"].file_server.port_opened or sys.modules["main"].file_server.tor_manager.start_onions or \ + sys.modules["main"].file_server.i2p_manager.start_dests: if notification: self.cmd("notification", ["info", "No peers found, but your content is ready to access.", 5000]) if callback: diff --git a/src/lib/pyelliptic/openssl.py b/src/lib/pyelliptic/openssl.py index 129537885..54858a9bd 100644 --- a/src/lib/pyelliptic/openssl.py +++ b/src/lib/pyelliptic/openssl.py @@ -8,6 +8,7 @@ import sys import ctypes +import ctypes.util import logging import os diff --git a/src/util/helper.py b/src/util/helper.py index c46a90428..306638817 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -7,6 +7,7 @@ import time import logging import base64 +from i2p.datatypes import Destination def atomicWrite(dest, content, mode="w"): @@ -52,11 +53,13 @@ def shellquote(*args): def packPeers(peers): - packed_peers = {"ip4": [], "onion": []} + packed_peers = {"ip4": [], "onion": [], "i2p": []} for peer in peers: try: if peer.ip.endswith(".onion"): packed_peers["onion"].append(peer.packMyAddress()) + elif peer.ip.endswith(".i2p"): + packed_peers["i2p"].append(peer.packMyAddress()) else: packed_peers["ip4"].append(peer.packMyAddress()) except Exception, err: @@ -86,6 +89,19 @@ def unpackOnionAddress(packed): return base64.b32encode(packed[0:-2]).lower() + ".onion", struct.unpack("H", packed[-2:])[0] +# Destination, port to packed (389+)-byte format +def packI2PAddress(dest, port): + if not isinstance(dest, Destination): + dest = dest.replace(".i2p", "") + dest = Destination(raw=dest, b64=True) + return dest.serialize() + struct.pack("H", port) + + +# From (389+)-byte format to Destination, port +def unpackI2PAddress(packed): + return Destination(raw=packed[0:-2]).base64() + ".i2p", struct.unpack("H", packed[-2:])[0] + + # Get dir from file # Return: data/site/content.json -> data/site def getDirname(path):