([^- ]+ - )?Revision \d+:', line): # it's a subversion index page: file.close() os.unlink(filename) return self._download_svn(url, filename) break # not an index page file.close() os.unlink(filename) raise DistutilsError("Unexpected HTML page found at "+url) def _download_svn(self, url, filename): url = url.split('#',1)[0] # remove any fragment for svn's sake creds = [] if url.lower().startswith('svn:') and '@' in url: scheme, netloc, path, p, q, f = urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: netloc, path = path[2:].split('/',1) auth, host = splituser(netloc) if auth: if ':' in auth: user, password = auth.split(':', 1) creds = ["--username", user, "--password", password] else: creds = ["--username", auth] netloc = host url = urlunparse((scheme, netloc, url, p, q, f)) self.info("Doing subversion checkout from %s to %s", url, filename) svn = ["svn", "checkout", "-q"] + creds + [url, filename] subprocess.check_call(svn) return filename @staticmethod def _vcs_split_rev_from_url(url): """ Given a possible VCS URL, return a clean URL and resolved revision if any. >>> vsrfu = PackageIndex._vcs_split_rev_from_url >>> vsrfu('git+https://github.com/pypa/setuptools@v69.0.0#egg-info=setuptools') ('https://github.com/pypa/setuptools', 'v69.0.0') >>> vsrfu('git+https://github.com/pypa/setuptools#egg-info=setuptools') ('https://github.com/pypa/setuptools', None) >>> vsrfu('http://foo/bar') ('http://foo/bar', None) """ parts = urlsplit(url) clean_scheme = parts.scheme.split('+', 1)[-1] # Some fragment identification fails no_fragment_path, _, _ = parts.path.partition('#') pre, sep, post = no_fragment_path.rpartition('@') clean_path, rev = (pre, post) if sep else (post, None) resolved = parts._replace( scheme=clean_scheme, path=clean_path, # discard the fragment fragment='', ).geturl() return resolved, rev def debug(self, msg, *args): log.debug(msg, *args) def info(self, msg, *args): log.info(msg, *args) def warn(self, msg, *args): log.warn(msg, *args) # This pattern matches a character entity reference (a decimal numeric # references, a hexadecimal numeric reference, or a named reference). entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub def uchr(c): if not isinstance(c, int): return c if c>255: return unichr(c) return chr(c) def decode_entity(match): what = match.group(1) if what.startswith('#x'): what = int(what[2:], 16) elif what.startswith('#'): what = int(what[1:]) else: what = name2codepoint.get(what, match.group(0)) return uchr(what) def htmldecode(text): """Decode HTML entities in the given text.""" return entity_sub(decode_entity, text) def socket_timeout(timeout=15): def _socket_timeout(func): def _socket_timeout(*args, **kwargs): old_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(timeout) try: return func(*args, **kwargs) finally: socket.setdefaulttimeout(old_timeout) return _socket_timeout return _socket_timeout def _encode_auth(auth): """ A function compatible with Python 2.3-3.3 that will encode auth from a URL suitable for an HTTP header. >>> _encode_auth('username%3Apassword') u'dXNlcm5hbWU6cGFzc3dvcmQ=' """ auth_s = unquote(auth) # convert to bytes auth_bytes = auth_s.encode() # use the legacy interface for Python 2.3 support encoded_bytes = base64.encodestring(auth_bytes) # convert back to a string encoded = encoded_bytes.decode() # strip the trailing carriage return return encoded.rstrip() def open_with_auth(url, opener=urllib2.urlopen): """Open a urllib2 request, handling HTTP authentication""" scheme, netloc, path, params, query, frag = urlparse(url) # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): raise httplib.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): auth, host = splituser(netloc) else: auth = None if auth: auth = "Basic " + _encode_auth(auth) new_url = urlunparse((scheme,host,path,params,query,frag)) request = urllib2.Request(new_url) request.add_header("Authorization", auth) else: request = urllib2.Request(url) request.add_header('User-Agent', user_agent) fp = opener(request) if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urlparse(fp.url) if s2==scheme and h2==host: fp.url = urlunparse((s2,netloc,path2,param2,query2,frag2)) return fp # adding a timeout to avoid freezing package_index open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) def fix_sf_url(url): return url # backward compatibility def local_open(url): """Read a local path, with special support for directories""" scheme, server, path, param, query, frag = urlparse(url) filename = url2pathname(path) if os.path.isfile(filename): return urllib2.urlopen(url) elif path.endswith('/') and os.path.isdir(filename): files = [] for f in os.listdir(filename): if f=='index.html': fp = open(os.path.join(filename,f),'rb') body = fp.read() fp.close() break elif os.path.isdir(os.path.join(filename,f)): f+='/' files.append("<a href=%r>%s</a>" % (f,f)) else: body = ("<html><head><title>%s