fix urlparse using six in tests and schema_salad core; no automated fixers

Author: manu-chroma

schema_salad/jsonld_context.py

@@ -4,6 +4,8 @@
 import json
 import ruamel.yaml as yaml
 import six
+# import urlparse
+from six.moves.urllib import parse
 try:
     from ruamel.yaml import CSafeLoader as SafeLoader
 except ImportError:
@@ -18,7 +20,6 @@
 from rdflib import Graph, URIRef
 import rdflib.namespace
 from rdflib.namespace import RDF, RDFS
-import urlparse
 import logging
 from schema_salad.utils import aslist
 from typing import (cast, Any, Dict, Iterable, List, Optional, Text, Tuple,
@@ -36,7 +37,7 @@ def pred(datatype,      # type: Dict[str, Union[Dict, str]]
          namespaces     # type: Dict[str, rdflib.namespace.Namespace]
          ):
     # type: (...) -> Union[Dict, Text]
-    split = urlparse.urlsplit(name)
+    split = parse.urlsplit(name)
 
     vee = None  # type: Optional[Union[str, Text]]
 
@@ -104,7 +105,7 @@ def process_type(t,             # type: Dict[str, Any]
         classnode = URIRef(recordname)
         g.add((classnode, RDF.type, RDFS.Class))
 
-        split = urlparse.urlsplit(recordname)
+        split = parse.urlsplit(recordname)
         predicate = recordname
         if t.get("inVocab", True):
             if split.scheme:
@@ -220,7 +221,7 @@ def makerdf(workflow,       # type: Union[str, Text]
             url = v
         if url == "@id":
             idfields.append(k)
-        doc_url, frg = urlparse.urldefrag(url)
+        doc_url, frg = parse.urldefrag(url)
         if "/" in frg:
             p = frg.split("/")[0]
             prefixes[p] = u"%s#%s/" % (doc_url, p)

schema_salad/main.py

@@ -6,7 +6,9 @@
 import traceback
 import json
 import os
-import urlparse
+# import urlparse
+from six.moves.urllib import parse
+
 
 import pkg_resources  # part of setuptools
 
@@ -110,7 +112,7 @@ def main(argsl=None):  # type: (List[str]) -> int
     # Load schema document and resolve refs
 
     schema_uri = args.schema
-    if not urlparse.urlparse(schema_uri)[0]:
+    if not parse.urlparse(schema_uri)[0]:
         schema_uri = "file://" + os.path.abspath(schema_uri)
     schema_raw_doc = metaschema_loader.fetch(schema_uri)
 
@@ -207,7 +209,7 @@ def main(argsl=None):  # type: (List[str]) -> int
     # Load target document and resolve refs
     try:
         uri = args.document
-        if not urlparse.urlparse(uri)[0]:
+        if not parse.urlparse(uri)[0]:
             doc = "file://" + os.path.abspath(uri)
         document, doc_metadata = document_loader.resolve_ref(uri)
     except (validate.ValidationException, RuntimeError) as e:

schema_salad/makedoc.py

@@ -1,20 +1,23 @@
 from __future__ import absolute_import
+
 import mistune
-from . import schema
+import argparse
 import json
 import os
 import copy
 import re
 import sys
 from StringIO import StringIO
 import logging
-import urlparse
+
+from . import schema
 from schema_salad.utils import add_dictlist, aslist
-import re
-import argparse
-from typing import cast, Any, Dict, IO, List, Optional, Set, Text, Union
+
 import six
 from six.moves import range
+from six.moves.urllib import parse
+
+from typing import cast, Any, Dict, IO, List, Optional, Set, Text, Union
 
 _logger = logging.getLogger("salad")
 
@@ -38,7 +41,7 @@ def has_types(items):  # type: (Any) -> List[basestring]
 
 
 def linkto(item):  # type: (Text) -> Text
-    _, frg = urlparse.urldefrag(item)
+    _, frg = parse.urldefrag(item)
     return "[%s](#%s)" % (frg, to_id(frg))
 
 
@@ -206,8 +209,8 @@ def __init__(self, toc, j, renderlist, redirects, primitiveType):
                             if tp not in self.uses:
                                 self.uses[tp] = []
                             if (t["name"], f["name"]) not in self.uses[tp]:
-                                _, frg1 = urlparse.urldefrag(t["name"])
-                                _, frg2 = urlparse.urldefrag(f["name"])
+                                _, frg1 = parse.urldefrag(t["name"])
+                                _, frg2 = parse.urldefrag(f["name"])
                                 self.uses[tp].append((frg1, frg2))
                             if tp not in basicTypes and tp not in self.record_refs[t["name"]]:
                                 self.record_refs[t["name"]].append(tp)
@@ -268,7 +271,7 @@ def typefmt(self,
             elif str(tp) in basicTypes:
                 return """<a href="%s">%s</a>""" % (self.primitiveType, schema.avro_name(str(tp)))
             else:
-                _, frg = urlparse.urldefrag(tp)
+                _, frg = parse.urldefrag(tp)
                 if frg is not '':
                     tp = frg
                 return """<a href="#%s">%s</a>""" % (to_id(tp), tp)
@@ -327,7 +330,7 @@ def extendsfrom(item, ex):
                 lines.append(l)
             f["doc"] = "\n".join(lines)
 
-            _, frg = urlparse.urldefrag(f["name"])
+            _, frg = parse.urldefrag(f["name"])
             num = self.toc.add_entry(depth, frg)
             doc = "%s %s %s\n" % (("#" * depth), num, frg)
         else:

schema_salad/ref_resolver.py

@@ -5,11 +5,15 @@
 import hashlib
 import logging
 import collections
+
+import six
+from six.moves import range
 import urllib
-import urlparse
+# import urlparse
+from six.moves.urllib import parse
+
 import re
 import copy
-import urllib
 from StringIO import StringIO
 
 from . import validate
@@ -29,8 +33,7 @@
 import xml.sax
 from typing import (cast, Any, AnyStr, Callable, Dict, List, Iterable,
         Optional, Set, Text, Tuple, TypeVar, Union)
-import six
-from six.moves import range
+
 
 _logger = logging.getLogger("salad")
 ContextType = Dict[six.text_type, Union[Dict, six.text_type, Iterable[six.text_type]]]
@@ -54,7 +57,7 @@ def file_uri(path, split_frag=False):  # type: (str, bool) -> str
         return "file://%s%s" % (urlpath, frag)
 
 def uri_file_path(url):  # type: (str) -> str
-    split = urlparse.urlsplit(url)
+    split = parse.urlsplit(url)
     if split.scheme == "file":
         return urllib.url2pathname(
             str(split.path)) + ("#" + urllib.unquote(str(split.fragment))
@@ -126,7 +129,7 @@ def fetch_text(self, url):
         if url in self.cache:
             return self.cache[url]
 
-        split = urlparse.urlsplit(url)
+        split = parse.urlsplit(url)
         scheme, path = split.scheme, split.path
 
         if scheme in [u'http', u'https'] and self.session is not None:
@@ -156,7 +159,7 @@ def check_exists(self, url):  # type: (Text) -> bool
         if url in self.cache:
             return True
 
-        split = urlparse.urlsplit(url)
+        split = parse.urlsplit(url)
         scheme, path = split.scheme, split.path
 
         if scheme in [u'http', u'https'] and self.session is not None:
@@ -172,7 +175,7 @@ def check_exists(self, url):  # type: (Text) -> bool
             raise ValueError('Unsupported scheme in url: %s' % url)
 
     def urljoin(self, base_url, url):  # type: (Text, Text) -> Text
-        return urlparse.urljoin(base_url, url)
+        return parse.urljoin(base_url, url)
 
 class Loader(object):
     def __init__(self,
@@ -187,7 +190,7 @@ def __init__(self,
                  ):
         # type: (...) -> None
 
-        normalize = lambda url: urlparse.urlsplit(url).geturl()
+        normalize = lambda url: parse.urlsplit(url).geturl()
         if idx is not None:
             self.idx = idx
         else:
@@ -273,20 +276,20 @@ def expand_url(self,
             if prefix in self.vocab:
                 url = self.vocab[prefix] + url[len(prefix) + 1:]
 
-        split = urlparse.urlsplit(url)
+        split = parse.urlsplit(url)
 
         if (bool(split.scheme) or url.startswith(u"$(")
             or url.startswith(u"${")):
             pass
         elif scoped_id and not bool(split.fragment):
-            splitbase = urlparse.urlsplit(base_url)
+            splitbase = parse.urlsplit(base_url)
             frg = u""
             if bool(splitbase.fragment):
                 frg = splitbase.fragment + u"/" + split.path
             else:
                 frg = split.path
             pt = splitbase.path if splitbase.path != '' else "/"
-            url = urlparse.urlunsplit(
+            url = parse.urlunsplit(
                 (splitbase.scheme, splitbase.netloc, pt, splitbase.query, frg))
         elif scoped_ref is not None and not split.fragment:
             pass
@@ -493,7 +496,7 @@ def resolve_ref(self,
                 doc_url = url
             else:
                 # Load structured document
-                doc_url, frg = urlparse.urldefrag(url)
+                doc_url, frg = parse.urldefrag(url)
                 if doc_url in self.idx and (not mixin):
                     # If the base document is in the index, it was already loaded,
                     # so if we didn't find the reference earlier then it must not
@@ -869,7 +872,7 @@ def fetch(self, url, inject_ids=True):  # type: (Text, bool) -> Any
 
     def validate_scoped(self, field, link, docid):
         # type: (Text, Text, Text) -> Text
-        split = urlparse.urlsplit(docid)
+        split = parse.urlsplit(docid)
         sp = split.fragment.split(u"/")
         n = self.scoped_ref_fields[field]
         while n > 0 and len(sp) > 0:
@@ -878,7 +881,7 @@ def validate_scoped(self, field, link, docid):
         tried = []
         while True:
             sp.append(link)
-            url = urlparse.urlunsplit((
+            url = parse.urlunsplit((
                 split.scheme, split.netloc, split.path, split.query,
                 u"/".join(sp)))
             tried.append(url)

schema_salad/schema.py

@@ -9,9 +9,10 @@
 import avro.schema
 from . import validate
 import json
-import urlparse
-import os
+# import urlparse
+from six.moves.urllib import parse
 import six
+import os
 AvroSchemaFromJSONData = avro.schema.make_avsc_object
 # AvroSchemaFromJSONData=avro.schema.SchemaFromJSONData
 from avro.schema import Names, SchemaParseException
@@ -378,7 +379,7 @@ def replace_type(items, spec, loader, found):
 
 
 def avro_name(url):  # type: (AnyStr) -> AnyStr
-    doc_url, frg = urlparse.urldefrag(url)
+    doc_url, frg = parse.urldefrag(url)
     if frg != '':
         if '/' in frg:
             return frg[frg.rindex('/') + 1:]

schema_salad/tests/test_fetch.py

@@ -9,7 +9,8 @@
 import ruamel.yaml as yaml
 import json
 import os
-import urlparse
+# import urlparse
+from six.moves.urllib import parse
 
 class TestFetcher(unittest.TestCase):
     def test_fetcher(self):
@@ -32,14 +33,14 @@ def check_exists(self, url):  # type: (unicode) -> bool
                     return False
 
             def urljoin(self, base, url):
-                urlsp = urlparse.urlsplit(url)
+                urlsp = parse.urlsplit(url)
                 if urlsp.scheme:
                     return url
-                basesp = urlparse.urlsplit(base)
+                basesp = parse.urlsplit(base)
 
                 if basesp.scheme == "keep":
                     return base + "/" + url
-                return urlparse.urljoin(base, url)
+                return parse.urljoin(base, url)
 
         loader = schema_salad.ref_resolver.Loader({}, fetcher_constructor=TestFetcher)
         self.assertEqual({"hello": "foo"}, loader.resolve_ref("foo.txt")[0])

schema_salad/validate.py

@@ -3,7 +3,8 @@
 import avro.schema
 from avro.schema import Schema
 import sys
-import urlparse
+# import urlparse
+from six.moves.urllib import parse
 import re
 import logging
 
@@ -300,7 +301,7 @@ def validate_ex(expected_schema,                  # type: Schema
                     if (d not in identifiers and strict) and (
                             d not in foreign_properties and strict_foreign_properties) and not raise_ex:
                         return False
-                    split = urlparse.urlsplit(d)
+                    split = parse.urlsplit(d)
                     if split.scheme:
                         err = sl.makeError(u"unrecognized extension field `%s`%s."
                                            "  Did you include "