---

 Doc/library/urlparse.rst                                           |   14 ++

 Doc/whatsnew/2.7.rst                                               |    7 +

 Lib/test/test_urlparse.py                                          |   49 ++++++++++

 Lib/urlparse.py                                                    |   12 ++

 Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst |    6 +

 5 files changed, 88 insertions(+)

​

--- a/Doc/library/urlparse.rst

+++ b/Doc/library/urlparse.rst

@@ -248,6 +248,10 @@ The :mod:`urlparse` module defines the f

    decomposed before parsing, or is not a Unicode string, no error will be

    raised.

+   Following the `WHATWG spec`_ that updates RFC 3986, ASCII

+   newline ``\n``, ``\r`` and tab ``\t`` characters are stripped

+   from the URL.

+

    .. versionadded:: 2.2

    .. versionchanged:: 2.5

@@ -257,6 +261,10 @@ The :mod:`urlparse` module defines the f

       Characters that affect netloc parsing under NFKC normalization will

       now raise :exc:`ValueError`.

+   .. versionchanged:: 3.6.14

+      ASCII newline and tab characters are stripped from the URL.

+

+.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser

 .. function:: urlunsplit(parts)

@@ -308,6 +316,11 @@ The :mod:`urlparse` module defines the f

 .. seealso::

+   `WHATWG`_ -  URL Living standard

+      Working Group for the URL Standard that defines URLs,

+      domains, IP addresses, the application/x-www-form-urlencoded format,

+      and their API.

+

    :rfc:`3986` - Uniform Resource Identifiers

       This is the current standard (STD66). Any changes to urlparse module

       should conform to this. Certain deviations could be observed, which are

@@ -332,6 +345,7 @@ The :mod:`urlparse` module defines the f

    :rfc:`1738` - Uniform Resource Locators (URL)

       This specifies the formal syntax and semantics of absolute URLs.

+.. _WHATWG: https://url.spec.whatwg.org/

 .. _urlparse-result-object:

--- a/Doc/whatsnew/2.7.rst

+++ b/Doc/whatsnew/2.7.rst

@@ -175,6 +175,13 @@ channel.  We reuse the ftp server IP add

 requiring the old behavior, set a ``trust_server_pasv_ipv4_address``

 attribute on your FTP instance to ``True``.  (See :issue:`43285`)

+The presence of newline or tab characters in parts of a URL allows for some

+forms of attacks. Following the WHATWG specification that updates RFC 3986,

+ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the

+URL by the parser :func:`urlparse` preventing such attacks. The removal

+characters are controlled by a new module level variable

+``urlparse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :issue:`43882`)

+

 Python 3.1 Features

 =======================

--- a/Lib/test/test_urlparse.py

+++ b/Lib/test/test_urlparse.py

@@ -492,6 +492,55 @@ class UrlParseTestCase(unittest.TestCase

         p = urlparse.urlsplit(url)

         self.assertEqual(p.port, None)

+    def test_urlsplit_remove_unsafe_bytes(self):

+        # Remove ASCII tabs and newlines from input, for http common case scenario.

+        url = "h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"

+        p = urlparse.urlsplit(url)

+        self.assertEqual(p.scheme, "http")

+        self.assertEqual(p.netloc, "www.python.org")

+        self.assertEqual(p.path, "/javascript:alert('msg')/")

+        self.assertEqual(p.query, "query=something")

+        self.assertEqual(p.fragment, "fragment")

+        self.assertEqual(p.username, None)

+        self.assertEqual(p.password, None)

+        self.assertEqual(p.hostname, "www.python.org")

+        self.assertEqual(p.port, None)

+        self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment")

+

+        # Remove ASCII tabs and newlines from input as bytes, for http common case scenario.

+        url = b"h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"

+        p = urlparse.urlsplit(url)

+        self.assertEqual(p.scheme, b"http")

+        self.assertEqual(p.netloc, b"www.python.org")

+        self.assertEqual(p.path, b"/javascript:alert('msg')/")

+        self.assertEqual(p.query, b"query=something")

+        self.assertEqual(p.fragment, b"fragment")

+        self.assertEqual(p.username, None)

+        self.assertEqual(p.password, None)

+        self.assertEqual(p.hostname, b"www.python.org")

+        self.assertEqual(p.port, None)

+        self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/?query=something#fragment")

+

+        # any scheme

+        url = "x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"

+        p = urlparse.urlsplit(url)

+        self.assertEqual(p.geturl(), "x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")

+

+        # Remove ASCII tabs and newlines from input as bytes, any scheme.

+        url = b"x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"

+        p = urlparse.urlsplit(url)

+        self.assertEqual(p.geturl(), b"x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")

+

+        # Unsafe bytes is not returned from urlparse cache.

+        # scheme is stored after parsing, sending an scheme with unsafe bytes *will not* return an unsafe scheme

+        url = "https://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"

+        scheme = "htt\nps"

+        for _ in range(2):

+            p = urlparse.urlsplit(url, scheme=scheme)

+            self.assertEqual(p.scheme, "https")

+            self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")

+

+

     def test_issue14072(self):

         p1 = urlparse.urlsplit('tel:+31-641044153')

         self.assertEqual(p1.scheme, 'tel')

--- a/Lib/urlparse.py

+++ b/Lib/urlparse.py

@@ -62,6 +62,9 @@ scheme_chars = ('abcdefghijklmnopqrstuvw

                 '0123456789'

                 '+-.')

+# Unsafe bytes to be removed per WHATWG spec

+_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']

+

 MAX_CACHE_SIZE = 20

 _parse_cache = {}

@@ -184,12 +187,21 @@ def _checknetloc(netloc):

                              "under NFKC normalization"

                              % netloc)

+

+def _remove_unsafe_bytes_from_url(url):

+    for b in _UNSAFE_URL_BYTES_TO_REMOVE:

+        url = url.replace(b, "")

+    return url

+

+

 def urlsplit(url, scheme='', allow_fragments=True):

     """Parse a URL into 5 components:

     <scheme>://<netloc>/<path>?<query>#<fragment>

     Return a 5-tuple: (scheme, netloc, path, query, fragment).

     Note that we don't break the components up in smaller bits

     (e.g. netloc is a single string) and we don't expand % escapes."""

+    url = _remove_unsafe_bytes_from_url(url)

+    scheme = _remove_unsafe_bytes_from_url(scheme)

     allow_fragments = bool(allow_fragments)

     key = url, scheme, allow_fragments, type(url), type(scheme)

     cached = _parse_cache.get(key, None)

--- /dev/null

+++ b/Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst

@@ -0,0 +1,6 @@

+The presence of newline or tab characters in parts of a URL could allow

+some forms of attacks.

+

+Following the controlling specification for URLs defined by WHATWG

+:func:`urlparse` now removes ASCII newlines and tabs from URLs,

+preventing such attacks.

​