From ed348dfe42b7a33bbf1dc9026c2ae4f63eacf742 Mon Sep 17 00:00:00 2001 From: "1.v3m" <16155914@mersin.edu.tr> Date: Mon, 18 May 2020 02:33:18 +0300 Subject: [PATCH 1/3] bpo-38870: fixing unhandled hexescape in docstrings at ast.unparse --- Lib/ast.py | 15 ++++++++++++--- Lib/test/test_unparse.py | 6 +++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index 61fbe030a78252..8245554fddaa95 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1088,6 +1088,16 @@ def visit_Name(self, node): self.write(node.id) def _write_docstring(self, node): + def esc_char(c): + if c in ("\n", "\t"): + # In the AST form, we don't know the author's intentation + # about how this should be displayed. We'll only escape + # \n and \t, because they are more likely to be unescaped + # in the source + return c + else: + return c.encode('unicode_escape').decode('ascii') + self.fill() if node.kind == "u": self.write("u") @@ -1095,11 +1105,10 @@ def _write_docstring(self, node): value = node.value if value: # Preserve quotes in the docstring by escaping them - value = value.replace("\\", "\\\\") - value = value.replace('"""', '""\"') - value = value.replace("\r", "\\r") + value = "".join(map(esc_char, value)) if value[-1] == '"': value = value.replace('"', '\\"', -1) + value = value.replace('"""', '""\\"') self.write(f'"""{value}"""') diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 67dcb1dae79ff2..751dc330d9db5c 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -324,7 +324,11 @@ def test_docstrings(self): '\\t', '\n', '\\n', - '\r\\r\t\\t\n\\n' + '\r\\r\t\\t\n\\n', + '""">>> content = \"\"\"blabla\"\"\" <<<"""', + r'foo\n\x00' + '🐍⛎𩸽üéş^\X\BB\N{LONG RIGHTWARDS SQUIGGLE ARROW}' + ) for docstring in docstrings: # check as Module docstrings for easy testing From 6d94cd29348d5d9412ef0a4b45cd1554151b5350 Mon Sep 17 00:00:00 2001 From: CyberSaxosTiGER <16155914@mersin.edu.tr> Date: Mon, 18 May 2020 20:48:01 +0300 Subject: [PATCH 2/3] remove else Co-authored-by: Pablo Galindo --- Lib/ast.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index 8245554fddaa95..2a4e0f7cc17d82 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1095,8 +1095,7 @@ def esc_char(c): # \n and \t, because they are more likely to be unescaped # in the source return c - else: - return c.encode('unicode_escape').decode('ascii') + return c.encode('unicode_escape').decode('ascii') self.fill() if node.kind == "u": From ee4dc06dbc0557fd5c2fe3354f7fd72fa1f4444f Mon Sep 17 00:00:00 2001 From: CyberSaxosTiGER <16155914@mersin.edu.tr> Date: Mon, 18 May 2020 21:03:15 +0300 Subject: [PATCH 3/3] implicit string concat Co-authored-by: Batuhan Taskaya --- Lib/test/test_unparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 751dc330d9db5c..6d828721b7740e 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -326,7 +326,7 @@ def test_docstrings(self): '\\n', '\r\\r\t\\t\n\\n', '""">>> content = \"\"\"blabla\"\"\" <<<"""', - r'foo\n\x00' + r'foo\n\x00', '🐍⛎𩸽üéş^\X\BB\N{LONG RIGHTWARDS SQUIGGLE ARROW}' )