fix: add more characters to illegal chars

This commit is contained in:
Callum Leslie committed 2022-07-11 17:56:24 +01:00
1 parent 29de549ef3
commit c1c72ca5af
1 file changed
+1 -1
+1 -1
View File
@@ -81,7 +81,7 @@ def sanitize_text(text: str) -> str:
result = re.sub(regex_urls, " ", text)
# note: not removing apostrophes
regex_expr = r"\s['|’]|['|’]\s|[\^_~@!&;#:\-%“”‘\"%\*/{}\[\]\(\)\\|<>=+]"
regex_expr = r"\s['|’]|['|’]\s|[\^_~@!&;#:\-–—%“”‘\"%\*/{}\[\]\(\)\\|<>=+]"
result = re.sub(regex_expr, " ", result)
result = result.replace("+", "plus").replace("&", "and")
# remove extra whitespace