Spanish fix

.gitignore

@@ -6,6 +6,10 @@ tests/.DS_Store
 .DS_Store
 *.egg-info
 .project
+env/
+venv/
 dist/
 dist/*
-
+*.csv
+__pycache__/
+*.json

email_reply_parser/__init__.py

@@ -1,80 +1,172 @@
 """
-    email_reply_parser is a python library port of GitHub's Email Reply Parser.
-
-    For more information, visit https://github.com/zapier/email-reply-parser
+email_reply_parser is a python library port of GitHub's Email Reply Parser.
+For more information, visit https://github.com/zapier/email_reply_parser
 """
-
+import os
 import re
+import json
 
 
 class EmailReplyParser(object):
     """ Represents a email message that is parsed.
     """
+    def __init__(self, language='en'):
+        dir_path = os.path.dirname(__file__)
+        with open(dir_path + "/languages_support.json", "r") as read_file:
+            self.words_map = json.load(read_file)
+        if language in self.words_map:
+            self.language = language
+        else:
+            self.language = 'en'
 
-    @staticmethod
-    def read(text):
+    def read(self, text):
         """ Factory method that splits email into list of fragments
-
             text - A string email body
-
             Returns an EmailMessage instance
         """
-        return EmailMessage(text).read()
+        return EmailMessage(text, self.language, self.words_map).read()
 
-    @staticmethod
-    def parse_reply(text):
+    def parse_reply(self, text):
         """ Provides the reply portion of email.
-
             text - A string email body
-
             Returns reply body message
         """
-        return EmailReplyParser.read(text).reply
+        return self.read(text).reply
 
 
 class EmailMessage(object):
     """ An email message represents a parsed email body.
     """
-
-    SIG_REGEX = re.compile(r'(--|__|-\w)|(^Sent from my (\w+\s*){1,3})')
-    QUOTE_HDR_REGEX = re.compile('On.*wrote:$')
-    QUOTED_REGEX = re.compile(r'(>+)')
-    HEADER_REGEX = re.compile(r'^\*?(From|Sent|To|Subject):\*? .+')
-    _MULTI_QUOTE_HDR_REGEX = r'(?!On.*On\s.+?wrote:)(On\s(.+?)wrote:)'
-    MULTI_QUOTE_HDR_REGEX = re.compile(_MULTI_QUOTE_HDR_REGEX, re.DOTALL | re.MULTILINE)
-    MULTI_QUOTE_HDR_REGEX_MULTILINE = re.compile(_MULTI_QUOTE_HDR_REGEX, re.DOTALL)
-
-    def __init__(self, text):
+    def __init__(self, text, language, words_map):
         self.fragments = []
         self.fragment = None
         self.text = text.replace('\r\n', '\n')
         self.found_visible = False
+        self.SIG_REGEX = None
+        self.QUOTE_HDR_REGEX = None
+        self.QUOTED_REGEX = None
+        self.HEADER_REGEX = None
+        self._MULTI_QUOTE_HDR_REGEX = None
+        self.MULTI_QUOTE_HDR_REGEX = None
+        self.MULTI_QUOTE_HDR_REGEX_MULTILINE = None
+        self.words_map = words_map
+        self.language = language
+        self.default_language = 'en'
+        self.set_regex()
+
+    def default_quoted_header(self):
+        self.QUOTED_REGEX = re.compile(r'(>+)')
+        self.HEADER_REGEX = re.compile(
+            r'^[* ]?(' + self.words_map[self.language]['From'] \
+            + '|' + self.words_map[self.language]['Sent'] \
+            + '|' + self.words_map[self.language]['To']
+            + ')\s*:\*? .+|.+(mailto:).+'
+        )
+
+    def warnings(self):
+        self.WARNING_REGEX = re.compile(
+            r'(CAUTION:|NOTICE:|Confidentiality Notice:|Please do not reply|This electronic mail'
+            r'|Disclaimer: This message is intended'
+            r'|This message and any attachments are solely'
+            r'|This email contains privileged information'
+            r'|The information contained|This email has been scanned|This message and any associated files'
+            r'|This email and any files transmitted|This message is for the recipients'
+            r'|The information provided within this communication'
+            r'|This message (including any attachments) is intended'
+            r'|The [cC]ontents are confidential|This communication with its contents'
+            r'|Please consider the environment before printing this email) [a-zA-Z0-9:;.,?!()@&/\'\"\“\” \-]*'
+        )
+
+    def nl_support(self):
+        self.SIG_REGEX = re.compile(
+            r'(--|__|-\w)|(^' + self.words_map[self.language]['Sent from'] + '(\w+\s*){1,3})'
+        )
+        self.QUOTE_HDR_REGEX = re.compile('Op.*schreef.*>:$')
+        self._MULTI_QUOTE_HDR_REGEX = r'(?!Op.*Op\s.+?schreef.*>:)(Op\s(.+?)schreef.*>:)'
+
+    def de_support(self):
+        self.SIG_REGEX = re.compile(r'(--|__|-\w)|(^' + self.words_map[self.language]['Sent from'] + '(\w+\s*){1,3})')
+        self.QUOTE_HDR_REGEX = re.compile('Am.*schrieb.*>:$')
+        self._MULTI_QUOTE_HDR_REGEX = r'(?!Am.*Am\s.+?schrieb.*>:)(Am\s(.+?)schrieb.*>:)'
+
+    def fr_support(self):
+        self.SIG_REGEX = re.compile(
+            r'(--|__|-\w)|(^' + self.words_map[self.language]['Sent from'] \
+            + '(\w+\s*){1,3})|(.*(cordialement|bonne r[ée]ception|salutations'
+            r'|cdlt|cdt|crdt|regards|best regard|bonne journ[ée]e))',
+            re.IGNORECASE
+        )
+        self.QUOTE_HDR_REGEX = re.compile('Le.*a écrit.*[> ]:$')
+        self._MULTI_QUOTE_HDR_REGEX = r'(?!Le.*Le\s.+?a écrit[a-zA-Z0-9.:;<>()&@ -]*:)(Le\s(.+?)a écrit[a-zA-Z0-9.:;<>()&@ -]*:)'
+
+    def en_support(self):
+        self.SIG_REGEX = re.compile(r'(--|__|-\w)|(^Sent from (\w+\s*){1,6})')
+        self.QUOTE_HDR_REGEX = re.compile('\s*On.*wrote\s*:$')
+        self.QUOTED_REGEX = re.compile(r'(>+)|((&gt;)+)')
+        self._MULTI_QUOTE_HDR_REGEX = r'(?!On.*On\s.+?wrote\s*:)(On\s(.+?)wrote\s*:)'
+
+    def es_support(self):
+        self.SIG_REGEX = re.compile(r'(--|__|-\w)|(^Enviado desde (\w+\s*){1,6})')
+        self.QUOTE_HDR_REGEX = re.compile('\s*El.*escribió\s*:$')
+        self._MULTI_QUOTE_HDR_REGEX = r'(?!El.*El\s.+?escribió\s*:)(El\s(.+?)escribió\s*:)'
+
+    def ja_support(self):
+        self.SIG_REGEX = re.compile(r'--|__|-\w')
+        self.QUOTE_HDR_REGEX = re.compile(
+            r'[0-9]*年[0-9]*月[0-9]*日[\u3000-\u303F\u3040-\u309F\u30A0-\u30FF\uFF00-\uFFEF\u4E00-\u9FAF\u2605-\u2606\u2190-\u2195\u203Ba-zA-Z0-9.:;<>()&@ -]*:?$'
+        )
+        self.QUOTED_REGEX = re.compile(r'(>+)|((&gt;)+)')
+        self._MULTI_QUOTE_HDR_REGEX = r'(?!On.*On\s.+?wrote\s*:)(On\s(.+?)wrote\s*:)' # Dummy multiline: doesnt work for japanese due to BeautifulSoup insreting new lines before ":" character
+
+    def fi_support(self):
+        self.SIG_REGEX = re.compile(r'(--|__|-\w)|(^Lähetetty (\w+\s*){1,3})|(^Hanki Outlook for.*)')
+        self.QUOTE_HDR_REGEX = re.compile('(.+?kirjoitti(.+?kello.+?)?:)')
+        self.QUOTED_REGEX = re.compile(r'(>+)|((&gt;)+)')
+        self._MULTI_QUOTE_HDR_REGEX = r'(?!.+?kirjoitti.+?kirjoitti[a-zA-Z0-9.:;<>()&@ -]*:$)((.+?)kirjoitti[a-zA-Z0-9.:;<>()&@ -]*:$)'
+
+    def set_regex(self):
+        if hasattr(self, self.language+"_support"):
+            getattr(self, self.language+"_support")()
+            self.default_quoted_header()
+        else:
+            self.SIG_REGEX = re.compile(
+                r'(--|__|-\w)|(^(' + self.words_map[self.language]['Sent from'] \
+                + '|' + self.words_map[self.default_language]['Sent from'] \
+                + ')(\w+\s*){1,3})'
+            )
+            self.QUOTE_HDR_REGEX = re.compile('.*' + self.words_map[self.language]['wrote'] + '\s?:$')
+            self.default_quoted_header()
+            self._MULTI_QUOTE_HDR_REGEX = r'(?!.+?' + self.words_map[self.language]['wrote'] \
+                + '\s*:\s*)(On\s(.+?)' + self.words_map[self.language]['wrote'] + ':)'
+        self.warnings()
+        self.FOLLOW_UP_HDR_REGEX = re.compile(r'(?<!^)This is a follow-up to your previous request.*', re.DOTALL)
+        self.MULTI_QUOTE_HDR_REGEX = re.compile(self._MULTI_QUOTE_HDR_REGEX, re.DOTALL | re.MULTILINE)
+        self.MULTI_QUOTE_HDR_REGEX_MULTILINE = re.compile(self._MULTI_QUOTE_HDR_REGEX, re.DOTALL)
 
     def read(self):
         """ Creates new fragment for each line
             and labels as a signature, quote, or hidden.
-
             Returns EmailMessage instance
         """
-
+        self.text = self.text.strip()
         self.found_visible = False
-
         is_multi_quote_header = self.MULTI_QUOTE_HDR_REGEX_MULTILINE.search(self.text)
         if is_multi_quote_header:
             self.text = self.MULTI_QUOTE_HDR_REGEX.sub(is_multi_quote_header.groups()[0].replace('\n', ''), self.text)
-
+        self.text = self.FOLLOW_UP_HDR_REGEX.sub('', self.text)
         # Fix any outlook style replies, with the reply immediately above the signature boundary line
         #   See email_2_2.txt for an example
         self.text = re.sub('([^\n])(?=\n ?[_-]{7,})', '\\1\n', self.text, re.MULTILINE)
 
+        self.text = re.sub(self.WARNING_REGEX, '\n', self.text)
         self.lines = self.text.split('\n')
         self.lines.reverse()
 
         for line in self.lines:
-            self._scan_line(line)
+            if line.strip():
+                self._scan_line(line.strip())
 
         self._finish_fragment()
-
         self.fragments.reverse()
 
         return self
@@ -85,42 +177,29 @@ def reply(self):
         """
         reply = []
         for f in self.fragments:
-            if not (f.hidden or f.quoted):
+            if not (f.hidden or f.quoted or f.signature):
                 reply.append(f.content)
         return '\n'.join(reply)
 
     def _scan_line(self, line):
         """ Reviews each line in email message and determines fragment type
-
             line - a row of text from an email message
         """
         is_quote_header = self.QUOTE_HDR_REGEX.match(line) is not None
         is_quoted = self.QUOTED_REGEX.match(line) is not None
         is_header = is_quote_header or self.HEADER_REGEX.match(line) is not None
-
-        if self.fragment and len(line.strip()) == 0:
-            if self.SIG_REGEX.match(self.fragment.lines[-1].strip()):
-                self.fragment.signature = True
-                self._finish_fragment()
-
-        if self.fragment \
+        if self.fragment and self.SIG_REGEX.match(line.strip()):
+            self.fragment.signature = True
+            self.fragment.lines.append(line)
+            self._finish_fragment()
+        elif self.fragment \
                 and ((self.fragment.headers == is_header and self.fragment.quoted == is_quoted) or
                          (self.fragment.quoted and (is_quote_header or len(line.strip()) == 0))):
-
             self.fragment.lines.append(line)
         else:
             self._finish_fragment()
             self.fragment = Fragment(is_quoted, line, headers=is_header)
 
-    def quote_header(self, line):
-        """ Determines whether line is part of a quoted area
-
-            line - a row of the email message
-
-            Returns True or False
-        """
-        return self.QUOTE_HDR_REGEX.match(line[::-1]) is not None
-
     def _finish_fragment(self):
         """ Creates fragment
         """