Merge pull request #668 from PyThaiNLP/3.0

Update dev base from 3.0 base

Author: wannaphong

pythainlp/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "3.0.6dev0"
+__version__ = "3.0.6"
 
 thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ"  # 44 chars
 

pythainlp/tokenize/nercut.py

@@ -25,6 +25,7 @@ def segment(
         "DATE",
         "TIME",
     ],
+    tagger=_thainer
 ) -> List[str]:
     """
     Dictionary-based maximal matching word segmentation, constrained with
@@ -33,18 +34,17 @@ def segment(
 
     :param str text: text to be tokenized to words
     :parm list taglist: a list of named-entity tags to be used
+    :parm class tagger: ner tagger engine
     :return: list of words, tokenized from the text
     """
-    if not text or not isinstance(text, str):
+    if not isinstance(text, str):
         return []
 
-    global _thainer
-    tagged_words = _thainer.tag(text, pos=False)
+    tagged_words = tagger.tag(text, pos=False)
 
     words = []
     combining_word = ""
-    combining_word = ""
-    for curr_word, curr_tag in tagged_words:
+    for idx, (curr_word, curr_tag) in enumerate(tagged_words):
         if curr_tag != "O":
             tag = curr_tag[2:]
         else:
@@ -68,5 +68,15 @@ def segment(
         else:
             combining_word = ""
             words.append(curr_word)
+        if idx + 1 == len(tagged_words):
+            if (
+                curr_tag.startswith("B-") or curr_tag == "O"
+            ) and combining_word != "":
+                words.append(combining_word)
+                combining_word = ""
+                words.append(curr_word)
+            else:  # if tag is O
+                combining_word += curr_word
+                words.append(combining_word)
 
     return words

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.0.5
+current_version = 3.0.6
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?

setup.py

@@ -113,7 +113,7 @@
 
 setup(
     name="pythainlp",
-    version="3.0.6dev0",
+    version="3.0.6",
     description="Thai Natural Language Processing library",
     long_description=readme,
     long_description_content_type="text/markdown",

tests/test_tokenize.py

@@ -610,7 +610,9 @@ def test_nercut(self):
         self.assertEqual(nercut.segment(None), [])
         self.assertEqual(nercut.segment(""), [])
         self.assertIsNotNone(nercut.segment("ทดสอบ"))
-        self.assertIsNotNone(nercut.segment("ทดสอบ"))
+        self.assertIsNotNone(nercut.segment("ทุ๊กกโคนน"))
+        self.assertIsNotNone(nercut.segment("อือหือ"))
+        self.assertIsNotNone(nercut.segment("อย่าลืมอัพการ์ดนะจ๊ะ"))
         self.assertIsNotNone(word_tokenize("ทดสอบ", engine="nercut"))
 
     def test_ssg(self):