Merge branch 'dev' into add-th_tdtb

wannaphong · web-flow · commit 753cd6cb3824 · 2024-06-11T02:16:53.000+07:00
diff --git a/CITATION.cff b/CITATION.cff
@@ -1,4 +1,4 @@
-cff-version: 1.1.0
+cff-version: "1.2.0"
 message: "If you use this software, please cite it as below."
 authors:
 - family-names: "Phatthiyaphaibun"
@@ -20,5 +20,6 @@ authors:
   given-names: "Pattarawat"
   orcid: "https://orcid.org/0000-0000-0000-0000"
 title: "PyThaiNLP: Thai Natural Language Processing in Python"
-version: v5.0.2
-date-released: 2024-04-03
+version: v5.0.4
+license: Apache-2.0
+date-released: 2024-06-02
diff --git a/README.md b/README.md
@@ -23,7 +23,7 @@ PyThaiNLP เป็นไลบารีภาษาไพทอนสำหร
 
 | Version | Description | Status |
 |:------:|:--:|:------:|
-| [5.0.2](https://github.com/PyThaiNLP/pythainlp/releases) | Stable | [Change Log](https://github.com/PyThaiNLP/pythainlp/issues/788) |
+| [5.0.4](https://github.com/PyThaiNLP/pythainlp/releases) | Stable | [Change Log](https://github.com/PyThaiNLP/pythainlp/issues/788) |
 | [`dev`](https://github.com/PyThaiNLP/pythainlp/tree/dev) | Release Candidate for 5.1 | [Change Log](https://github.com/PyThaiNLP/pythainlp/issues/900) |
 
 ## Getting Started
diff --git a/README_TH.md b/README_TH.md
@@ -20,7 +20,7 @@ PyThaiNLP เป็นไลบารีภาษาไพทอนสำหร
 
 | รุ่น | คำอธิบาย | สถานะ |
 |:------:|:--:|:------:|
-| [5.0.2](https://github.com/PyThaiNLP/pythainlp/releases) | Stable | [Change Log](https://github.com/PyThaiNLP/pythainlp/issues/788) |
+| [5.0.4](https://github.com/PyThaiNLP/pythainlp/releases) | Stable | [Change Log](https://github.com/PyThaiNLP/pythainlp/issues/788) |
 | [`dev`](https://github.com/PyThaiNLP/pythainlp/tree/dev) | Release Candidate for 5.1  | [Change Log](https://github.com/PyThaiNLP/pythainlp/issues/900) |
 
 ติดตามพวกเราบน [PyThaiNLP Facebook page](https://www.facebook.com/pythainlp/) เพื่อรับข่าวสารเพิ่มเติม
diff --git a/pythainlp/__init__.py b/pythainlp/__init__.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
-__version__ = "5.0.2"
+__version__ = "5.0.4"
 
 thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ"  # 44 chars
 
diff --git a/pythainlp/corpus/corpus_license.md b/pythainlp/corpus/corpus_license.md
@@ -46,13 +46,15 @@ https://creativecommons.org/licenses/by/4.0/
 
 | Filename                  | Description                                                                                           |
 | ------------------------- | ----------------------------------------------------------------------------------------------------- |
-| pos_orchid_perceptron.pkl | Part-of-speech tagging model, trained from ORCHID data, using perceptron                              |
+| pos_orchid_perceptron.json | Part-of-speech tagging model, trained from ORCHID data, using perceptron                              |
 | pos_orchid_unigram.json   | Part-of-speech tagging model, trained from ORCHID data, using unigram                                 |
-| pos_ud_perceptron.pkl     | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using perceptron |
-| pos_ud_unigram.json       | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using unigram    |
+| pos_ud_perceptron-v0.2.json     | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using perceptron |
+| pos_ud_unigram-v0.2.json       | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using unigram    |
 | sentenceseg_crfcut.model  | Sentence segmentation model, trained from TED subtitles, using CRF                                    |
 | tdtb-pt_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using perceptron                              |
 | tdtb-unigram_tagger.json   | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using unigram                                 |
+| pos_tud_perceptron.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using perceptron                              |
+| pos_tud_unigram.json   | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using unigram                                 |
 
 
 ## Thai Dictionary for ICU BreakIterator
diff --git a/pythainlp/corpus/pos_tud_perceptron.json b/pythainlp/corpus/pos_tud_perceptron.json
diff --git a/pythainlp/corpus/pos_tud_unigram.json b/pythainlp/corpus/pos_tud_unigram.json
diff --git a/pythainlp/tag/perceptron.py b/pythainlp/tag/perceptron.py
@@ -21,10 +21,14 @@
 
 _BLACKBOARD_NAME = "blackboard_pt_tagger"
 
+_TUD_FILENAME = "pos_tud_perceptron.json"
+_TUD_PATH = os.path.join(corpus_path(), _TUD_FILENAME)
+
 _ORCHID_TAGGER = None
 _PUD_TAGGER = None
 _BLACKBOARD_TAGGER = None
 _TDTB_TAGGER = None
+_TUD_TAGGER = None
 
 
 def _orchid_tagger():
@@ -55,6 +59,13 @@ def _tdtb():
     return _TDTB_TAGGER
 
 
+def _tud_tagger():
+    global _TUD_TAGGER
+    if not _TUD_TAGGER:
+        _TUD_TAGGER = PerceptronTagger(path=_TUD_PATH)
+    return _TUD_TAGGER
+
+
 def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
     """
     :param list words: a list of tokenized words
@@ -80,6 +91,9 @@ def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
         word_tags = blackboard.post_process(word_tags, to_ud)
     elif corpus in ("tdtb"):
         word_tags = _tdtb().tag(words)
+    elif corpus in ("tud"):
+        tagger = _tud_tagger()
+        word_tags = tagger.tag(words)
     else:  # by default, use "pud" for corpus
         tagger = _pud_tagger()
         word_tags = tagger.tag(words)
diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py
@@ -32,6 +32,9 @@ def pos_tag(
             <https://github.com/nlp-chula/thai-discourse-treebank/tree/main>`_ \
             , natively use Universal POS tags
         * *tnc* - Thai National Corpus (support tltk engine only)
+        * *tdtb* - `Thai Discourse Treebank <https://github.com/nlp-chula/thai-discourse-treebank>`_
+        * *tud* - `Thai Universal Dependency Treebank (TUD)\
+            <https://github.com/nlp-chula/TUD>`_ \
     :return: a list of tuples (word, POS tag)
     :rtype: list[tuple[str, str]]
 
@@ -100,6 +103,7 @@ def pos_tag(
         "orchid_ud",
         "pud",
         "tdtb",
+        "tud",
     ]
 
     if engine == "perceptron" and corpus in _support_corpus:
diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py
@@ -83,7 +83,7 @@ class ThaiNameTagger:
     :Example:
     ::
 
-        from pythainlp.tag.named_entity import ThaiNameTagger
+        from pythainlp.tag.thainer import ThaiNameTagger
 
         thainer14 = ThaiNameTagger(version="1.4")
         thainer14.get_ner("วันที่ 15 ก.ย. 61 ทดสอบระบบเวลา 14:49 น.")
@@ -133,7 +133,7 @@ def get_ner(
 
         :Example:
 
-            >>> from pythainlp.tag.named_entity import ThaiNameTagger
+            >>> from pythainlp.tag.thainer import ThaiNameTagger
             >>>
             >>> ner = ThaiNameTagger()
             >>> ner.get_ner("วันที่ 15 ก.ย. 61 ทดสอบระบบเวลา 14:49 น.")
diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py
@@ -22,10 +22,14 @@
 
 _BLACKBOARD_NAME = "blackboard_unigram_tagger"
 
+_TUD_FILENAME = "pos_tud_unigram.json"
+_TUD_PATH = os.path.join(corpus_path(), _TUD_FILENAME)
+
 _ORCHID_TAGGER = None
 _PUD_TAGGER = None
 _BLACKBOARD_TAGGER = None
 _TDTB_TAGGER = None
+_TUD_TAGGER = None
 
 
 def _orchid_tagger():
@@ -61,6 +65,14 @@ def _thai_tdtb():
     return _TDTB_TAGGER
 
 
+def _tud_tagger():
+    global _TUD_TAGGER
+    if not _TUD_TAGGER:
+        with open(_TUD_PATH, encoding="utf-8-sig") as fh:
+            _TUD_TAGGER = json.load(fh)
+    return _TUD_TAGGER
+
+
 def _find_tag(
     words: List[str], dictdata: dict, default_tag: str = ""
 ) -> List[Tuple[str, str]]:
@@ -96,6 +108,8 @@ def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
         word_tags = blackboard.post_process(word_tags, to_ud)
     elif corpus in ("tdtb"):
         word_tags = _find_tag(words, _thai_tdtb())
+    elif corpus in ("tud"):
+        word_tags = _find_tag(words, _tud_tagger())
     else:  # by default, use "pud" for corpus
         word_tags = _find_tag(words, _pud_tagger())
 
diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py
@@ -61,8 +61,8 @@ def rejoin_formatted_num(segments: List[str]) -> List[str]:
                 connected_token += segments[segment_idx]
                 pos += len(segments[segment_idx])
                 segment_idx += 1
-
-            tokens_joined.append(connected_token)
+            if connected_token:
+                tokens_joined.append(connected_token)
             match = next(matching_results, None)
         else:
             tokens_joined.append(token)
diff --git a/pythainlp/util/normalize.py b/pythainlp/util/normalize.py
@@ -283,9 +283,12 @@ def maiyamok(sent: Union[str, List[str]]) -> List[str]:
         if "ๆ" == text:
             text = _list_word[i - 1]
         elif "ๆ" in text:
-            text = text.replace("ๆ", "")
-            _list_word.append(text)
+            count = text.count("ๆ")
+            text = _list_word[i - 1]
+            for _ in range(count):
+                _list_word.append(text)
             i += 1
+            continue
         _list_word.append(text)
         i += 1
     return _list_word
diff --git a/setup.cfg b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 5.0.2
+current_version = 5.0.4
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?
diff --git a/setup.py b/setup.py
@@ -161,7 +161,7 @@
 
 setup(
     name="pythainlp",
-    version="5.0.2",
+    version="5.0.4",
     description="Thai Natural Language Processing library",
     long_description=readme,
     long_description_content_type="text/markdown",
diff --git a/tests/test_tag.py b/tests/test_tag.py
@@ -51,6 +51,8 @@ def test_pos_tag(self):
         self.assertEqual(unigram.tag([], corpus="orchid"), [])
         self.assertEqual(unigram.tag(None, corpus="blackboard"), [])
         self.assertEqual(unigram.tag([], corpus="blackboard"), [])
+        self.assertEqual(unigram.tag(None, corpus="tud"), [])
+        self.assertEqual(unigram.tag([], corpus="tud"), [])
         self.assertIsNotNone(
             pos_tag(tokens, engine="unigram", corpus="orchid")
         )
@@ -74,6 +76,8 @@ def test_pos_tag(self):
         self.assertIsNotNone(
             pos_tag([""], engine="unigram", corpus="tdtb")
         )
+        self.assertIsNotNone(pos_tag(tokens, engine="unigram", corpus="tud"))
+        self.assertIsNotNone(pos_tag([""], engine="unigram", corpus="tud"))
         self.assertEqual(
             pos_tag(["คุณ", "กำลัง", "ประชุม"], engine="unigram"),
             [("คุณ", "PPRS"), ("กำลัง", "XVBM"), ("ประชุม", "VACT")],
@@ -94,6 +98,8 @@ def test_pos_tag(self):
         self.assertEqual(perceptron.tag([], corpus="pud"), [])
         self.assertEqual(perceptron.tag(None, corpus="blackboard"), [])
         self.assertEqual(perceptron.tag([], corpus="blackboard"), [])
+        self.assertEqual(perceptron.tag(None, corpus="tud"), [])
+        self.assertEqual(perceptron.tag([], corpus="tud"), [])
         self.assertIsNotNone(
             pos_tag(tokens, engine="perceptron", corpus="orchid")
         )
@@ -115,6 +121,9 @@ def test_pos_tag(self):
         self.assertIsNotNone(
             pos_tag(tokens, engine="perceptron", corpus="tdtb")
         )
+        self.assertIsNotNone(
+            pos_tag(tokens, engine="perceptron", corpus="tud")
+        )
         self.assertIsNotNone(pos_tag(tokens, engine="tltk"))
 
         self.assertEqual(pos_tag_sents(None), [])
diff --git a/tests/test_util.py b/tests/test_util.py
@@ -531,6 +531,10 @@ def test_normalize(self):
             maiyamok("เด็กๆชอบไปโรงเรียน"),
             ["เด็ก", "เด็ก", "ชอบ", "ไป", "โรงเรียน"],
         )
+        self.assertEqual(
+            maiyamok("เด็กๆๆชอบไปโรงเรียน"),
+            ['เด็ก', 'เด็ก', 'เด็ก', 'ชอบ', 'ไป', 'โรงเรียน'],
+        ) # 914
         self.assertEqual(
             maiyamok(
                 ["ทำไม", "คน", "ดี", " ", "ๆ", "ๆ", " ", "ถึง", "ทำ", "ไม่ได้"]
