Merge pull request #826 from PyThaiNLP/add-abbreviation_to_full_text

Add pythainlp.util.abbreviation_to_full_text

Author: wannaphong

docs/api/util.rst

@@ -7,6 +7,7 @@ The :class:`pythainlp.util` contains utility functions, like text conversion and
 Modules
 -------
 
+.. autofunction:: abbreviation_to_full_text
 .. autofunction:: arabic_digit_to_thai_digit
 .. autofunction:: bahttext
 .. autofunction:: convert_years

docs/notes/installation.rst

@@ -39,6 +39,7 @@ where ``extras`` can be
   - ``wangchanglm`` (to support wangchanglm model)
   - ``wsd`` (to support pythainlp.wsd)
   - ``el`` (to support pythainlp.el)
+  - ``abbreviation`` (to support pythainlp.util.abbreviation_to_full_text)
   - ``full`` (install everything)
 
 For dependency details, look at `extras` variable in `setup.py <https://github.com/PyThaiNLP/pythainlp/blob/dev/setup.py>`_.

pythainlp/util/__init__.py

@@ -18,6 +18,7 @@
 
 __all__ = [
     "Trie",
+    "abbreviation_to_full_text",
     "arabic_digit_to_thai_digit",
     "bahttext",
     "convert_years",
@@ -125,3 +126,4 @@
 from pythainlp.util.phoneme import nectec_to_ipa, ipa_to_rtgs, remove_tone_ipa
 from pythainlp.util.encoding import tis620_to_utf8
 import pythainlp.util.spell_words as spell_words
+from pythainlp.util.abbreviation import abbreviation_to_full_text

pythainlp/util/abbreviation.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2016-2023 PyThaiNLP Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Thai abbreviation tools
+"""
+from typing import List, Tuple, Union
+
+
+def abbreviation_to_full_text(text: str, top_k: int=2) -> List[Tuple[str, Union[float, None]]]:
+    """
+    This function convert Thai text (with abbreviation) to full text.
+
+    This function use KhamYo for handles abbreviations.
+    See more `KhamYo <https://github.com/wannaphong/KhamYo>`_.
+
+    :param str text: Thai text
+    :param int top_k: Top K
+    :return: Thai full text that handles abbreviations as full text and cos scores (original text -  modified text).
+    :rtype: List[Tuple[str, Union[float, None]]]
+
+    :Example:
+    ::
+
+        from pythainlp.util import abbreviation_to_full_text
+
+        text = "รร.ของเราน่าอยู่"
+
+        abbreviation_to_full_text(text)
+        # output: [
+        # ('โรงเรียนของเราน่าอยู่', tensor(0.3734)), 
+        # ('โรงแรมของเราน่าอยู่', tensor(0.2438))
+        # ]
+    """
+    try:
+        from khamyo import replace as _replace
+    except ImportError:
+        raise ImportError(
+            """
+            This funtion need to use khamyo.
+            You can install by pip install khamyo or 
+            pip install pythainlp[abbreviation].
+            """
+        )
+    return _replace(text, top_k=top_k)

setup.py

@@ -128,6 +128,9 @@
     "el":{
         "multiel>=0.5"
     },
+    "abbreviation":{
+        "khamyo>=0.2.0"
+    },
     "full": [
         "PyYAML>=5.3.1",
         "attacut>=1.0.4",
@@ -162,6 +165,7 @@
         "ufal.chu-liu-edmonds>=1.0.2",
         "panphon>=0.20.0",
         "sentence-transformers>=2.2.2",
+        "khamyo>=0.2.0",
     ],
 }
 

tests/test_util.py

@@ -11,6 +11,7 @@
 from pythainlp.corpus.common import _THAI_WORDS_FILENAME
 from pythainlp.util import (
     Trie,
+    abbreviation_to_full_text,
     arabic_digit_to_thai_digit,
     bahttext,
     collate,
@@ -851,3 +852,6 @@ def test_spell_word(self):
         self.assertEqual(spell_word("เสื้อ"),['สอ', 'เอือ', 'ไม้โท', 'เสื้อ'])
         self.assertEqual(spell_word("คน"),['คอ', 'นอ', 'คน'])
         self.assertEqual(spell_word("คนดี"),['คอ', 'นอ', 'คน', 'ดอ', 'อี', 'ดี', 'คนดี'])
+    
+    def test_abbreviation_to_full_text(self):
+        self.assertIsInstance(abbreviation_to_full_text("รร.ของเราน่าอยู่", list))