Specify a limited test suite

.github/workflows/unittest.yml

@@ -1,4 +1,4 @@
-name: Unit test and code coverage
+name: Unit test and coverage
 
 on:
   push:
@@ -18,9 +18,14 @@ jobs:
       fail-fast: false
       matrix:
         os: ["macos-latest", "ubuntu-latest", "windows-latest"]
-        python-version: ["3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
 
     runs-on: ${{ matrix.os }}
+    env:
+      PYICU_WIN_VER: 2.14
+      INSTALL_PYICU_WIN: false
+      INSTALL_TORCH: false
+      INSTALL_FULL_DEPS: false
 
     steps:
     - name: Checkout
@@ -32,8 +37,11 @@ jobs:
         cache: "pip"
     - name: Install build tools
       run: |
-        python -m pip install --upgrade "pip<24.1" "setuptools==73.0.1"
-        python -m pip install coverage coveralls
+        pip install --upgrade "pip<24.1" "setuptools>=65.0.2,<=73.0.1"
+        pip install coverage coveralls
+      # pip<24.1 because https://github.com/omry/omegaconf/pull/1195
+      # setuptools>=65.0.2 because https://github.com/pypa/setuptools/commit/d03da04e024ad4289342077eef6de40013630a44#diff-9ea6e1e3dde6d4a7e08c7c88eceed69ca745d0d2c779f8f85219b22266efff7fR1
+      # setuptools<=73.0.1 because https://github.com/pypa/setuptools/issues/4620
     - name: Install ICU (macOS)
       if: startsWith(matrix.os, 'macos-')
       run: |
@@ -43,26 +51,38 @@ jobs:
         ICU_VER=$(pkg-config --modversion icu-i18n)
         echo "ICU_VER=${ICU_VER}"
         echo "ICU_VER=${ICU_VER}" >> "${GITHUB_ENV}"
-    - name: Install ICU (Windows)
-      if: startsWith(matrix.os, 'windows-')
+    - name: Install PyICU (Windows)
+      if: startsWith(matrix.os, 'windows-') && env.INSTALL_PYICU_WIN == 'true'
+      shell: powershell
       run: |
-        python -m pip install "https://github.com/cgohlke/pyicu-build/releases/download/v2.14/PyICU-2.14-cp310-cp310-win_amd64.whl"
-      # if needed, get pip wheel link from https://github.com/cgohlke/pyicu-build/releases
+        $PYTHON_WIN_VER = "${{ matrix.python-version }}"
+        $CP_VER = "cp" + $PYTHON_WIN_VER.Replace(".", "")
+        $WHEEL_URL = "https://github.com/cgohlke/pyicu-build/releases/download/v${{ env.PYICU_WIN_VER }}/PyICU-${{ env.PYICU_WIN_VER }}-${CP_VER}-${CP_VER}-win_amd64.whl"
+        pip install "$WHEEL_URL"
+      # Get wheel URL from https://github.com/cgohlke/pyicu-build/releases
     - name: Install PyTorch
+      if: env.INSTALL_TORCH == 'true'
       run: pip install torch
-      # if needed, get pip wheel link from http://download.pytorch.org/whl/torch/
-    # - name: Install dependencies
-    #   env:
-    #     SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True
-    #   run: |
-    #     python -m pip install -r docker_requirements.txt
+      # If torch for the platform is not available in PyPI, use this command:
+      # pip install "<torch_wheel_url>"
+      # Get wheel URL from http://download.pytorch.org/whl/torch/
+    - name: Install dependencies
+      if: env.INSTALL_FULL_DEPS == 'true'
+      env:
+        SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL: True
+      run: pip install -r docker_requirements.txt
     - name: Install PyThaiNLP
-      run: |
-        python -m pip install .
-    - name: Test
+      run: pip install .
+      # Use the command below, if you want to install a small set of external
+      # packages, which includes numpy, pyicu, python-crfsuite, and requests:
+      # pip install .[compact]
+    - name: Unit test and code coverage
+      run: coverage run -m unittest tests
+      # Use 'unittest tests' instead of 'unittest discover' to avoid loading
+      # tests with external imports.
+      # Test cases loaded is defined in __init__.py in the tests directory.
+    - name: Coverage report
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         COVERALLS_SERVICE_NAME: github
-      run: |
-        coveralls
-#        coverage run -m unittest discover
+      run: coveralls

pythainlp/ancient/aksonhan.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
-from pythainlp.util import Trie
 from pythainlp import thai_consonants, thai_tonemarks
-from pythainlp.tokenize import Tokenizer
 from pythainlp.corpus import thai_orst_words
-
+from pythainlp.tokenize import Tokenizer
+from pythainlp.util import Trie
 
 _dict_aksonhan = {}
 for i in list(thai_consonants):

pythainlp/augment/lm/fasttext.py

@@ -3,8 +3,10 @@
 # SPDX-License-Identifier: Apache-2.0
 import itertools
 from typing import List, Tuple
+
 from gensim.models.fasttext import FastText as FastText_gensim
 from gensim.models.keyedvectors import KeyedVectors
+
 from pythainlp.tokenize import word_tokenize
 
 

pythainlp/augment/lm/phayathaibert.py

@@ -2,21 +2,20 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 
-from typing import List
 import random
 import re
+from typing import List
 
 from pythainlp.phayathaibert.core import ThaiTextProcessor
 
-
 _MODEL_NAME = "clicknext/phayathaibert"
 
 
 class ThaiTextAugmenter:
     def __init__(self) -> None:
         from transformers import (
-            AutoTokenizer,
             AutoModelForMaskedLM,
+            AutoTokenizer,
             pipeline,
         )
 

pythainlp/augment/word2vec/__init__.py

@@ -8,5 +8,5 @@
 __all__ = ["Word2VecAug", "Thai2fitAug", "LTW2VAug"]
 
 from pythainlp.augment.word2vec.core import Word2VecAug
-from pythainlp.augment.word2vec.thai2fit import Thai2fitAug
 from pythainlp.augment.word2vec.ltw2v import LTW2VAug
+from pythainlp.augment.word2vec.thai2fit import Thai2fitAug

pythainlp/augment/word2vec/bpemb_wv.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
+
 from pythainlp.augment.word2vec.core import Word2VecAug
 
 

pythainlp/augment/word2vec/core.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
-from typing import List, Tuple
 import itertools
+from typing import List, Tuple
 
 
 class Word2VecAug:

pythainlp/augment/word2vec/ltw2v.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
+
 from pythainlp.augment.word2vec.core import Word2VecAug
 from pythainlp.corpus import get_corpus_path
 from pythainlp.tokenize import word_tokenize

pythainlp/augment/word2vec/thai2fit.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
+
 from pythainlp.augment.word2vec.core import Word2VecAug
 from pythainlp.corpus import get_corpus_path
 from pythainlp.tokenize import THAI2FIT_TOKENIZER

pythainlp/augment/wordnet.py

@@ -9,15 +9,15 @@
     "postype2wordnet",
 ]
 
-from collections import OrderedDict
 import itertools
+from collections import OrderedDict
 from typing import List
 
 from nltk.corpus import wordnet as wn
+
 from pythainlp.corpus import wordnet
-from pythainlp.tokenize import word_tokenize
 from pythainlp.tag import pos_tag
-
+from pythainlp.tokenize import word_tokenize
 
 orchid = {
     "": "",

pythainlp/classify/param_free.py

@@ -3,9 +3,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import gzip
+import json
 from typing import List, Tuple
+
 import numpy as np
-import json
 
 
 class GzipModel:

pythainlp/cli/__init__.py

@@ -2,11 +2,10 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 """Command line helpers."""
+
 import sys
 from argparse import ArgumentParser
 
-from pythainlp.cli import data, soundex, tag, tokenize, benchmark
-
 # a command should start with a verb when possible
 COMMANDS = sorted(["data", "soundex", "tag", "tokenize", "benchmark"])
 

pythainlp/cli/benchmark.py

@@ -8,6 +8,7 @@
 import os
 
 import yaml
+
 from pythainlp import cli
 from pythainlp.benchmarks import word_tokenization
 

pythainlp/coref/__init__.py

@@ -5,4 +5,5 @@
 PyThaiNLP Coreference Resolution
 """
 __all__ = ["coreference_resolution"]
+
 from pythainlp.coref.core import coreference_resolution

pythainlp/coref/_fastcoref.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 from typing import List
+
 import spacy
 
 

pythainlp/coref/core.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from typing import List
 
-model = None
+_MODEL = None
 
 
 def coreference_resolution(
@@ -40,17 +40,17 @@ def coreference_resolution(
         # 'clusters': [[(0, 10), (50, 52)]]}
         # ]
     """
-    global model
+    global _MODEL
     if isinstance(texts, str):
         texts = [texts]
 
-    if model is None and model_name == "han-coref-v1.0":
+    if _MODEL is None and model_name == "han-coref-v1.0":
         from pythainlp.coref.han_coref import HanCoref
 
-        model = HanCoref(device=device)
+        _MODEL = HanCoref(device=device)
 
-    if model:
-        return model.predict(texts)
+    if _MODEL:
+        return _MODEL.predict(texts)
 
     return [
         {"text": text, "clusters_string": [], "clusters": []} for text in texts

pythainlp/coref/han_coref.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
 import spacy
+
 from pythainlp.coref._fastcoref import FastCoref
 
 

pythainlp/corpus/common.py

@@ -23,8 +23,8 @@
     "thai_wsd_dict",
 ]
 
-from typing import FrozenSet, List, Union
 import warnings
+from typing import FrozenSet, List, Union
 
 from pythainlp.corpus import get_corpus, get_corpus_as_is, get_corpus_path
 

pythainlp/corpus/core.py

@@ -4,14 +4,13 @@
 """
 Corpus related functions.
 """
+import json
 import os
 from typing import Union
-import json
 
+from pythainlp import __version__
 from pythainlp.corpus import corpus_db_path, corpus_db_url, corpus_path
 from pythainlp.tools import get_full_data_path
-from pythainlp import __version__
-
 
 _CHECK_MODE = os.getenv("PYTHAINLP_READ_MODE")
 
@@ -293,9 +292,10 @@ def _download(url: str, dst: str) -> int:
     """
     _CHUNK_SIZE = 64 * 1024  # 64 KiB
 
-    import requests
     from urllib.request import urlopen
 
+    import requests
+
     file_size = int(urlopen(url).info().get("Content-Length", -1))
     r = requests.get(url, stream=True)
     with open(get_full_data_path(dst), "wb") as f:

pythainlp/corpus/icu.py

@@ -8,7 +8,6 @@
 
 from pythainlp.corpus.common import get_corpus
 
-
 _THAI_ICU_FILENAME = "icubrk_th.txt"
 
 

pythainlp/corpus/tnc.py

@@ -15,9 +15,7 @@
 from collections import defaultdict
 from typing import List, Tuple
 
-from pythainlp.corpus import get_corpus
-from pythainlp.corpus import get_corpus_path
-
+from pythainlp.corpus import get_corpus, get_corpus_path
 
 _FILENAME = "tnc_freq.txt"
 _BIGRAM = "tnc_bigram_word_freqs"

pythainlp/el/_multiel.py

@@ -8,14 +8,16 @@ def __init__(self, model_name="bela", device="cuda"):
         self.model_name = model_name
         self.device = device
         self.load_model()
+
     def load_model(self):
         try:
             from multiel import BELA
-        except ImportError:
+        except ImportError as exc:
             raise ImportError(
                 "Can't import multiel package, you can install by pip install multiel."
-            )
+            ) from exc
         self._bela_run = BELA(device=self.device)
+
     def process_batch(self, list_text):
         if isinstance(list_text, str):
             list_text = [list_text]

pythainlp/generate/__init__.py

@@ -5,6 +5,6 @@
 Thai Text Generation
 """
 
-__all__ = ["Unigram", "Bigram", "Trigram"]
+__all__ = ["Bigram", "Trigram", "Unigram"]
 
-from pythainlp.generate.core import Unigram, Bigram, Trigram
+from pythainlp.generate.core import Bigram, Trigram, Unigram

pythainlp/generate/core.py

@@ -9,13 +9,14 @@
 """
 import random
 from typing import List, Union
-from pythainlp.corpus.tnc import unigram_word_freqs as tnc_word_freqs_unigram
-from pythainlp.corpus.tnc import bigram_word_freqs as tnc_word_freqs_bigram
-from pythainlp.corpus.tnc import trigram_word_freqs as tnc_word_freqs_trigram
-from pythainlp.corpus.ttc import unigram_word_freqs as ttc_word_freqs_unigram
+
 from pythainlp.corpus.oscar import (
     unigram_word_freqs as oscar_word_freqs_unigram,
 )
+from pythainlp.corpus.tnc import bigram_word_freqs as tnc_word_freqs_bigram
+from pythainlp.corpus.tnc import trigram_word_freqs as tnc_word_freqs_trigram
+from pythainlp.corpus.tnc import unigram_word_freqs as tnc_word_freqs_unigram
+from pythainlp.corpus.ttc import unigram_word_freqs as ttc_word_freqs_unigram
 
 
 class Unigram: