Add normalize language function and tests (#41)

Fixes nvaccess/nvda#17527

Summary of the issue
In the add-on store, sometimes translated add-ons appear as untranslated. This can be produced when an add-on has a locale folder with a non normalized language, like ES instead of es.
In the transformation to build the views Branch using addon-datastore-transform repo, a function is used to get the available languages. Available languages are stored in a set. Moreover, when a folder is created to store json files for each available language, uppercase and lowercase folders with the same name aren't created. Consequently, if a non standard language is available, but in fact add-ons aren't translated to this language, the english translation can be copied in the folder corresponding to the standard language, even overriding the available language for the standard language.
This can be observed for "es" (spanish).

Development approach
A function has been created to normalize language, and this is used in the function to get manifest localizations. Tests have been added just for this function.

Author: nvdaes

_tests/test_createJson.py

@@ -10,7 +10,8 @@
 import json
 from _validate import (
 	createJson,
-	addonManifest
+	addonManifest,
+	manifestLoader
 )
 
 TOP_DIR = os.path.abspath(os.path.dirname(__file__))
@@ -110,3 +111,19 @@ def test_validVersion(self):
 			"1.2.0.json",
 			msg="Name of the output file should be named based on version number"
 		)
+
+
+class Test_normalizeLanguage(unittest.TestCase):
+	"""Set of unit tests for `manifestLoader.normalizeLanguage`."""
+
+	def test_normalization_no_country_info(self):
+		"""Makes sure that if no country info is provided language is normalized to lower case."""
+		self.assertEqual("en", manifestLoader.normalizeLanguage("en"))
+		self.assertEqual("en", manifestLoader.normalizeLanguage("EN"))
+		self.assertEqual("kmr", manifestLoader.normalizeLanguage("kmr"))
+
+	def test_underscore_used_as_separator_after_normalization(self):
+		"""Ensures that underscore is used to separate country info from language.
+		Also implicitly test the fact that country code is converted to upper case."""
+		self.assertEqual("pt_BR", manifestLoader.normalizeLanguage("pt_BR"))
+		self.assertEqual("pt_BR", manifestLoader.normalizeLanguage("pt-BR"))

_validate/manifestLoader.py

@@ -47,6 +47,22 @@ def getAddonManifestLocalizations(
 		languageCode = pathlib.Path(translationFile).parent.name
 		try:
 			translatedManifest = AddonManifest(translationFile)
-			yield languageCode, translatedManifest
+			yield normalizeLanguage(languageCode), translatedManifest
 		except Exception:
 			print(f"Error in {translationFile}")
+
+
+def normalizeLanguage(lang: str) -> str:
+	"""
+	Normalizes a language-dialect string  into a standard form we can deal with.
+	Converts any dash to underline, and makes sure that language is lowercase and dialect is uppercase.
+	Based on NVDA`s `languageHandler` module.
+	:param lang: A language code.
+	:return: A normalized language code.
+	"""
+	lang = lang.replace("-", "_")
+	ld = lang.split("_")
+	ld[0] = ld[0].lower()
+	if len(ld) >= 2:
+		ld[1] = ld[1].upper()
+	return "_".join(ld)