From 60debf5bbcfc4e434bf262ab2e6b8a2e7eba4f56 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 12 Oct 2021 21:10:46 +0300 Subject: [PATCH 001/116] early async draft --- core/db_adapter/ignite_adapter.py | 85 ++++++++++++++++----- scenarios/behaviors/behavior_description.py | 6 +- scenarios/behaviors/behaviors.py | 44 +++++------ smart_kit/start_points/base_main_loop.py | 30 ++++---- 4 files changed, 106 insertions(+), 59 deletions(-) diff --git a/core/db_adapter/ignite_adapter.py b/core/db_adapter/ignite_adapter.py index 3e87db43..7619caee 100644 --- a/core/db_adapter/ignite_adapter.py +++ b/core/db_adapter/ignite_adapter.py @@ -1,7 +1,10 @@ # coding: utf-8 import random +from concurrent.futures._base import CancelledError import pyignite +from pyignite import AioClient +from pyignite.aio_cache import AioCache from pyignite.exceptions import ReconnectError, SocketError import core.logging.logger_constants as log_const @@ -12,6 +15,9 @@ class IgniteAdapter(DBAdapter): + IS_ASYNC = True + _client: AioClient + _cache = AioCache def __init__(self, config): self._init_params = config.get("init_params", {}) @@ -34,11 +40,11 @@ def _glob(self, path, pattern): def _path_exists(self, path): raise error.NotSupportedOperation - def connect(self): + async def connect(self): try: - self._client = pyignite.Client(**self._init_params) - self._client.connect(self._url) - self._cache = self._client.get_or_create_cache(self._cache_name) + self._client = pyignite.aio_client.AioClient(**self._init_params) + await self._client.connect(self._url) + self._cache = await self._client.get_or_create_cache(self._cache_name) logger_args = { log_const.KEY_NAME: log_const.IGNITE_VALUE, "pyignite_args": str(self._init_params), @@ -47,37 +53,76 @@ def connect(self): log("IgniteAdapter to servers %(pyignite_addresses)s created", params=logger_args, level="WARNING") except Exception: log("IgniteAdapter connect error", - params={log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE}, - level="ERROR", - exc_info=True) + params={log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE}, + level="ERROR", + exc_info=True) monitoring.got_counter("ignite_connection_exception") raise - def _save(self, id, data): - return self.cache.put(id, data) + async def _save(self, id, data): + cache = await self.get_cache() + return await cache.put(id, data) - def _replace_if_equals(self, id, sample, data): - return self._cache.replace_if_equals(id, sample, data) + async def _replace_if_equals(self, id, sample, data): + cache = await self.get_cache() + return await cache.replace_if_equals(id, sample, data) - def _get(self, id): - data = self.cache.get(id) + async def _get(self, id): + cache = await self.get_cache() + data = await cache.get(id) return data - @property - def cache(self): - if self._cache is None: + async def get_cache(self): + if self._client is None: log('Attempt to recreate ignite instance', level="WARNING") - self.connect() + await self.connect() monitoring.got_counter("ignite_reconnection") return self._cache @property def _handled_exception(self): # TypeError is raised during reconnection if all nodes are exhausted - return OSError, SocketError, ReconnectError + return OSError, SocketError, ReconnectError, CancelledError def _on_prepare(self): - self._cache = None + self._client = None def _get_counter_name(self): - return "ignite_adapter" + return "ignite_async_adapter" + + @monitoring.got_histogram("save_time") + async def save(self, id, data): + return await self._async_run(self._save, id, data) + + @monitoring.got_histogram("save_time") + async def replace_if_equals(self, id, sample, data): + return await self._async_run(self._replace_if_equals, id, sample, data) + + @monitoring.got_histogram("get_time") + async def get(self, id): + return await self._async_run(self._get, id) + + async def _async_run(self, action, *args, _try_count=None, **kwargs): + if _try_count is None: + _try_count = self.try_count + if _try_count <= 0: + self._on_all_tries_fail() + _try_count = _try_count - 1 + try: + result = await action(*args, **kwargs) + except self._handled_exception as e: + params = { + "class_name": str(self.__class__), + "exception": str(e), + "try_count": _try_count, + log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE + } + log("%(class_name)s run failed with %(exception)s.\n Got %(try_count)s tries left.", + params=params, + level="ERROR") + self._on_prepare() + result = await self._async_run(action, *args, _try_count=_try_count, **kwargs) + counter_name = self._get_counter_name() + if counter_name: + monitoring.got_counter(f"{counter_name}_exception") + return result diff --git a/scenarios/behaviors/behavior_description.py b/scenarios/behaviors/behavior_description.py index 87c52680..f8276dab 100644 --- a/scenarios/behaviors/behavior_description.py +++ b/scenarios/behaviors/behavior_description.py @@ -18,10 +18,10 @@ def __init__(self, items, id=None): self.version = items.get("version", -1) self.loop_def = items.get("loop_def", True) - def get_expire_time_from_now(self, user): - return time.time() + self.timeout(user) + async def get_expire_time_from_now(self, user): + return time.time() + await self.timeout(user) - def timeout(self, user): + async def timeout(self, user): setting_timeout = user.settings["template_settings"].get("services_timeout", {}).get(self.id) return setting_timeout or self._timeout diff --git a/scenarios/behaviors/behaviors.py b/scenarios/behaviors/behaviors.py index 62283933..a72410fc 100644 --- a/scenarios/behaviors/behaviors.py +++ b/scenarios/behaviors/behaviors.py @@ -52,8 +52,8 @@ def _add_returned_callback(self, callback_id): def get_returned_callbacks(self): return self._returned_callbacks - def add(self, callback_id: str, behavior_id, scenario_id=None, text_preprocessing_result_raw=None, - action_params=None): + async def add(self, callback_id: str, behavior_id, scenario_id=None, text_preprocessing_result_raw=None, + action_params=None): text_preprocessing_result_raw = text_preprocessing_result_raw or {} # behavior will be removed after timeout + EXPIRATION_DELAY expiration_time = ( @@ -94,8 +94,8 @@ def _delete(self, callback_id): def clear_all(self): self._callbacks = {} - def _log_callback(self, callback_id: str, log_name: str, metric, behavior_result: str, - callback_action_params: Dict): + async def _log_callback(self, callback_id: str, log_name: str, metric, behavior_result: str, + callback_action_params: Dict): callback = self._get_callback(callback_id) behavior = self.descriptions[callback.behavior_id] if callback else None callback_action_params = callback_action_params or {} @@ -116,7 +116,7 @@ def _log_callback(self, callback_id: str, log_name: str, metric, behavior_result user=self._user, params=log_params) - def success(self, callback_id: str): + async def success(self, callback_id: str): log(f"behavior.success started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_SUCCESS_VALUE, @@ -127,7 +127,7 @@ def success(self, callback_id: str): self._add_returned_callback(callback_id) behavior = self.descriptions[callback.behavior_id] callback_action_params = callback.action_params - self._log_callback( + await self._log_callback( callback_id, "behavior_success", smart_kit_metrics.counter_behavior_success, @@ -139,7 +139,7 @@ def success(self, callback_id: str): self._delete(callback_id) return result - def fail(self, callback_id: str): + async def fail(self, callback_id: str): log(f"behavior.fail started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_FAIL_VALUE, @@ -150,15 +150,15 @@ def fail(self, callback_id: str): self._add_returned_callback(callback_id) behavior = self.descriptions[callback.behavior_id] callback_action_params = callback.action_params - self._log_callback(callback_id, "behavior_fail", - smart_kit_metrics.counter_behavior_fail, "fail", - callback_action_params) + await self._log_callback(callback_id, "behavior_fail", + smart_kit_metrics.counter_behavior_fail, "fail", + callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) result = behavior.fail_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result - def timeout(self, callback_id: str): + async def timeout(self, callback_id: str): log(f"behavior.timeout started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_TIMEOUT_VALUE, @@ -169,15 +169,15 @@ def timeout(self, callback_id: str): self._add_returned_callback(callback_id) behavior = self.descriptions[callback.behavior_id] callback_action_params = callback.action_params - self._log_callback(callback_id, "behavior_timeout", - smart_kit_metrics.counter_behavior_timeout, "timeout", - callback_action_params) + await self._log_callback(callback_id, "behavior_timeout", + smart_kit_metrics.counter_behavior_timeout, "timeout", + callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) result = behavior.timeout_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result - def misstate(self, callback_id: str): + async def misstate(self, callback_id: str): log(f"behavior.misstate started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_MISSTATE_VALUE, @@ -188,9 +188,9 @@ def misstate(self, callback_id: str): self._add_returned_callback(callback_id) behavior = self.descriptions[callback.behavior_id] callback_action_params = callback.action_params - self._log_callback(callback_id, "behavior_misstate", - smart_kit_metrics.counter_behavior_misstate, "misstate", - callback_action_params) + await self._log_callback(callback_id, "behavior_misstate", + smart_kit_metrics.counter_behavior_misstate, "misstate", + callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) result = behavior.misstate_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) @@ -209,7 +209,7 @@ def get_callback_action_params(self, callback_id): if callback: return callback.action_params - def check_misstate(self, callback_id: str): + async def check_misstate(self, callback_id: str): log(f"behavior.check_misstate started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_CHECK_MISSTATE_VALUE, @@ -230,7 +230,7 @@ def check_misstate(self, callback_id: str): level="WARNING") return last_scenario_equal_callback_scenario - def expire(self): + async def expire(self): callback_id_for_delete = [] for callback_id, ( behavior_id, expiration_time, scenario_id, text_preprocessing_result, @@ -252,7 +252,7 @@ def expire(self): params=log_params, level="WARNING", user=self._user) self._delete(callback_id) - def check_got_saved_id(self, behavior_id): + async def check_got_saved_id(self, behavior_id): if self.descriptions[behavior_id].loop_def: for callback_id, (_behavior_id, expiration_time, scenario_id, text_preprocessing_result, action_params) in self._callbacks.items(): @@ -273,7 +273,7 @@ def check_got_saved_id(self, behavior_id): def raw(self): return {key: callback._asdict() for key, callback in self._callbacks.items()} - def _get_to_message_name(self, callback_id): + async def _get_to_message_name(self, callback_id): callback_action_params = self.get_callback_action_params(callback_id) or {} to_message_name = callback_action_params.get(TO_MESSAGE_NAME) return to_message_name diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index 8d072079..79997877 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -61,14 +61,14 @@ def __init__( smart_kit_metrics.init_metrics(app_name=self.app_name) log("%(class_name)s.__init__ completed.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "class_name": self.__class__.__name__}) + "class_name": self.__class__.__name__}) except: log("%(class_name)s.__init__ exception.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "class_name": self.__class__.__name__}, - level="ERROR", exc_info=True) + "class_name": self.__class__.__name__}, + level="ERROR", exc_info=True) raise - def get_db(self): + async def get_db(self): db_adapter = db_adapter_factory(self.settings["template_settings"].get("db_adapter", {})) if db_adapter.IS_ASYNC: raise Exception( @@ -80,7 +80,7 @@ def get_db(self): def _generate_answers(self, user, commands, message, **kwargs): raise NotImplementedError - def _create_health_check_server(self, settings): + async def _create_health_check_server(self, settings): health_check_server = None if settings["health_check"].get("enabled"): log("Init health_check started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE}) @@ -93,18 +93,19 @@ def _create_health_check_server(self, settings): ) return health_check_server - def _create_jaeger_tracer(self, template_settings): + async def _create_jaeger_tracer(self, template_settings): jaeger_config = template_settings["jaeger_config"] config = ExtendedConfig(config=jaeger_config, service_name=self.app_name, validate=True) tracer = config.initialize_tracer() return tracer - def load_user(self, db_uid, message): + async def load_user(self, db_uid, message): db_data = None load_error = False try: - db_data = self.db_adapter.get(db_uid) + db_adapter = await self.db_adapter + db_data = db_adapter.get(db_uid) except (DBAdapterException, ValueError): log("Failed to get user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") @@ -118,9 +119,9 @@ def load_user(self, db_uid, message): descriptions=self.model.scenario_descriptions, parametrizer_cls=self.parametrizer_cls, load_error=load_error - ) + ) - def save_user(self, db_uid, user, message): + async def save_user(self, db_uid, user, message): no_collisions = True if user.do_not_save: log("User %(uid)s will not saved", user=user, params={"uid": user.id, @@ -134,12 +135,13 @@ def save_user(self, db_uid, user, message): params={"uid": user.id, log_const.KEY_NAME: "user_save", "user_length": len(str_data)}) + db_adapter = await self.db_adapter if user.initial_db_data and self.user_save_check_for_collisions: - no_collisions = self.db_adapter.replace_if_equals(db_uid, - sample=user.initial_db_data, - data=str_data) + no_collisions = db_adapter.replace_if_equals(db_uid, + sample=user.initial_db_data, + data=str_data) else: - self.db_adapter.save(db_uid, str_data) + db_adapter.save(db_uid, str_data) except (DBAdapterException, ValueError): log("Failed to set user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") From 4cd37961a35a3539caf06c6a83f118624e6091c7 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 18 Oct 2021 23:07:40 +0300 Subject: [PATCH 002/116] delete ignite_thread_adapter --- core/db_adapter/ignite_thread_adapter.py | 98 ------------------------ smart_kit/resources/__init__.py | 2 - 2 files changed, 100 deletions(-) delete mode 100644 core/db_adapter/ignite_thread_adapter.py diff --git a/core/db_adapter/ignite_thread_adapter.py b/core/db_adapter/ignite_thread_adapter.py deleted file mode 100644 index addc8b81..00000000 --- a/core/db_adapter/ignite_thread_adapter.py +++ /dev/null @@ -1,98 +0,0 @@ -# coding: utf-8 -import random -import threading - -import pyignite -from pyignite.exceptions import ReconnectError, SocketError - -import core.logging.logger_constants as log_const -from core.db_adapter import error -from core.db_adapter.db_adapter import DBAdapter -from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring - - -class IgniteThreadAdapter(DBAdapter): - - def __init__(self, config): - self._init_params = config.get("init_params", {}) - self._url = config["url"] - if config.get("randomize_url"): - random.shuffle(self._url) - self._cache_name = config["cache_name"] - self._clients = {} - self._caches = {} - super(IgniteThreadAdapter, self).__init__(config) - - def _open(self, filename, *args, **kwargs): - pass - - def _list_dir(self, path): - raise error.NotSupportedOperation - - def _glob(self, path, pattern): - raise error.NotSupportedOperation - - def _path_exists(self, path): - raise error.NotSupportedOperation - - def connect(self): - self._get_cache() - - def _connect_thread(self, thread_id): - try: - client = pyignite.Client(**self._init_params) - client.connect(self._url) - cache = client.get_or_create_cache(self._cache_name) - self._clients[thread_id] = client - self._caches[thread_id] = cache - logger_args = { - log_const.KEY_NAME: log_const.IGNITE_VALUE, - "pyignite_args": str(self._init_params), - "pyignite_addresses": str(self._url) - } - log("IgniteAdapter to servers %(pyignite_addresses)s created", params=logger_args, level="WARNING") - except Exception: - log( - "IgniteAdapter connect error", - params={log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE}, - level="ERROR", - exc_info=True - ) - monitoring.got_counter("ignite_connection_exception") - raise - - def _get_cache(self): - thread_id = threading.get_ident() - if thread_id not in self._caches: - self._connect_thread(thread_id) - return self._caches[thread_id] - - def _save(self, id, data): - return self._get_cache().put(id, data) - - def _replace_if_equals(self, id, sample, data): - return self._get_cache().replace_if_equals(id, sample, data) - - def _get(self, id): - data = self._get_cache().get(id) - return data - - @property - def cache(self): - if self._get_cache() is None: - log('Attempt to recreate ignite instance', level="WARNING") - self.connect() - monitoring.got_counter("ignite_reconnection") - return self._get_cache() - - @property - def _handled_exception(self): - # TypeError is raised during reconnection if all nodes are exhausted - return OSError, SocketError, ReconnectError - - def _on_prepare(self): - self._cache = None - - def _get_counter_name(self): - return "ignite_adapter" diff --git a/smart_kit/resources/__init__.py b/smart_kit/resources/__init__.py index 3e918a19..0f19622a 100644 --- a/smart_kit/resources/__init__.py +++ b/smart_kit/resources/__init__.py @@ -37,7 +37,6 @@ from core.db_adapter.aioredis_adapter import AIORedisAdapter from core.db_adapter.db_adapter import db_adapters from core.db_adapter.ignite_adapter import IgniteAdapter -from core.db_adapter.ignite_thread_adapter import IgniteThreadAdapter from core.db_adapter.memory_adapter import MemoryAdapter from core.db_adapter.redis_adapter import RedisAdapter from core.descriptions.descriptions import registered_description_factories @@ -380,7 +379,6 @@ def init_requests(self): def init_db_adapters(self): db_adapters[None] = MemoryAdapter db_adapters["ignite"] = IgniteAdapter - db_adapters["ignite_thread"] = IgniteThreadAdapter db_adapters["memory"] = MemoryAdapter db_adapters["redis"] = RedisAdapter db_adapters["aioredis"] = AIORedisAdapter From 7f2367f38da38e22278a6f4a11cfe23b004bd07c Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 19 Oct 2021 03:15:13 +0300 Subject: [PATCH 003/116] async db_adapter in BaseMainLoop --- scenarios/behaviors/behavior_description.py | 4 +- scenarios/behaviors/behaviors.py | 44 ++++++++++----------- smart_kit/start_points/base_main_loop.py | 33 +++++++++------- 3 files changed, 42 insertions(+), 39 deletions(-) diff --git a/scenarios/behaviors/behavior_description.py b/scenarios/behaviors/behavior_description.py index f8276dab..aaf6de51 100644 --- a/scenarios/behaviors/behavior_description.py +++ b/scenarios/behaviors/behavior_description.py @@ -18,10 +18,10 @@ def __init__(self, items, id=None): self.version = items.get("version", -1) self.loop_def = items.get("loop_def", True) - async def get_expire_time_from_now(self, user): + def get_expire_time_from_now(self, user): return time.time() + await self.timeout(user) - async def timeout(self, user): + def timeout(self, user): setting_timeout = user.settings["template_settings"].get("services_timeout", {}).get(self.id) return setting_timeout or self._timeout diff --git a/scenarios/behaviors/behaviors.py b/scenarios/behaviors/behaviors.py index a72410fc..62283933 100644 --- a/scenarios/behaviors/behaviors.py +++ b/scenarios/behaviors/behaviors.py @@ -52,8 +52,8 @@ def _add_returned_callback(self, callback_id): def get_returned_callbacks(self): return self._returned_callbacks - async def add(self, callback_id: str, behavior_id, scenario_id=None, text_preprocessing_result_raw=None, - action_params=None): + def add(self, callback_id: str, behavior_id, scenario_id=None, text_preprocessing_result_raw=None, + action_params=None): text_preprocessing_result_raw = text_preprocessing_result_raw or {} # behavior will be removed after timeout + EXPIRATION_DELAY expiration_time = ( @@ -94,8 +94,8 @@ def _delete(self, callback_id): def clear_all(self): self._callbacks = {} - async def _log_callback(self, callback_id: str, log_name: str, metric, behavior_result: str, - callback_action_params: Dict): + def _log_callback(self, callback_id: str, log_name: str, metric, behavior_result: str, + callback_action_params: Dict): callback = self._get_callback(callback_id) behavior = self.descriptions[callback.behavior_id] if callback else None callback_action_params = callback_action_params or {} @@ -116,7 +116,7 @@ async def _log_callback(self, callback_id: str, log_name: str, metric, behavior_ user=self._user, params=log_params) - async def success(self, callback_id: str): + def success(self, callback_id: str): log(f"behavior.success started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_SUCCESS_VALUE, @@ -127,7 +127,7 @@ async def success(self, callback_id: str): self._add_returned_callback(callback_id) behavior = self.descriptions[callback.behavior_id] callback_action_params = callback.action_params - await self._log_callback( + self._log_callback( callback_id, "behavior_success", smart_kit_metrics.counter_behavior_success, @@ -139,7 +139,7 @@ async def success(self, callback_id: str): self._delete(callback_id) return result - async def fail(self, callback_id: str): + def fail(self, callback_id: str): log(f"behavior.fail started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_FAIL_VALUE, @@ -150,15 +150,15 @@ async def fail(self, callback_id: str): self._add_returned_callback(callback_id) behavior = self.descriptions[callback.behavior_id] callback_action_params = callback.action_params - await self._log_callback(callback_id, "behavior_fail", - smart_kit_metrics.counter_behavior_fail, "fail", - callback_action_params) + self._log_callback(callback_id, "behavior_fail", + smart_kit_metrics.counter_behavior_fail, "fail", + callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) result = behavior.fail_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result - async def timeout(self, callback_id: str): + def timeout(self, callback_id: str): log(f"behavior.timeout started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_TIMEOUT_VALUE, @@ -169,15 +169,15 @@ async def timeout(self, callback_id: str): self._add_returned_callback(callback_id) behavior = self.descriptions[callback.behavior_id] callback_action_params = callback.action_params - await self._log_callback(callback_id, "behavior_timeout", - smart_kit_metrics.counter_behavior_timeout, "timeout", - callback_action_params) + self._log_callback(callback_id, "behavior_timeout", + smart_kit_metrics.counter_behavior_timeout, "timeout", + callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) result = behavior.timeout_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result - async def misstate(self, callback_id: str): + def misstate(self, callback_id: str): log(f"behavior.misstate started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_MISSTATE_VALUE, @@ -188,9 +188,9 @@ async def misstate(self, callback_id: str): self._add_returned_callback(callback_id) behavior = self.descriptions[callback.behavior_id] callback_action_params = callback.action_params - await self._log_callback(callback_id, "behavior_misstate", - smart_kit_metrics.counter_behavior_misstate, "misstate", - callback_action_params) + self._log_callback(callback_id, "behavior_misstate", + smart_kit_metrics.counter_behavior_misstate, "misstate", + callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) result = behavior.misstate_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) @@ -209,7 +209,7 @@ def get_callback_action_params(self, callback_id): if callback: return callback.action_params - async def check_misstate(self, callback_id: str): + def check_misstate(self, callback_id: str): log(f"behavior.check_misstate started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_CHECK_MISSTATE_VALUE, @@ -230,7 +230,7 @@ async def check_misstate(self, callback_id: str): level="WARNING") return last_scenario_equal_callback_scenario - async def expire(self): + def expire(self): callback_id_for_delete = [] for callback_id, ( behavior_id, expiration_time, scenario_id, text_preprocessing_result, @@ -252,7 +252,7 @@ async def expire(self): params=log_params, level="WARNING", user=self._user) self._delete(callback_id) - async def check_got_saved_id(self, behavior_id): + def check_got_saved_id(self, behavior_id): if self.descriptions[behavior_id].loop_def: for callback_id, (_behavior_id, expiration_time, scenario_id, text_preprocessing_result, action_params) in self._callbacks.items(): @@ -273,7 +273,7 @@ async def check_got_saved_id(self, behavior_id): def raw(self): return {key: callback._asdict() for key, callback in self._callbacks.items()} - async def _get_to_message_name(self, callback_id): + def _get_to_message_name(self, callback_id): callback_action_params = self.get_callback_action_params(callback_id) or {} to_message_name = callback_action_params.get(TO_MESSAGE_NAME) return to_message_name diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index 79997877..86ec9636 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -41,7 +41,7 @@ def __init__( self.model: SmartAppModel = model self.user_cls = user_cls self.parametrizer_cls = parametrizer_cls - self.db_adapter = self.get_db() + self.db_adapter = await self.get_db() self.is_work = True self.to_msg_validators: Iterable[MessageValidator] = to_msg_validators self.from_msg_validators: Iterable[MessageValidator] = from_msg_validators @@ -71,16 +71,15 @@ def __init__( async def get_db(self): db_adapter = db_adapter_factory(self.settings["template_settings"].get("db_adapter", {})) if db_adapter.IS_ASYNC: - raise Exception( - f"Async adapter {db_adapter.__class__.__name__} doesnt compare with {self.__class__.__name__}" - ) - db_adapter.connect() + await db_adapter.connect() + else: + db_adapter.connect() return db_adapter def _generate_answers(self, user, commands, message, **kwargs): raise NotImplementedError - async def _create_health_check_server(self, settings): + def _create_health_check_server(self, settings): health_check_server = None if settings["health_check"].get("enabled"): log("Init health_check started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE}) @@ -93,7 +92,7 @@ async def _create_health_check_server(self, settings): ) return health_check_server - async def _create_jaeger_tracer(self, template_settings): + def _create_jaeger_tracer(self, template_settings): jaeger_config = template_settings["jaeger_config"] config = ExtendedConfig(config=jaeger_config, service_name=self.app_name, validate=True) @@ -104,8 +103,7 @@ async def load_user(self, db_uid, message): db_data = None load_error = False try: - db_adapter = await self.db_adapter - db_data = db_adapter.get(db_uid) + db_data = await self.db_adapter.get(db_uid) if self.db_adapter.IS_ASYNC else self.db_adapter.get(db_uid) except (DBAdapterException, ValueError): log("Failed to get user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") @@ -119,7 +117,7 @@ async def load_user(self, db_uid, message): descriptions=self.model.scenario_descriptions, parametrizer_cls=self.parametrizer_cls, load_error=load_error - ) + ) async def save_user(self, db_uid, user, message): no_collisions = True @@ -135,13 +133,18 @@ async def save_user(self, db_uid, user, message): params={"uid": user.id, log_const.KEY_NAME: "user_save", "user_length": len(str_data)}) - db_adapter = await self.db_adapter if user.initial_db_data and self.user_save_check_for_collisions: - no_collisions = db_adapter.replace_if_equals(db_uid, - sample=user.initial_db_data, - data=str_data) + if self.db_adapter.IS_ASYNC: + no_collisions = await self.db_adapter.replace_if_equals(db_uid, + sample=user.initial_db_data, + data=str_data) + else: + no_collisions = self.db_adapter.replace_if_equals(db_uid, + sample=user.initial_db_data, + data=str_data) else: - db_adapter.save(db_uid, str_data) + await self.db_adapter.save(db_uid, str_data) if self.db_adapter.IS_ASYNC else \ + self.db_adapter.save(db_uid, str_data) except (DBAdapterException, ValueError): log("Failed to set user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") From af2cf9ab3f903b79fa38987a18314179332e7f29 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Tue, 19 Oct 2021 10:21:00 +0300 Subject: [PATCH 004/116] main_coro in main_loop_kafka --- smart_kit/start_points/main_loop_kafka.py | 49 +++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 24b2c397..394ff462 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -1,4 +1,5 @@ # coding=utf-8 +import asyncio import json import time from collections import namedtuple @@ -40,6 +41,7 @@ class MainLoop(BaseMainLoop): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.loop = asyncio.get_event_loop() log("%(class_name)s.__init__ started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) try: @@ -107,6 +109,53 @@ def run(self): log("Kafka publisher connection is closed", level="WARNING") log("Kafka handler is stopped", level="WARNING") + async def main_coro(self): + tasks = [self.iterate_coro(kafka_key) for kafka_key in self.consumers] + tasks.append(self.healthcheck_coro()) + await asyncio.gather(*tasks) + + async def healthcheck_coro(self): + if self.health_check_server: + with StatsTimer() as health_check_server_timer: + self.health_check_server.iterate() + + if health_check_server_timer.msecs >= self.MAX_LOG_TIME: + log("Health check iterate time: {} msecs\n".format(health_check_server_timer.msecs), + params={log_const.KEY_NAME: "slow_health_check", + "time_msecs": health_check_server_timer.msecs}, level="WARNING") + + async def iterate_coro(self, kafka_key): + consumer = self.consumers[kafka_key] + mq_message = None + message_value = None + try: + mq_message = None + message_value = None + with StatsTimer() as poll_timer: + mq_message = consumer.poll() + + if mq_message: + stats = "Polling time: {} msecs\n".format(poll_timer.msecs) + message_value = mq_message.value() # DRY! + self.process_message(mq_message, consumer, kafka_key, stats) + + except KafkaException as kafka_exp: + log("kafka error: %(kafka_exp)s. MESSAGE: {}.".format(message_value), + params={log_const.KEY_NAME: log_const.STARTUP_VALUE, + "kafka_exp": str(kafka_exp), + log_const.REQUEST_VALUE: str(message_value)}, + level="ERROR", exc_info=True) + except Exception: + try: + log("%(class_name)s iterate error. Kafka key %(kafka_key)s MESSAGE: {}.".format(message_value), + params={log_const.KEY_NAME: log_const.STARTUP_VALUE, + "kafka_key": kafka_key}, + level="ERROR", exc_info=True) + consumer.commit_offset(mq_message) + except Exception: + log("Error handling worker fail exception.", + level="ERROR", exc_info=True) + def _generate_answers(self, user, commands, message, **kwargs): topic_key = kwargs["topic_key"] kafka_key = kwargs["kafka_key"] From 2b1ea81007623d78bd905984569520d611a222f7 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 20 Oct 2021 02:20:01 +0300 Subject: [PATCH 005/116] async Behaviors and main_loop_kafka --- scenarios/behaviors/behavior_description.py | 3 - scenarios/behaviors/behaviors.py | 10 +- smart_kit/start_points/main_loop_kafka.py | 154 ++++++++++++++++++-- 3 files changed, 143 insertions(+), 24 deletions(-) diff --git a/scenarios/behaviors/behavior_description.py b/scenarios/behaviors/behavior_description.py index aaf6de51..d4459cab 100644 --- a/scenarios/behaviors/behavior_description.py +++ b/scenarios/behaviors/behavior_description.py @@ -18,9 +18,6 @@ def __init__(self, items, id=None): self.version = items.get("version", -1) self.loop_def = items.get("loop_def", True) - def get_expire_time_from_now(self, user): - return time.time() + await self.timeout(user) - def timeout(self, user): setting_timeout = user.settings["template_settings"].get("services_timeout", {}).get(self.id) return setting_timeout or self._timeout diff --git a/scenarios/behaviors/behaviors.py b/scenarios/behaviors/behaviors.py index 62283933..27de1d06 100644 --- a/scenarios/behaviors/behaviors.py +++ b/scenarios/behaviors/behaviors.py @@ -40,8 +40,8 @@ def initialize(self): for key, value in callback_action_params.get(LOCAL_VARS, {}).items(): self._user.local_vars.set(key, value) - def _add_behavior_timeout(self, expire_time_us, callback_id): - self._behavior_timeouts.append((expire_time_us, callback_id)) + def _add_behavior_timeout(self, time_left, callback_id): + self._behavior_timeouts.append((time_left, callback_id)) def get_behavior_timeouts(self): return self._behavior_timeouts @@ -72,8 +72,7 @@ def add(self, callback_id: str, behavior_id, scenario_id=None, text_preprocessin action_params=action_params, ) self._callbacks[callback_id] = callback - log( - f"behavior.add: adding behavior %({log_const.BEHAVIOR_ID_VALUE})s with scenario_id" + log(f"behavior.add: adding behavior %({log_const.BEHAVIOR_ID_VALUE})s with scenario_id" f" %({log_const.CHOSEN_SCENARIO_VALUE})s for callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s" f" expiration_time: %(expiration_time)s.", user=self._user, @@ -84,8 +83,7 @@ def add(self, callback_id: str, behavior_id, scenario_id=None, text_preprocessin "expiration_time": expiration_time}) behavior_description = self.descriptions[behavior_id] - expire_time_us = behavior_description.get_expire_time_from_now(self._user) - self._add_behavior_timeout(expire_time_us, callback_id) + self._add_behavior_timeout(behavior_description.timeout(self._user) + self.EXPIRATION_DELAY, callback_id) def _delete(self, callback_id): if callback_id in self._callbacks: diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 394ff462..73ccc6cc 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -1,7 +1,7 @@ # coding=utf-8 import asyncio import json -import time +from time import time from collections import namedtuple from functools import lru_cache @@ -9,6 +9,7 @@ from lazy import lazy import scenarios.logging.logger_constants as log_const +from core.db_adapter.db_adapter import DBAdapterException from core.jaeger_custom_client import jaeger_utils from core.jaeger_custom_client import kafka_codec as jaeger_kafka_codec from core.logging.logger_utils import log, UID_STR, MESSAGE_ID_STR @@ -81,6 +82,67 @@ def __init__(self, *args, **kwargs): level="ERROR", exc_info=True) raise + async def save_user_async(self, db_uid, user, message): + no_collisions = True + if user.do_not_save: + log("User %(uid)s will not saved", user=user, params={"uid": user.id, + log_const.KEY_NAME: "user_will_not_saved"}) + else: + + no_collisions = True + try: + str_data = user.raw_str + log("Saving User %(uid)s. Serialized utf8 json length is %(user_length)s symbols.", user=user, + params={"uid": user.id, + log_const.KEY_NAME: "user_save", + "user_length": len(str_data)}) + if user.initial_db_data and self.user_save_check_for_collisions: + if self.db_adapter.IS_ASYNC: + no_collisions = await self.db_adapter.replace_if_equals(db_uid, + sample=user.initial_db_data, + data=str_data) + else: + no_collisions = self.db_adapter.replace_if_equals(db_uid, + sample=user.initial_db_data, + data=str_data) + else: + if self.db_adapter.IS_ASYNC: + await self.db_adapter.save(db_uid, str_data) + else: + self.db_adapter.save(db_uid, str_data) + except (DBAdapterException, ValueError): + log("Failed to set user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, + log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") + smart_kit_metrics.counter_save_error(self.app_name) + if not no_collisions: + smart_kit_metrics.counter_save_collision(self.app_name) + return no_collisions + + async def load_user_async(self, db_uid, message): + db_data = None + load_error = False + try: + if self.db_adapter.IS_ASYNC: + db_data = await self.db_adapter.get(db_uid) + else: + db_data = self.db_adapter.get(db_uid) + except (DBAdapterException, ValueError): + log("Failed to get user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, + log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") + load_error = True + smart_kit_metrics.counter_load_error(self.app_name) + # to skip message when load failed + raise + return self.user_cls( + message.uid, + message=message, + db_data=db_data, # плохо ломать базовые интерфейсы + settings=self.settings, + descriptions=self.model.scenario_descriptions, + parametrizer_cls=self.parametrizer_cls, + load_error=load_error + ) + def pre_handle(self): self.iterate_behavior_timeouts() @@ -479,24 +541,86 @@ def masking_fields(self): return self.settings["template_settings"].get("masking_fields") def save_behavior_timeouts(self, user, mq_message, kafka_key): - for i, (expire_time_us, callback_id) in enumerate(user.behaviors.get_behavior_timeouts()): - # two behaviors can be created in one query, so we need add some salt to make theirs key unique - unique_key = expire_time_us + i * 1e-5 - log( - "%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout on %(unique_key)s", + for i, (timeout, callback_id) in enumerate(user.behaviors.get_behavior_timeouts()): + # если колбеков много, разносим их на 1 секунду друг от друга во избежание коллизий + timeout = timeout + i + log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout in %(when)s seconds.", params={log_const.KEY_NAME: "adding_local_timeout", "class_name": self.__class__.__name__, "callback_id": callback_id, - "unique_key": unique_key}) - self.behaviors_timeouts.push(unique_key, self.behaviors_timeouts_value_cls._make( - (user.message.db_uid, callback_id, mq_message, kafka_key))) + "timeout": timeout}) - for callback_id in user.behaviors.get_returned_callbacks(): - log("%(class_name)s: removing local_timeout on callback %(callback_id)s", - params={log_const.KEY_NAME: "removing_local_timeout", - "class_name": self.__class__.__name__, - "callback_id": callback_id}) - self.behaviors_timeouts.remove(callback_id) + self.loop.call_later( + timeout, + self.loop.create_task, + self.do_behavior_timeout(user.message.db_uid, callback_id, mq_message, kafka_key) + ) def stop(self, signum, frame): self.is_work = False + + async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): + if not self.is_work: + return + try: + save_tries = 0 + user_save_ok = False + answers = [] + user = None + while save_tries < self.user_save_collisions_tries and not user_save_ok: + callback_found = False + log(f"MainLoop.do_behavior_timeout: handling callback {callback_id}. for db_uid {db_uid}. try {save_tries}.") + + save_tries += 1 + + orig_message_raw = json.loads(mq_message.value()) + orig_message_raw[SmartAppFromMessage.MESSAGE_NAME] = message_names.LOCAL_TIMEOUT + + timeout_from_message = self._get_timeout_from_message(orig_message_raw, callback_id, + headers=mq_message.headers()) + + user = await self.load_user_async(db_uid, timeout_from_message) + # TODO: not to load user to check behaviors.has_callback ? + if user.behaviors.has_callback(callback_id): + callback_found = True + commands = self.model.answer(timeout_from_message, user) + topic_key = self._get_topic_key(mq_message, kafka_key) + answers = self._generate_answers(user=user, commands=commands, message=timeout_from_message, + topic_key=topic_key, + kafka_key=kafka_key) + + user_save_ok = await self.save_user_async(db_uid, user, mq_message) + + if not user_save_ok: + log("MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(db_version)s.", + user=user, + params={log_const.KEY_NAME: "ignite_collision", + "db_uid": db_uid, + "message_key": mq_message.key(), + "kafka_key": kafka_key, + "uid": user.id, + "db_version": str(user.variables.get(user.USER_DB_VERSION))}, + level="WARNING") + + if not user_save_ok and callback_found: + log("MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", + user=user, + params={log_const.KEY_NAME: "ignite_collision", + "db_uid": db_uid, + "message_key": mq_message.key(), + "message_partition": mq_message.partition(), + "kafka_key": kafka_key, + "uid": user.id, + "db_version": str(user.variables.get(user.USER_DB_VERSION))}, + level="WARNING") + + smart_kit_metrics.counter_save_collision_tries_left(self.app_name) + if user_save_ok: + self.save_behavior_timeouts(user, mq_message, kafka_key) + for answer in answers: + self._send_request(user, answer, mq_message) + except: + log("%(class_name)s error.", params={log_const.KEY_NAME: "error_handling_timeout", + "class_name": self.__class__.__name__, + log_const.REQUEST_VALUE: str(mq_message.value())}, + level="ERROR", exc_info=True) From 5ff2dd4087618ea972637d89b1124065f581b8c6 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Wed, 20 Oct 2021 09:14:16 +0300 Subject: [PATCH 006/116] async run main_loop_kafka, minor async in iterate_coro --- smart_kit/start_points/main_loop_kafka.py | 70 +++++++++++++++-------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 73ccc6cc..c50bfbf1 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -146,22 +146,45 @@ async def load_user_async(self, db_uid, message): def pre_handle(self): self.iterate_behavior_timeouts() + # def run(self): + # log("%(class_name)s.run started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, + # "class_name": self.__class__.__name__}) + # while self.is_work: + # self.pre_handle() + # for kafka_key in self.consumers: + # self.iterate(kafka_key) + # + # if self.health_check_server: + # with StatsTimer() as health_check_server_timer: + # self.health_check_server.iterate() + # + # if health_check_server_timer.msecs >= self.MAX_LOG_TIME: + # log("Health check iterate time: {} msecs\n".format(health_check_server_timer.msecs), + # params={log_const.KEY_NAME: "slow_health_check", + # "time_msecs": health_check_server_timer.msecs}, level="WARNING") + # + # log("Stopping Kafka handler", level="WARNING") + # for kafka_key in self.consumers: + # self.consumers[kafka_key].close() + # log("Kafka consumer connection is closed", level="WARNING") + # self.publishers[kafka_key].close() + # log("Kafka publisher connection is closed", level="WARNING") + # log("Kafka handler is stopped", level="WARNING") + def run(self): - log("%(class_name)s.run started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "class_name": self.__class__.__name__}) - while self.is_work: - self.pre_handle() - for kafka_key in self.consumers: - self.iterate(kafka_key) - - if self.health_check_server: - with StatsTimer() as health_check_server_timer: - self.health_check_server.iterate() - - if health_check_server_timer.msecs >= self.MAX_LOG_TIME: - log("Health check iterate time: {} msecs\n".format(health_check_server_timer.msecs), - params={log_const.KEY_NAME: "slow_health_check", - "time_msecs": health_check_server_timer.msecs}, level="WARNING") + try: + loop = asyncio.get_event_loop() + loop.run_until_complete(self.main_coro()) + except (SystemExit,): + log.info("MainLoop stopped") + + async def main_coro(self): + log("%(class_name)s.main_coro started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, + "class_name": self.__class__.__name__}) + + tasks = [self.iterate_coro(kafka_key) for kafka_key in self.consumers] + tasks.append(self.healthcheck_coro()) + await asyncio.gather(*tasks) log("Stopping Kafka handler", level="WARNING") for kafka_key in self.consumers: @@ -171,11 +194,6 @@ def run(self): log("Kafka publisher connection is closed", level="WARNING") log("Kafka handler is stopped", level="WARNING") - async def main_coro(self): - tasks = [self.iterate_coro(kafka_key) for kafka_key in self.consumers] - tasks.append(self.healthcheck_coro()) - await asyncio.gather(*tasks) - async def healthcheck_coro(self): if self.health_check_server: with StatsTimer() as health_check_server_timer: @@ -190,11 +208,12 @@ async def iterate_coro(self, kafka_key): consumer = self.consumers[kafka_key] mq_message = None message_value = None + loop = asyncio.get_event_loop() try: mq_message = None message_value = None with StatsTimer() as poll_timer: - mq_message = consumer.poll() + mq_message = await loop.run_in_executor(None, consumer.poll) if mq_message: stats = "Polling time: {} msecs\n".format(poll_timer.msecs) @@ -569,7 +588,8 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): user = None while save_tries < self.user_save_collisions_tries and not user_save_ok: callback_found = False - log(f"MainLoop.do_behavior_timeout: handling callback {callback_id}. for db_uid {db_uid}. try {save_tries}.") + log( + f"MainLoop.do_behavior_timeout: handling callback {callback_id}. for db_uid {db_uid}. try {save_tries}.") save_tries += 1 @@ -592,7 +612,8 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): user_save_ok = await self.save_user_async(db_uid, user, mq_message) if not user_save_ok: - log("MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(db_version)s.", + log( + "MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, @@ -603,7 +624,8 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): level="WARNING") if not user_save_ok and callback_found: - log("MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", + log( + "MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, From 1dab8d438df4feb2d42aaadbc415e90728a943bf Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Wed, 20 Oct 2021 22:09:57 +0300 Subject: [PATCH 007/116] asynced: MainLoop.{__init__, main_coro, main_work} --- smart_kit/start_points/main_loop_kafka.py | 84 +++++++++++++++-------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index c50bfbf1..7b26126e 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -41,10 +41,10 @@ class MainLoop(BaseMainLoop): BAD_ANSWER_COMMAND = Command(message_names.ERROR, {"code": -1, "description": "Invalid Answer Message"}) def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.loop = asyncio.get_event_loop() log("%(class_name)s.__init__ started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) + self.loop = asyncio.get_event_loop() + super().__init__(*args, **kwargs) try: kafka_config = _enrich_config_from_secret( self.settings["kafka"]["template-engine"], self.settings.get("secret_kafka", {}) @@ -71,9 +71,16 @@ def __init__(self, *args, **kwargs): for key in self.consumers: self.consumers[key].subscribe() self.publishers = publishers + + # is needed? start # self.behaviors_timeouts_value_cls = namedtuple('behaviors_timeouts_value', 'db_uid, callback_id, mq_message, kafka_key') self.behaviors_timeouts = HeapqKV(value_to_key_func=lambda val: val.callback_id) + # is needed? end # + + self.iterate_tries = 0 + self.concurrent_messages = 0 + log("%(class_name)s.__init__ completed.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) except: @@ -172,20 +179,24 @@ def pre_handle(self): # log("Kafka handler is stopped", level="WARNING") def run(self): + log("%(class_name)s.run started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, + "class_name": self.__class__.__name__}) try: loop = asyncio.get_event_loop() loop.run_until_complete(self.main_coro()) except (SystemExit,): - log.info("MainLoop stopped") + log("%(class_name)s.run stopped", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, + "class_name": self.__class__.__name__}) async def main_coro(self): log("%(class_name)s.main_coro started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) - tasks = [self.iterate_coro(kafka_key) for kafka_key in self.consumers] + tasks = [self.main_work(kafka_key) for kafka_key in self.consumers] tasks.append(self.healthcheck_coro()) await asyncio.gather(*tasks) + # is needed? start # log("Stopping Kafka handler", level="WARNING") for kafka_key in self.consumers: self.consumers[kafka_key].close() @@ -193,6 +204,10 @@ async def main_coro(self): self.publishers[kafka_key].close() log("Kafka publisher connection is closed", level="WARNING") log("Kafka handler is stopped", level="WARNING") + # is needed? end # + + log("%(class_name)s.main_coro completed", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, + "class_name": self.__class__.__name__}) async def healthcheck_coro(self): if self.health_check_server: @@ -204,38 +219,51 @@ async def healthcheck_coro(self): params={log_const.KEY_NAME: "slow_health_check", "time_msecs": health_check_server_timer.msecs}, level="WARNING") - async def iterate_coro(self, kafka_key): + async def main_work(self, kafka_key): + # is needed? start # + # from DP # + self.iterate_tries = self.iterate_tries % 1000 + 1 + # is needed? end # + consumer = self.consumers[kafka_key] - mq_message = None message_value = None - loop = asyncio.get_event_loop() - try: - mq_message = None - message_value = None - with StatsTimer() as poll_timer: - mq_message = await loop.run_in_executor(None, consumer.poll) + max_concurent_messages = self.settings["template_settings"].get("max_concurent_messages", 20) - if mq_message: - stats = "Polling time: {} msecs\n".format(poll_timer.msecs) - message_value = mq_message.value() # DRY! - self.process_message(mq_message, consumer, kafka_key, stats) + loop = asyncio.get_event_loop() - except KafkaException as kafka_exp: - log("kafka error: %(kafka_exp)s. MESSAGE: {}.".format(message_value), - params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "kafka_exp": str(kafka_exp), - log_const.REQUEST_VALUE: str(message_value)}, - level="ERROR", exc_info=True) - except Exception: + while self.is_work: + if self.concurrent_messages >= max_concurent_messages: + log(f"%(class_name)s.main_work: max {max_concurent_messages} concurent messages occured", + params={"class_name": self.__class__.__name__, + log_const.KEY_NAME: "max_concurent_messages"}, level='WARNING') + await asyncio.sleep(0.2) + continue try: - log("%(class_name)s iterate error. Kafka key %(kafka_key)s MESSAGE: {}.".format(message_value), + mq_message = None + with StatsTimer() as poll_timer: + # Max delay between polls configured in consumer.poll_timeout param + mq_message = await loop.run_in_executor(None, consumer.poll) + if mq_message: + stats = "Polling time: {} msecs\n".format(poll_timer.msecs) + message_value = mq_message.value() # DRY! + self.process_message(mq_message, consumer, kafka_key, stats) + + except KafkaException as kafka_exp: + log("kafka error: %(kafka_exp)s. MESSAGE: {}.".format(message_value), params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "kafka_key": kafka_key}, + "kafka_exp": str(kafka_exp), + log_const.REQUEST_VALUE: str(message_value)}, level="ERROR", exc_info=True) - consumer.commit_offset(mq_message) except Exception: - log("Error handling worker fail exception.", - level="ERROR", exc_info=True) + try: + log("%(class_name)s iterate error. Kafka key %(kafka_key)s MESSAGE: {}.".format(message_value), + params={log_const.KEY_NAME: log_const.STARTUP_VALUE, + "kafka_key": kafka_key}, + level="ERROR", exc_info=True) + consumer.commit_offset(mq_message) + except Exception: + log("Error handling worker fail exception.", + level="ERROR", exc_info=True) def _generate_answers(self, user, commands, message, **kwargs): topic_key = kwargs["topic_key"] From edcc4e6ae15dee4da78e0fd562e33b7c3c00d397 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 20 Oct 2021 23:39:12 +0300 Subject: [PATCH 008/116] fix imports --- smart_kit/start_points/main_loop_kafka.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 73ccc6cc..1808c8bf 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -1,7 +1,7 @@ # coding=utf-8 import asyncio import json -from time import time +import time from collections import namedtuple from functools import lru_cache @@ -544,7 +544,7 @@ def save_behavior_timeouts(self, user, mq_message, kafka_key): for i, (timeout, callback_id) in enumerate(user.behaviors.get_behavior_timeouts()): # если колбеков много, разносим их на 1 секунду друг от друга во избежание коллизий timeout = timeout + i - log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout in %(when)s seconds.", + log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout in %(timeout)s seconds.", params={log_const.KEY_NAME: "adding_local_timeout", "class_name": self.__class__.__name__, "callback_id": callback_id, From 36eea5e1948d59d5de5e8144e9d18d99fd7366e0 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Thu, 21 Oct 2021 09:56:54 +0300 Subject: [PATCH 009/116] async MainLoop.healthcheck_coro --- smart_kit/start_points/main_loop_kafka.py | 50 ++++++++--------------- 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 293b7a86..d6eca4b3 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -44,6 +44,7 @@ def __init__(self, *args, **kwargs): log("%(class_name)s.__init__ started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) self.loop = asyncio.get_event_loop() + self.health_check_server_future = None super().__init__(*args, **kwargs) try: kafka_config = _enrich_config_from_secret( @@ -153,31 +154,6 @@ async def load_user_async(self, db_uid, message): def pre_handle(self): self.iterate_behavior_timeouts() - # def run(self): - # log("%(class_name)s.run started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - # "class_name": self.__class__.__name__}) - # while self.is_work: - # self.pre_handle() - # for kafka_key in self.consumers: - # self.iterate(kafka_key) - # - # if self.health_check_server: - # with StatsTimer() as health_check_server_timer: - # self.health_check_server.iterate() - # - # if health_check_server_timer.msecs >= self.MAX_LOG_TIME: - # log("Health check iterate time: {} msecs\n".format(health_check_server_timer.msecs), - # params={log_const.KEY_NAME: "slow_health_check", - # "time_msecs": health_check_server_timer.msecs}, level="WARNING") - # - # log("Stopping Kafka handler", level="WARNING") - # for kafka_key in self.consumers: - # self.consumers[kafka_key].close() - # log("Kafka consumer connection is closed", level="WARNING") - # self.publishers[kafka_key].close() - # log("Kafka publisher connection is closed", level="WARNING") - # log("Kafka handler is stopped", level="WARNING") - def run(self): log("%(class_name)s.run started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) @@ -210,14 +186,22 @@ async def main_coro(self): "class_name": self.__class__.__name__}) async def healthcheck_coro(self): - if self.health_check_server: - with StatsTimer() as health_check_server_timer: - self.health_check_server.iterate() - - if health_check_server_timer.msecs >= self.MAX_LOG_TIME: - log("Health check iterate time: {} msecs\n".format(health_check_server_timer.msecs), - params={log_const.KEY_NAME: "slow_health_check", - "time_msecs": health_check_server_timer.msecs}, level="WARNING") + while True: + # does it work? start # + # from DP # + if not self.health_check_server_future or self.health_check_server_future.done() or self.health_check_server_future.cancelled(): + self.health_check_server_future = self.loop.run_in_executor(None, self.health_check_server.iterate) + await asyncio.sleep(0.5) + # does it work? end # + + # if self.health_check_server: + # with StatsTimer() as health_check_server_timer: + # self.health_check_server.iterate() + # + # if health_check_server_timer.msecs >= self.MAX_LOG_TIME: + # log("Health check iterate time: {} msecs\n".format(health_check_server_timer.msecs), + # params={log_const.KEY_NAME: "slow_health_check", + # "time_msecs": health_check_server_timer.msecs}, level="WARNING") async def main_work(self, kafka_key): # is needed? start # From 1b427c05300cca28dcd7c1b8e01cc085718cbb26 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Thu, 21 Oct 2021 10:27:42 +0300 Subject: [PATCH 010/116] asyncify part MailLoop.process_message --- smart_kit/start_points/main_loop_kafka.py | 28 ++--------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index d6eca4b3..d6d8f11e 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -126,31 +126,6 @@ async def save_user_async(self, db_uid, user, message): smart_kit_metrics.counter_save_collision(self.app_name) return no_collisions - async def load_user_async(self, db_uid, message): - db_data = None - load_error = False - try: - if self.db_adapter.IS_ASYNC: - db_data = await self.db_adapter.get(db_uid) - else: - db_data = self.db_adapter.get(db_uid) - except (DBAdapterException, ValueError): - log("Failed to get user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, - log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") - load_error = True - smart_kit_metrics.counter_load_error(self.app_name) - # to skip message when load failed - raise - return self.user_cls( - message.uid, - message=message, - db_data=db_data, # плохо ломать базовые интерфейсы - settings=self.settings, - descriptions=self.model.scenario_descriptions, - parametrizer_cls=self.parametrizer_cls, - load_error=load_error - ) - def pre_handle(self): self.iterate_behavior_timeouts() @@ -395,7 +370,8 @@ def process_message(self, mq_message, consumer, kafka_key, stats): with self.tracer.scope_manager.activate(span, True) as scope: with StatsTimer() as load_timer: - user = self.load_user(db_uid, message) + user = await self.load_user(db_uid, message) + # method async updated before that line # with self.tracer.scope_manager.activate(span, True) as scope: with self.tracer.start_span('Loading time', child_of=scope.span) as span: From 1297cf1f5e4eb41663c1a17d6be26f2dc489e537 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Mon, 25 Oct 2021 10:27:47 +0300 Subject: [PATCH 011/116] main_loop_kafka healthcheck --- smart_kit/start_points/base_main_loop.py | 2 +- smart_kit/start_points/main_loop_kafka.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index 86ec9636..0d330653 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -41,7 +41,7 @@ def __init__( self.model: SmartAppModel = model self.user_cls = user_cls self.parametrizer_cls = parametrizer_cls - self.db_adapter = await self.get_db() + self.db_adapter = self.get_db() self.is_work = True self.to_msg_validators: Iterable[MessageValidator] = to_msg_validators self.from_msg_validators: Iterable[MessageValidator] = from_msg_validators diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index d6d8f11e..5019f61b 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -45,7 +45,8 @@ def __init__(self, *args, **kwargs): "class_name": self.__class__.__name__}) self.loop = asyncio.get_event_loop() self.health_check_server_future = None - super().__init__(*args, **kwargs) + BaseMainLoop.__init__(self, *args, **kwargs) + try: kafka_config = _enrich_config_from_secret( self.settings["kafka"]["template-engine"], self.settings.get("secret_kafka", {}) @@ -144,7 +145,8 @@ async def main_coro(self): "class_name": self.__class__.__name__}) tasks = [self.main_work(kafka_key) for kafka_key in self.consumers] - tasks.append(self.healthcheck_coro()) + if self.health_check_server is not None: + tasks.append(self.healthcheck_coro()) await asyncio.gather(*tasks) # is needed? start # @@ -164,7 +166,8 @@ async def healthcheck_coro(self): while True: # does it work? start # # from DP # - if not self.health_check_server_future or self.health_check_server_future.done() or self.health_check_server_future.cancelled(): + if not self.health_check_server_future or self.health_check_server_future.done() or \ + self.health_check_server_future.cancelled(): self.health_check_server_future = self.loop.run_in_executor(None, self.health_check_server.iterate) await asyncio.sleep(0.5) # does it work? end # @@ -205,7 +208,7 @@ async def main_work(self, kafka_key): if mq_message: stats = "Polling time: {} msecs\n".format(poll_timer.msecs) message_value = mq_message.value() # DRY! - self.process_message(mq_message, consumer, kafka_key, stats) + await self.process_message(mq_message, consumer, kafka_key, stats) except KafkaException as kafka_exp: log("kafka error: %(kafka_exp)s. MESSAGE: {}.".format(message_value), @@ -323,7 +326,7 @@ def _get_topic_key(self, mq_message, kafka_key): topic_names_2_key = self._topic_names_2_key(kafka_key) return self.default_topic_key(kafka_key) or topic_names_2_key[mq_message.topic()] - def process_message(self, mq_message, consumer, kafka_key, stats): + async def process_message(self, mq_message, consumer, kafka_key, stats): topic_key = self._get_topic_key(mq_message, kafka_key) save_tries = 0 From 170f9f39e3b3372ed53938893123302ba4660970 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 25 Oct 2021 21:21:09 +0300 Subject: [PATCH 012/116] fix --- smart_kit/start_points/base_main_loop.py | 2 +- smart_kit/start_points/main_loop_kafka.py | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index 0d330653..86ec9636 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -41,7 +41,7 @@ def __init__( self.model: SmartAppModel = model self.user_cls = user_cls self.parametrizer_cls = parametrizer_cls - self.db_adapter = self.get_db() + self.db_adapter = await self.get_db() self.is_work = True self.to_msg_validators: Iterable[MessageValidator] = to_msg_validators self.from_msg_validators: Iterable[MessageValidator] = from_msg_validators diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 5019f61b..66bd4d1e 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -45,7 +45,7 @@ def __init__(self, *args, **kwargs): "class_name": self.__class__.__name__}) self.loop = asyncio.get_event_loop() self.health_check_server_future = None - BaseMainLoop.__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) try: kafka_config = _enrich_config_from_secret( @@ -172,15 +172,6 @@ async def healthcheck_coro(self): await asyncio.sleep(0.5) # does it work? end # - # if self.health_check_server: - # with StatsTimer() as health_check_server_timer: - # self.health_check_server.iterate() - # - # if health_check_server_timer.msecs >= self.MAX_LOG_TIME: - # log("Health check iterate time: {} msecs\n".format(health_check_server_timer.msecs), - # params={log_const.KEY_NAME: "slow_health_check", - # "time_msecs": health_check_server_timer.msecs}, level="WARNING") - async def main_work(self, kafka_key): # is needed? start # # from DP # From cd7ccbda366bbb5f5ecce4d1538e7ff8c236ff2d Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 25 Oct 2021 21:44:51 +0300 Subject: [PATCH 013/116] remove async method from __init__ --- smart_kit/start_points/base_main_loop.py | 8 +++++--- smart_kit/start_points/main_loop_kafka.py | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index 86ec9636..6f8bfbed 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -1,6 +1,7 @@ # coding=utf-8 from typing import Type, Iterable +import asyncio import signal import scenarios.logging.logger_constants as log_const @@ -36,12 +37,13 @@ def __init__( try: signal.signal(signal.SIGINT, self.stop) signal.signal(signal.SIGTERM, self.stop) + self.loop = asyncio.get_event_loop() self.settings = settings self.app_name = self.settings.app_name self.model: SmartAppModel = model self.user_cls = user_cls self.parametrizer_cls = parametrizer_cls - self.db_adapter = await self.get_db() + self.db_adapter = self.get_db() self.is_work = True self.to_msg_validators: Iterable[MessageValidator] = to_msg_validators self.from_msg_validators: Iterable[MessageValidator] = from_msg_validators @@ -68,10 +70,10 @@ def __init__( level="ERROR", exc_info=True) raise - async def get_db(self): + def get_db(self): db_adapter = db_adapter_factory(self.settings["template_settings"].get("db_adapter", {})) if db_adapter.IS_ASYNC: - await db_adapter.connect() + self.loop.run_until_complete(db_adapter.connect()) else: db_adapter.connect() return db_adapter diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 66bd4d1e..454c20b2 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -43,7 +43,6 @@ class MainLoop(BaseMainLoop): def __init__(self, *args, **kwargs): log("%(class_name)s.__init__ started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) - self.loop = asyncio.get_event_loop() self.health_check_server_future = None super().__init__(self, *args, **kwargs) From 7e52bce3c549b422bb46c471b7a25c17f5add56b Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 27 Oct 2021 12:52:12 +0300 Subject: [PATCH 014/116] async exc_handler --- core/utils/exception_handlers.py | 45 ++++++++++++++++------- smart_kit/start_points/main_loop_kafka.py | 3 +- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/core/utils/exception_handlers.py b/core/utils/exception_handlers.py index f272d4c1..375877fc 100644 --- a/core/utils/exception_handlers.py +++ b/core/utils/exception_handlers.py @@ -1,3 +1,4 @@ +import asyncio import sys from functools import wraps @@ -6,20 +7,38 @@ def exc_handler(on_error_obj_method_name=None, handled_exceptions=None): handled_exceptions = tuple(handled_exceptions) if handled_exceptions else (Exception,) def exc_handler_decorator(funct): - @wraps(funct) - def _wrapper(obj, *args, **kwarg): - result = None - try: - result = funct(obj, *args, **kwarg) - except handled_exceptions: + if asyncio.iscoroutinefunction(funct): + @wraps(funct) + async def _wrapper(obj, *args, **kwarg): + result = None try: - on_error = getattr(obj, on_error_obj_method_name) if \ - on_error_obj_method_name else (lambda *x: None) - result = on_error(*args, **kwarg) - except: - print(sys.exc_info()) - return result + result = await funct(obj, *args, **kwarg) + except handled_exceptions: + try: + on_error = getattr(obj, on_error_obj_method_name) if \ + on_error_obj_method_name else (lambda *x: None) + result = on_error(*args, **kwarg) + except: + print(sys.exc_info()) + return result - return _wrapper + return _wrapper + + else: + @wraps(funct) + def _wrapper(obj, *args, **kwarg): + result = None + try: + result = funct(obj, *args, **kwarg) + except handled_exceptions: + try: + on_error = getattr(obj, on_error_obj_method_name) if \ + on_error_obj_method_name else (lambda *x: None) + result = on_error(*args, **kwarg) + except: + print(sys.exc_info()) + return result + + return _wrapper return exc_handler_decorator diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 454c20b2..f4dfed5d 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -133,8 +133,7 @@ def run(self): log("%(class_name)s.run started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) try: - loop = asyncio.get_event_loop() - loop.run_until_complete(self.main_coro()) + self.loop.run_until_complete(self.main_coro()) except (SystemExit,): log("%(class_name)s.run stopped", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) From f16bccf659c35f4809b2d35fb7b6fbd69cbe8f09 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Wed, 27 Oct 2021 13:00:22 +0300 Subject: [PATCH 015/116] fix bug main_loop_kafka --- smart_kit/start_points/main_loop_kafka.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index f4dfed5d..ed2bd98c 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -44,7 +44,7 @@ def __init__(self, *args, **kwargs): log("%(class_name)s.__init__ started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) self.health_check_server_future = None - super().__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) try: kafka_config = _enrich_config_from_secret( From c9b10b472d31ed7a612399512a90b626d724befe Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Wed, 27 Oct 2021 14:35:16 +0300 Subject: [PATCH 016/116] fix bug in main_loop_kafka and make it more async --- smart_kit/models/smartapp_model.py | 2 +- smart_kit/start_points/main_loop_kafka.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/smart_kit/models/smartapp_model.py b/smart_kit/models/smartapp_model.py index bf5a4855..a6cd7541 100644 --- a/smart_kit/models/smartapp_model.py +++ b/smart_kit/models/smartapp_model.py @@ -54,7 +54,7 @@ def get_handler(self, message_type): return self._handlers[message_type] @exc_handler(on_error_obj_method_name="on_answer_error") - def answer(self, message, user): + async def answer(self, message, user): user.expire() handler = self.get_handler(message.type) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index ed2bd98c..430bb2db 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -166,7 +166,7 @@ async def healthcheck_coro(self): # from DP # if not self.health_check_server_future or self.health_check_server_future.done() or \ self.health_check_server_future.cancelled(): - self.health_check_server_future = self.loop.run_in_executor(None, self.health_check_server.iterate) + self.health_check_server_future = self.loop.run_in_executor(None, self.health_check_server.iterate) await asyncio.sleep(0.5) # does it work? end # @@ -216,7 +216,7 @@ async def main_work(self, kafka_key): log("Error handling worker fail exception.", level="ERROR", exc_info=True) - def _generate_answers(self, user, commands, message, **kwargs): + async def _generate_answers(self, user, commands, message, **kwargs): topic_key = kwargs["topic_key"] kafka_key = kwargs["kafka_key"] answers = [] @@ -363,25 +363,23 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): with self.tracer.scope_manager.activate(span, True) as scope: with StatsTimer() as load_timer: user = await self.load_user(db_uid, message) - # method async updated before that line # - with self.tracer.scope_manager.activate(span, True) as scope: with self.tracer.start_span('Loading time', child_of=scope.span) as span: smart_kit_metrics.sampling_load_time(self.app_name, load_timer.secs) stats += "Loading time: {} msecs\n".format(load_timer.msecs) with StatsTimer() as script_timer: - commands = self.model.answer(message, user) + commands = await self.model.answer(message, user) with self.tracer.start_span('Script time', child_of=scope.span) as span: - answers = self._generate_answers(user=user, commands=commands, message=message, - topic_key=topic_key, - kafka_key=kafka_key) + answers = await self._generate_answers(user=user, commands=commands, message=message, + topic_key=topic_key, + kafka_key=kafka_key) smart_kit_metrics.sampling_script_time(self.app_name, script_timer.secs) stats += "Script time: {} msecs\n".format(script_timer.msecs) with self.tracer.start_span('Saving time', child_of=scope.span) as span: with StatsTimer() as save_timer: - user_save_no_collisions = self.save_user(db_uid, user, message) + user_save_no_collisions = await self.save_user(db_uid, user, message) smart_kit_metrics.sampling_save_time(self.app_name, save_timer.secs) stats += "Saving time: {} msecs\n".format(save_timer.msecs) @@ -405,6 +403,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): mq_message.set_headers([]) self.tracer.inject(span_context=span.context, format=jaeger_kafka_codec.KAFKA_MAP, carrier=mq_message.headers()) + # method async updated before that line # if answers: for answer in answers: From af485af6c723dff099268c1468c1171d82d52e67 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Wed, 27 Oct 2021 23:15:59 +0300 Subject: [PATCH 017/116] remove unneccessary async in main_loop_kafka --- smart_kit/start_points/main_loop_kafka.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 430bb2db..7188ff5e 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -216,7 +216,7 @@ async def main_work(self, kafka_key): log("Error handling worker fail exception.", level="ERROR", exc_info=True) - async def _generate_answers(self, user, commands, message, **kwargs): + def _generate_answers(self, user, commands, message, **kwargs): topic_key = kwargs["topic_key"] kafka_key = kwargs["kafka_key"] answers = [] @@ -265,14 +265,14 @@ def iterate_behavior_timeouts(self): timeout_from_message = self._get_timeout_from_message(orig_message_raw, callback_id, headers=mq_message.headers()) - user = self.load_user(db_uid, timeout_from_message) + user = await self.load_user(db_uid, timeout_from_message) commands = self.model.answer(timeout_from_message, user) topic_key = self._get_topic_key(mq_message, kafka_key) answers = self._generate_answers(user=user, commands=commands, message=timeout_from_message, topic_key=topic_key, kafka_key=kafka_key) - user_save_no_collisions = self.save_user(db_uid, user, mq_message) + user_save_no_collisions = await self.save_user(db_uid, user, mq_message) if user and not user_save_no_collisions: log( @@ -371,7 +371,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): commands = await self.model.answer(message, user) with self.tracer.start_span('Script time', child_of=scope.span) as span: - answers = await self._generate_answers(user=user, commands=commands, message=message, + answers = self._generate_answers(user=user, commands=commands, message=message, topic_key=topic_key, kafka_key=kafka_key) smart_kit_metrics.sampling_script_time(self.app_name, script_timer.secs) @@ -403,7 +403,6 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): mq_message.set_headers([]) self.tracer.inject(span_context=span.context, format=jaeger_kafka_codec.KAFKA_MAP, carrier=mq_message.headers()) - # method async updated before that line # if answers: for answer in answers: From adf15b4299a0a5509b9fe965b4fb483d25040fc1 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Thu, 28 Oct 2021 00:28:42 +0300 Subject: [PATCH 018/116] main_loop_kafka remove outdated code --- smart_kit/start_points/main_loop_kafka.py | 97 +++-------------------- 1 file changed, 11 insertions(+), 86 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 7188ff5e..c0ab3c25 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -79,7 +79,6 @@ def __init__(self, *args, **kwargs): self.behaviors_timeouts = HeapqKV(value_to_key_func=lambda val: val.callback_id) # is needed? end # - self.iterate_tries = 0 self.concurrent_messages = 0 log("%(class_name)s.__init__ completed.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, @@ -90,44 +89,9 @@ def __init__(self, *args, **kwargs): level="ERROR", exc_info=True) raise - async def save_user_async(self, db_uid, user, message): - no_collisions = True - if user.do_not_save: - log("User %(uid)s will not saved", user=user, params={"uid": user.id, - log_const.KEY_NAME: "user_will_not_saved"}) - else: - - no_collisions = True - try: - str_data = user.raw_str - log("Saving User %(uid)s. Serialized utf8 json length is %(user_length)s symbols.", user=user, - params={"uid": user.id, - log_const.KEY_NAME: "user_save", - "user_length": len(str_data)}) - if user.initial_db_data and self.user_save_check_for_collisions: - if self.db_adapter.IS_ASYNC: - no_collisions = await self.db_adapter.replace_if_equals(db_uid, - sample=user.initial_db_data, - data=str_data) - else: - no_collisions = self.db_adapter.replace_if_equals(db_uid, - sample=user.initial_db_data, - data=str_data) - else: - if self.db_adapter.IS_ASYNC: - await self.db_adapter.save(db_uid, str_data) - else: - self.db_adapter.save(db_uid, str_data) - except (DBAdapterException, ValueError): - log("Failed to set user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, - log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") - smart_kit_metrics.counter_save_error(self.app_name) - if not no_collisions: - smart_kit_metrics.counter_save_collision(self.app_name) - return no_collisions - - def pre_handle(self): - self.iterate_behavior_timeouts() + # TODO find where it should be used + async def pre_handle(self): + await self.iterate_behavior_timeouts() def run(self): log("%(class_name)s.run started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, @@ -162,31 +126,24 @@ async def main_coro(self): async def healthcheck_coro(self): while True: - # does it work? start # - # from DP # if not self.health_check_server_future or self.health_check_server_future.done() or \ self.health_check_server_future.cancelled(): self.health_check_server_future = self.loop.run_in_executor(None, self.health_check_server.iterate) await asyncio.sleep(0.5) - # does it work? end # async def main_work(self, kafka_key): - # is needed? start # - # from DP # - self.iterate_tries = self.iterate_tries % 1000 + 1 - # is needed? end # - consumer = self.consumers[kafka_key] message_value = None - max_concurent_messages = self.settings["template_settings"].get("max_concurent_messages", 20) + max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 20) loop = asyncio.get_event_loop() while self.is_work: - if self.concurrent_messages >= max_concurent_messages: - log(f"%(class_name)s.main_work: max {max_concurent_messages} concurent messages occured", + # seems useless because self.concurrent_messages do not change + if self.concurrent_messages >= max_concurrent_messages: + log(f"%(class_name)s.main_work: max {max_concurrent_messages} concurrent messages occured", params={"class_name": self.__class__.__name__, - log_const.KEY_NAME: "max_concurent_messages"}, level='WARNING') + log_const.KEY_NAME: "max_concurrent_messages"}, level='WARNING') await asyncio.sleep(0.2) continue try: @@ -247,7 +204,7 @@ def _get_timeout_from_message(self, orig_message_raw, callback_id, headers): timeout_from_message.callback_id = callback_id return timeout_from_message - def iterate_behavior_timeouts(self): + async def iterate_behavior_timeouts(self): now = time.time() while now > (self.behaviors_timeouts.get_head_key() or float("inf")): _, behavior_timeout_value = self.behaviors_timeouts.pop() @@ -435,38 +392,6 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): smart_kit_metrics.counter_save_collision_tries_left(self.app_name) consumer.commit_offset(mq_message) - def iterate(self, kafka_key): - consumer = self.consumers[kafka_key] - mq_message = None - message_value = None - try: - mq_message = None - message_value = None - with StatsTimer() as poll_timer: - mq_message = consumer.poll() - - if mq_message: - stats = "Polling time: {} msecs\n".format(poll_timer.msecs) - message_value = mq_message.value() # DRY! - self.process_message(mq_message, consumer, kafka_key, stats) - - except KafkaException as kafka_exp: - log("kafka error: %(kafka_exp)s. MESSAGE: {}.".format(message_value), - params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "kafka_exp": str(kafka_exp), - log_const.REQUEST_VALUE: str(message_value)}, - level="ERROR", exc_info=True) - except Exception: - try: - log("%(class_name)s iterate error. Kafka key %(kafka_key)s MESSAGE: {}.".format(message_value), - params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "kafka_key": kafka_key}, - level="ERROR", exc_info=True) - consumer.commit_offset(mq_message) - except Exception: - log("Error handling worker fail exception.", - level="ERROR", exc_info=True) - def check_message_key(self, from_message, message_key, user): sub = from_message.sub channel = from_message.channel @@ -577,7 +502,7 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): timeout_from_message = self._get_timeout_from_message(orig_message_raw, callback_id, headers=mq_message.headers()) - user = await self.load_user_async(db_uid, timeout_from_message) + user = await self.load_user(db_uid, timeout_from_message) # TODO: not to load user to check behaviors.has_callback ? if user.behaviors.has_callback(callback_id): callback_found = True @@ -587,7 +512,7 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): topic_key=topic_key, kafka_key=kafka_key) - user_save_ok = await self.save_user_async(db_uid, user, mq_message) + user_save_ok = await self.save_user(db_uid, user, mq_message) if not user_save_ok: log( From 0cd3ead36a745e9b21c8256e471eb3a67e0b050b Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Thu, 28 Oct 2021 10:34:29 +0300 Subject: [PATCH 019/116] main_loop_kafka fix self.concurrent_messages --- smart_kit/start_points/main_loop_kafka.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index c0ab3c25..f27d70a2 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -134,12 +134,11 @@ async def healthcheck_coro(self): async def main_work(self, kafka_key): consumer = self.consumers[kafka_key] message_value = None - max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 20) + max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 10) loop = asyncio.get_event_loop() while self.is_work: - # seems useless because self.concurrent_messages do not change if self.concurrent_messages >= max_concurrent_messages: log(f"%(class_name)s.main_work: max {max_concurrent_messages} concurrent messages occured", params={"class_name": self.__class__.__name__, @@ -147,6 +146,7 @@ async def main_work(self, kafka_key): await asyncio.sleep(0.2) continue try: + self.concurrent_messages += 1 mq_message = None with StatsTimer() as poll_timer: # Max delay between polls configured in consumer.poll_timeout param @@ -157,12 +157,14 @@ async def main_work(self, kafka_key): await self.process_message(mq_message, consumer, kafka_key, stats) except KafkaException as kafka_exp: + self.concurrent_messages -= 1 log("kafka error: %(kafka_exp)s. MESSAGE: {}.".format(message_value), params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "kafka_exp": str(kafka_exp), log_const.REQUEST_VALUE: str(message_value)}, level="ERROR", exc_info=True) except Exception: + self.concurrent_messages -= 1 try: log("%(class_name)s iterate error. Kafka key %(kafka_key)s MESSAGE: {}.".format(message_value), params={log_const.KEY_NAME: log_const.STARTUP_VALUE, @@ -172,6 +174,8 @@ async def main_work(self, kafka_key): except Exception: log("Error handling worker fail exception.", level="ERROR", exc_info=True) + else: + self.concurrent_messages -= 1 def _generate_answers(self, user, commands, message, **kwargs): topic_key = kwargs["topic_key"] From 88c260b443a6cd999a2598221b3ab96b79ea9f54 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Thu, 28 Oct 2021 10:41:55 +0300 Subject: [PATCH 020/116] main_loop_kafka refactoring --- smart_kit/start_points/main_loop_kafka.py | 27 +++++++++-------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index c0ab3c25..90fd0c73 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -246,8 +246,7 @@ async def iterate_behavior_timeouts(self): continue if not user_save_no_collisions: - log( - "MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", + log("MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, @@ -298,8 +297,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): smart_kit_metrics.sampling_mq_waiting_time(self.app_name, waiting_message_time / 1000) self.check_message_key(message, mq_message.key(), user) - log( - "INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(incoming_data)s", + log("INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(incoming_data)s", params={log_const.KEY_NAME: "incoming_message", "topic": mq_message.topic(), "message_partition": mq_message.partition(), @@ -311,7 +309,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): "surface": message.device.surface, MESSAGE_ID_STR: message.incremental_id}, user=user - ) + ) db_uid = message.db_uid @@ -329,8 +327,8 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): with self.tracer.start_span('Script time', child_of=scope.span) as span: answers = self._generate_answers(user=user, commands=commands, message=message, - topic_key=topic_key, - kafka_key=kafka_key) + topic_key=topic_key, + kafka_key=kafka_key) smart_kit_metrics.sampling_script_time(self.app_name, script_timer.secs) stats += "Script time: {} msecs\n".format(script_timer.msecs) @@ -341,8 +339,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): smart_kit_metrics.sampling_save_time(self.app_name, save_timer.secs) stats += "Saving time: {} msecs\n".format(save_timer.msecs) if not user_save_no_collisions: - log( - "MainLoop.iterate: save user got collision on uid %(uid)s db_version %(db_version)s.", + log("MainLoop.iterate: save user got collision on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, @@ -377,8 +374,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): "data": data}, level="ERROR") smart_kit_metrics.counter_invalid_message(self.app_name) if user and not user_save_no_collisions: - log( - "MainLoop.iterate: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", + log("MainLoop.iterate: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, @@ -491,8 +487,7 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): user = None while save_tries < self.user_save_collisions_tries and not user_save_ok: callback_found = False - log( - f"MainLoop.do_behavior_timeout: handling callback {callback_id}. for db_uid {db_uid}. try {save_tries}.") + log(f"MainLoop.do_behavior_timeout: handling callback {callback_id}. for db_uid {db_uid}. try {save_tries}.") save_tries += 1 @@ -515,8 +510,7 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): user_save_ok = await self.save_user(db_uid, user, mq_message) if not user_save_ok: - log( - "MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(db_version)s.", + log("MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, @@ -527,8 +521,7 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): level="WARNING") if not user_save_ok and callback_found: - log( - "MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", + log("MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, From ea19cc0021ec65c63698209d7dd6da9a4872cca9 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Thu, 28 Oct 2021 10:43:30 +0300 Subject: [PATCH 021/116] main_loop_kafka refactoring --- smart_kit/start_points/main_loop_kafka.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 76507774..ebe93bf1 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -236,8 +236,7 @@ async def iterate_behavior_timeouts(self): user_save_no_collisions = await self.save_user(db_uid, user, mq_message) if user and not user_save_no_collisions: - log( - "MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(db_version)s.", + log("MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, From 07f696e5cadc5041a890a45b1b1b38a6a9588dd0 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Thu, 28 Oct 2021 23:22:53 +0300 Subject: [PATCH 022/116] refactoring main_loop_kafka --- smart_kit/start_points/main_loop_kafka.py | 75 +++++++++++++---------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index ebe93bf1..e0dc6b03 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -2,6 +2,7 @@ import asyncio import json import time +import sys from collections import namedtuple from functools import lru_cache @@ -103,27 +104,11 @@ def run(self): "class_name": self.__class__.__name__}) async def main_coro(self): - log("%(class_name)s.main_coro started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "class_name": self.__class__.__name__}) - tasks = [self.main_work(kafka_key) for kafka_key in self.consumers] if self.health_check_server is not None: tasks.append(self.healthcheck_coro()) await asyncio.gather(*tasks) - # is needed? start # - log("Stopping Kafka handler", level="WARNING") - for kafka_key in self.consumers: - self.consumers[kafka_key].close() - log("Kafka consumer connection is closed", level="WARNING") - self.publishers[kafka_key].close() - log("Kafka publisher connection is closed", level="WARNING") - log("Kafka handler is stopped", level="WARNING") - # is needed? end # - - log("%(class_name)s.main_coro completed", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "class_name": self.__class__.__name__}) - async def healthcheck_coro(self): while True: if not self.health_check_server_future or self.health_check_server_future.done() or \ @@ -134,46 +119,60 @@ async def healthcheck_coro(self): async def main_work(self, kafka_key): consumer = self.consumers[kafka_key] message_value = None - max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 10) - - loop = asyncio.get_event_loop() + max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 10) + max_concurrent_messages_delay = self.settings["template_settings"].get("max_concurrent_messages_delay", 0.1) + last_poll_begin_time = time.time() while self.is_work: + from_last_poll_begin_ms = int((time.time() - last_poll_begin_time) * 1000) + stats = "From last poll time: {} msecs\n".format(from_last_poll_begin_ms) + log_params = { + log_const.KEY_NAME: "timings", + "from_last_poll_begin_ms": from_last_poll_begin_ms + } + last_poll_begin_time = time.time() if self.concurrent_messages >= max_concurrent_messages: - log(f"%(class_name)s.main_work: max {max_concurrent_messages} concurrent messages occured", - params={"class_name": self.__class__.__name__, - log_const.KEY_NAME: "max_concurrent_messages"}, level='WARNING') - await asyncio.sleep(0.2) + log(f"main_loop.main_work: max {max_concurrent_messages} concurrent messages occured", + params={log_const.KEY_NAME: "max_concurrent_messages"}, level='WARNING') + await asyncio.sleep(max_concurrent_messages_delay) + total_delay = log_params.get("waiting_max_concurrent_messages", 0) + total_delay += max_concurrent_messages_delay + log_params["waiting_max_concurrent_messages"] = total_delay continue try: self.concurrent_messages += 1 mq_message = None with StatsTimer() as poll_timer: # Max delay between polls configured in consumer.poll_timeout param - mq_message = await loop.run_in_executor(None, consumer.poll) + mq_message = await self.loop.run_in_executor(None, consumer.poll) if mq_message: - stats = "Polling time: {} msecs\n".format(poll_timer.msecs) - message_value = mq_message.value() # DRY! + headers = mq_message.headers() + if headers is None: + raise Exception("No incoming message headers found.") + stats += "Polling time: {} msecs\n".format(poll_timer.msecs) + log_params["kafka_polling"] = poll_timer.msecs + message_value = json.loads(mq_message.value()) await self.process_message(mq_message, consumer, kafka_key, stats) except KafkaException as kafka_exp: self.concurrent_messages -= 1 - log("kafka error: %(kafka_exp)s. MESSAGE: {}.".format(message_value), + log("kafka error: %(kafka_exp)s.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "kafka_exp": str(kafka_exp), log_const.REQUEST_VALUE: str(message_value)}, level="ERROR", exc_info=True) except Exception: self.concurrent_messages -= 1 + log("%(class_name)s iterate error. Kafka key %(kafka_key)s", + params={log_const.KEY_NAME: "worker_exception", + "kafka_key": kafka_key, + log_const.REQUEST_VALUE: str(message_value)}, + level="ERROR", exc_info=True) try: - log("%(class_name)s iterate error. Kafka key %(kafka_key)s MESSAGE: {}.".format(message_value), - params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "kafka_key": kafka_key}, - level="ERROR", exc_info=True) consumer.commit_offset(mq_message) except Exception: - log("Error handling worker fail exception.", - level="ERROR", exc_info=True) + log("Error handling worker fail exception.", level="ERROR", exc_info=True) + raise else: self.concurrent_messages -= 1 @@ -478,8 +477,18 @@ def save_behavior_timeouts(self, user, mq_message, kafka_key): ) def stop(self, signum, frame): + log("Stop call!") + log("Stopping Kafka handler", level="WARNING") + for kafka_key in self.consumers: + self.consumers[kafka_key].close() + log("Kafka consumer connection is closed", level="WARNING") + self.publishers[kafka_key].close() + log("Kafka publisher connection is closed", level="WARNING") + log("Kafka handler is stopped", level="WARNING") self.is_work = False + sys.exit() + async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): if not self.is_work: return From 579eb7f469cb2fee37f7145fc09ebbef9f3cdc0c Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Mon, 1 Nov 2021 09:36:04 +0300 Subject: [PATCH 023/116] done async for main_loop_kafka --- smart_kit/start_points/base_main_loop.py | 27 ++++---- smart_kit/start_points/main_loop_kafka.py | 75 ++++++++++++++++++----- 2 files changed, 73 insertions(+), 29 deletions(-) diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index 6f8bfbed..c8d0a74e 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -72,10 +72,11 @@ def __init__( def get_db(self): db_adapter = db_adapter_factory(self.settings["template_settings"].get("db_adapter", {})) - if db_adapter.IS_ASYNC: - self.loop.run_until_complete(db_adapter.connect()) - else: - db_adapter.connect() + if not db_adapter.IS_ASYNC: + raise Exception( + f"Blocking adapter {db_adapter.__class__.__name__} is not good for {self.__class__.__name__}" + ) + self.loop.run_until_complete(db_adapter.connect()) return db_adapter def _generate_answers(self, user, commands, message, **kwargs): @@ -105,12 +106,14 @@ async def load_user(self, db_uid, message): db_data = None load_error = False try: - db_data = await self.db_adapter.get(db_uid) if self.db_adapter.IS_ASYNC else self.db_adapter.get(db_uid) + db_data = await self.db_adapter.get(db_uid) except (DBAdapterException, ValueError): log("Failed to get user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") load_error = True smart_kit_metrics.counter_load_error(self.app_name) + # to skip message when load failed + raise return self.user_cls( message.uid, message=message, @@ -136,17 +139,11 @@ async def save_user(self, db_uid, user, message): log_const.KEY_NAME: "user_save", "user_length": len(str_data)}) if user.initial_db_data and self.user_save_check_for_collisions: - if self.db_adapter.IS_ASYNC: - no_collisions = await self.db_adapter.replace_if_equals(db_uid, - sample=user.initial_db_data, - data=str_data) - else: - no_collisions = self.db_adapter.replace_if_equals(db_uid, - sample=user.initial_db_data, - data=str_data) + no_collisions = await self.db_adapter.replace_if_equals(db_uid, + sample=user.initial_db_data, + data=str_data) else: - await self.db_adapter.save(db_uid, str_data) if self.db_adapter.IS_ASYNC else \ - self.db_adapter.save(db_uid, str_data) + await self.db_adapter.save(db_uid, str_data) except (DBAdapterException, ValueError): log("Failed to set user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index e0dc6b03..160f1a96 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -152,7 +152,7 @@ async def main_work(self, kafka_key): stats += "Polling time: {} msecs\n".format(poll_timer.msecs) log_params["kafka_polling"] = poll_timer.msecs message_value = json.loads(mq_message.value()) - await self.process_message(mq_message, consumer, kafka_key, stats) + await self.process_message(mq_message, consumer, kafka_key, stats, log_params) except KafkaException as kafka_exp: self.concurrent_messages -= 1 @@ -273,14 +273,14 @@ def _get_topic_key(self, mq_message, kafka_key): topic_names_2_key = self._topic_names_2_key(kafka_key) return self.default_topic_key(kafka_key) or topic_names_2_key[mq_message.topic()] - async def process_message(self, mq_message, consumer, kafka_key, stats): + async def process_message(self, mq_message, consumer, kafka_key, stats, log_params): topic_key = self._get_topic_key(mq_message, kafka_key) - save_tries = 0 - user_save_no_collisions = False + user_save_ok = False user = None db_uid = None - while save_tries < self.user_save_collisions_tries and not user_save_no_collisions: + validation_failed = False + while save_tries < self.user_save_collisions_tries and not user_save_ok: save_tries += 1 message_value = mq_message.value() message = SmartAppFromMessage(message_value, @@ -294,10 +294,16 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): if message.creation_time: waiting_message_time = time.time() * 1000 - message.creation_time stats += "Waiting message: {} msecs\n".format(waiting_message_time) + log_params["waiting_message"] = waiting_message_time stats += "Mid: {}\n".format(message.incremental_id) + log_params[MESSAGE_ID_STR] = message.incremental_id smart_kit_metrics.sampling_mq_waiting_time(self.app_name, waiting_message_time / 1000) + if self._is_message_timeout_to_skip(message, waiting_message_time): + skip_timeout = True + break + self.check_message_key(message, mq_message.key(), user) log("INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(incoming_data)s", params={log_const.KEY_NAME: "incoming_message", @@ -318,8 +324,9 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): span = jaeger_utils.get_incoming_spam(self.tracer, message, mq_message) with self.tracer.scope_manager.activate(span, True) as scope: - with StatsTimer() as load_timer: - user = await self.load_user(db_uid, message) + with self.tracer.start_span('Loading time', child_of=scope.span) as span: + with StatsTimer() as load_timer: + user = await self.load_user(db_uid, message) with self.tracer.start_span('Loading time', child_of=scope.span) as span: smart_kit_metrics.sampling_load_time(self.app_name, load_timer.secs) @@ -331,16 +338,18 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): answers = self._generate_answers(user=user, commands=commands, message=message, topic_key=topic_key, kafka_key=kafka_key) - smart_kit_metrics.sampling_script_time(self.app_name, script_timer.secs) stats += "Script time: {} msecs\n".format(script_timer.msecs) + log_params["script_time"] = script_timer.msecs + smart_kit_metrics.sampling_script_time(self.app_name, script_timer.secs) with self.tracer.start_span('Saving time', child_of=scope.span) as span: with StatsTimer() as save_timer: - user_save_no_collisions = await self.save_user(db_uid, user, message) + user_save_ok = await self.save_user(db_uid, user, message) + stats += "Saving user to DB time: {} msecs\n".format(save_timer.msecs) + log_params["user_saving"] = save_timer.msecs smart_kit_metrics.sampling_save_time(self.app_name, save_timer.secs) - stats += "Saving time: {} msecs\n".format(save_timer.msecs) - if not user_save_no_collisions: + if not user_save_ok: log("MainLoop.iterate: save user got collision on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", @@ -353,7 +362,8 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): level="WARNING") continue - self.save_behavior_timeouts(user, mq_message, kafka_key) + if answers: + self.save_behavior_timeouts(user, mq_message, kafka_key) if mq_message.headers() is None: mq_message.set_headers([]) @@ -364,9 +374,12 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): for answer in answers: with StatsTimer() as publish_timer: self._send_request(user, answer, mq_message) + smart_kit_metrics.counter_outgoing(self.app_name, answer.command.name, answer, user) stats += "Publishing time: {} msecs".format(publish_timer.msecs) + log_params["kafka_publishing"] = publish_timer.msecs log(stats) else: + validation_failed = True try: data = message.masked_value except: @@ -375,7 +388,11 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): params={log_const.KEY_NAME: "invalid_message", "data": data}, level="ERROR") smart_kit_metrics.counter_invalid_message(self.app_name) - if user and not user_save_no_collisions: + break + if stats: + log(stats, user=user, params=log_params) + + if user and not user_save_ok and not validation_failed: log("MainLoop.iterate: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", @@ -386,10 +403,40 @@ async def process_message(self, mq_message, consumer, kafka_key, stats): "uid": user.id, "db_version": str(user.variables.get(user.USER_DB_VERSION))}, level="WARNING") - smart_kit_metrics.counter_save_collision_tries_left(self.app_name) + consumer.commit_offset(mq_message) + def _is_message_timeout_to_skip(self, message, waiting_message_time): + # Returns True if timeout is found + waiting_message_timeout = self.settings["template_settings"].get("waiting_message_timeout", {}) + warning_delay = waiting_message_timeout.get('warning', 200) + skip_delay = waiting_message_timeout.get('skip', 8000) + log_level = None + make_break = False + + if waiting_message_time >= skip_delay: + # Too old message + log_level = "ERROR" + make_break = True + + elif waiting_message_time >= warning_delay: + # Warn, but continue message processing + log_level = "WARNING" + smart_kit_metrics.counter_mq_long_waiting(self.app_name) + + if log_level is not None: + log( + f"Out of time message %(waiting_message_time)s msecs, " + f"mid: %(mid)s {message.as_dict}", + params={ + log_const.KEY_NAME: "waiting_message_timeout", + "waiting_message_time": waiting_message_time, + "mid": message.incremental_id + }, + level=log_level) + return make_break + def check_message_key(self, from_message, message_key, user): sub = from_message.sub channel = from_message.channel From 504622b67441acc8cd34ec751aaf6b295940f390 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 1 Nov 2021 21:27:36 +0300 Subject: [PATCH 024/116] async handlers and smartAppModel --- scenarios/user/user_model.py | 1 + smart_kit/handlers/handle_close_app.py | 4 ++-- smart_kit/handlers/handle_respond.py | 2 +- smart_kit/handlers/handle_server_action.py | 2 +- smart_kit/handlers/handler_base.py | 2 +- smart_kit/handlers/handler_text.py | 6 +++--- smart_kit/handlers/handler_timeout.py | 4 ++-- smart_kit/models/smartapp_model.py | 2 +- smart_kit/start_points/main_loop_kafka.py | 5 ++--- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/scenarios/user/user_model.py b/scenarios/user/user_model.py index 9cdd9939..889d7ac1 100644 --- a/scenarios/user/user_model.py +++ b/scenarios/user/user_model.py @@ -18,6 +18,7 @@ from smart_kit.utils.monitoring import smart_kit_metrics import scenarios.logging.logger_constants as log_const + class User(BaseUser): forms: Forms diff --git a/smart_kit/handlers/handle_close_app.py b/smart_kit/handlers/handle_close_app.py index eed21e61..d13c6a62 100644 --- a/smart_kit/handlers/handle_close_app.py +++ b/smart_kit/handlers/handle_close_app.py @@ -11,8 +11,8 @@ def __init__(self, app_name): super(HandlerCloseApp, self).__init__(app_name) self._clear_current_scenario = ClearCurrentScenarioAction(None) - def run(self, payload, user): - super().run(payload, user) + async def run(self, payload, user): + await super().run(payload, user) text_preprocessing_result = TextPreprocessingResult(payload.get("message", {})) params = { log_const.KEY_NAME: "HandlerCloseApp", diff --git a/smart_kit/handlers/handle_respond.py b/smart_kit/handlers/handle_respond.py index 35bb6988..3174075a 100644 --- a/smart_kit/handlers/handle_respond.py +++ b/smart_kit/handlers/handle_respond.py @@ -24,7 +24,7 @@ def get_action_params(self, payload, user): callback_id = user.message.callback_id return user.behaviors.get_callback_action_params(callback_id) - def run(self, payload, user): + async def run(self, payload, user): callback_id = user.message.callback_id action_params = self.get_action_params(payload, user) action_name = self.get_action_name(payload, user) diff --git a/smart_kit/handlers/handle_server_action.py b/smart_kit/handlers/handle_server_action.py index 3eeb10c2..8388349d 100644 --- a/smart_kit/handlers/handle_server_action.py +++ b/smart_kit/handlers/handle_server_action.py @@ -22,7 +22,7 @@ def get_action_name(self, payload, user): def get_action_params(self, payload): return payload[SERVER_ACTION].get("parameters", {}) - def run(self, payload, user): + async def run(self, payload, user): action_params = pickle_deepcopy(self.get_action_params(payload)) params = {log_const.KEY_NAME: "handling_server_action", "server_action_params": str(action_params), diff --git a/smart_kit/handlers/handler_base.py b/smart_kit/handlers/handler_base.py index 697ef96b..98291441 100644 --- a/smart_kit/handlers/handler_base.py +++ b/smart_kit/handlers/handler_base.py @@ -9,7 +9,7 @@ class HandlerBase: def __init__(self, app_name): self.app_name = app_name - def run(self, payload, user): + async def run(self, payload, user): # отправка события о входящем сообщении в систему мониторинга smart_kit_metrics.counter_incoming(self.app_name, user.message.message_name, self.__class__.__name__, user, app_info=user.message.app_info) diff --git a/smart_kit/handlers/handler_text.py b/smart_kit/handlers/handler_text.py index ea6c8c96..d586d19c 100644 --- a/smart_kit/handlers/handler_text.py +++ b/smart_kit/handlers/handler_text.py @@ -19,8 +19,8 @@ def __init__(self, app_name, dialogue_manager): f"{self.__class__.__name__}.__init__ finished.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE} ) - def run(self, payload, user): - super().run(payload, user) + async def run(self, payload, user): + await super().run(payload, user) text_preprocessing_result = TextPreprocessingResult(payload.get("message", {})) log("text preprocessing result", user, @@ -29,6 +29,6 @@ def run(self, payload, user): answer = self._handle_base(text_preprocessing_result, user) return answer - def _handle_base(self, text_preprocessing_result, user): + async def _handle_base(self, text_preprocessing_result, user): answer, is_answer_found = self.dialogue_manager.run(text_preprocessing_result, user) return answer or [] diff --git a/smart_kit/handlers/handler_timeout.py b/smart_kit/handlers/handler_timeout.py index 570bcf5e..da5b3fb7 100644 --- a/smart_kit/handlers/handler_timeout.py +++ b/smart_kit/handlers/handler_timeout.py @@ -10,8 +10,8 @@ class HandlerTimeout(HandlerBase): - def run(self, payload, user): - super().run(payload, user) + async def run(self, payload, user): + await super().run(payload, user) callback_id = user.message.callback_id if user.behaviors.has_callback(callback_id): params = {log_const.KEY_NAME: "handling_timeout"} diff --git a/smart_kit/models/smartapp_model.py b/smart_kit/models/smartapp_model.py index a6cd7541..c9afb824 100644 --- a/smart_kit/models/smartapp_model.py +++ b/smart_kit/models/smartapp_model.py @@ -59,7 +59,7 @@ async def answer(self, message, user): handler = self.get_handler(message.type) if not user.load_error: - commands = handler.run(message.payload, user) + commands = await handler.run(message.payload, user) else: log("Error in loading user data", user, level="ERROR", exc_info=True) raise Exception("Error in loading user data") diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 160f1a96..48da294e 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -10,7 +10,6 @@ from lazy import lazy import scenarios.logging.logger_constants as log_const -from core.db_adapter.db_adapter import DBAdapterException from core.jaeger_custom_client import jaeger_utils from core.jaeger_custom_client import kafka_codec as jaeger_kafka_codec from core.logging.logger_utils import log, UID_STR, MESSAGE_ID_STR @@ -226,7 +225,7 @@ async def iterate_behavior_timeouts(self): headers=mq_message.headers()) user = await self.load_user(db_uid, timeout_from_message) - commands = self.model.answer(timeout_from_message, user) + commands = await self.model.answer(timeout_from_message, user) topic_key = self._get_topic_key(mq_message, kafka_key) answers = self._generate_answers(user=user, commands=commands, message=timeout_from_message, topic_key=topic_key, @@ -560,7 +559,7 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): # TODO: not to load user to check behaviors.has_callback ? if user.behaviors.has_callback(callback_id): callback_found = True - commands = self.model.answer(timeout_from_message, user) + commands = await self.model.answer(timeout_from_message, user) topic_key = self._get_topic_key(mq_message, kafka_key) answers = self._generate_answers(user=user, commands=commands, message=timeout_from_message, topic_key=topic_key, From 5adbe1d4043c2190a69e31234c21c37f7e7f8716 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 2 Nov 2021 02:20:57 +0300 Subject: [PATCH 025/116] async scenarios --- core/basic_models/scenarios/base_scenario.py | 2 +- scenarios/scenario_descriptions/form_filling_scenario.py | 2 +- .../scenario_descriptions/tree_scenario/tree_scenario.py | 2 +- smart_kit/handlers/handler_text.py | 2 +- smart_kit/models/dialogue_manager.py | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/basic_models/scenarios/base_scenario.py b/core/basic_models/scenarios/base_scenario.py index 5a92d9c3..a72ee76a 100644 --- a/core/basic_models/scenarios/base_scenario.py +++ b/core/basic_models/scenarios/base_scenario.py @@ -96,5 +96,5 @@ def get_action_results(self, user, text_preprocessing_result, def history(self): return {"scenario_path": [{"scenario": self.id, "node": None}]} - def run(self, text_preprocessing_result, user, params: Dict[str, Any] = None): + async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = None): return self.get_action_results(user, text_preprocessing_result, self.actions, params) diff --git a/scenarios/scenario_descriptions/form_filling_scenario.py b/scenarios/scenario_descriptions/form_filling_scenario.py index 8dbe0f02..12ba79e7 100644 --- a/scenarios/scenario_descriptions/form_filling_scenario.py +++ b/scenarios/scenario_descriptions/form_filling_scenario.py @@ -175,7 +175,7 @@ def get_reply(self, user, text_preprocessing_result, reply_actions, field, form) return action_messages @monitoring.got_histogram("scenario_time") - def run(self, text_preprocessing_result, user, params: Dict[str, Any] = None): + async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = None): form = self._get_form(user) user.last_scenarios.add(self.id, text_preprocessing_result) user.preprocessing_messages_for_scenarios.add(text_preprocessing_result) diff --git a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py index 8c2a60a4..28c7a4b6 100644 --- a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py +++ b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py @@ -85,7 +85,7 @@ def get_fields_data(self, main_form, form_type): return all_forms_fields @monitoring.got_histogram("scenario_time") - def run(self, text_preprocessing_result, user, params: Dict[str, Any] = None): + async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = None): main_form = self._get_form(user) user.last_scenarios.add(self.id, text_preprocessing_result) user.preprocessing_messages_for_scenarios.add(text_preprocessing_result) diff --git a/smart_kit/handlers/handler_text.py b/smart_kit/handlers/handler_text.py index d586d19c..6142d6ab 100644 --- a/smart_kit/handlers/handler_text.py +++ b/smart_kit/handlers/handler_text.py @@ -30,5 +30,5 @@ async def run(self, payload, user): return answer async def _handle_base(self, text_preprocessing_result, user): - answer, is_answer_found = self.dialogue_manager.run(text_preprocessing_result, user) + answer, is_answer_found = await self.dialogue_manager.run(text_preprocessing_result, user) return answer or [] diff --git a/smart_kit/models/dialogue_manager.py b/smart_kit/models/dialogue_manager.py index 50183ec8..5c88cc00 100644 --- a/smart_kit/models/dialogue_manager.py +++ b/smart_kit/models/dialogue_manager.py @@ -27,7 +27,7 @@ def __init__(self, scenario_descriptions, app_name, **kwargs): def _nothing_found_action(self): return self.actions.get(self.NOTHING_FOUND_ACTION) or NothingFoundAction() - def run(self, text_preprocessing_result, user): + async def run(self, text_preprocessing_result, user): scenarios_names = user.last_scenarios.scenarios_names scenario_key = user.message.payload[field.INTENT] @@ -49,9 +49,9 @@ def run(self, text_preprocessing_result, user): return self._nothing_found_action.run(user, text_preprocessing_result), False - return self.run_scenario(scenario_key, text_preprocessing_result, user), True + return await self.run_scenario(scenario_key, text_preprocessing_result, user), True - def run_scenario(self, scen_id, text_preprocessing_result, user): + async def run_scenario(self, scen_id, text_preprocessing_result, user): initial_last_scenario = user.last_scenarios.last_scenario_name scenario = self.scenarios[scen_id] params = {log_const.KEY_NAME: log_const.CHOSEN_SCENARIO_VALUE, @@ -59,7 +59,7 @@ def run_scenario(self, scen_id, text_preprocessing_result, user): log_const.SCENARIO_DESCRIPTION_VALUE: scenario.scenario_description } log(log_const.LAST_SCENARIO_MESSAGE, user, params) - run_scenario_result = scenario.run(text_preprocessing_result, user) + run_scenario_result = await scenario.run(text_preprocessing_result, user) actual_last_scenario = user.last_scenarios.last_scenario_name if actual_last_scenario and actual_last_scenario != initial_last_scenario: From 80e2d8a257f42f1d973da3beef5c69460b70cef0 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 2 Nov 2021 02:50:00 +0300 Subject: [PATCH 026/116] async tests --- .../scenarios_test/test_tree_scenario.py | 8 ++++---- .../handlers/test_handle_close_app.py | 4 ++-- .../smart_kit_tests/handlers/test_handle_respond.py | 6 +++--- tests/smart_kit_tests/handlers/test_handler_text.py | 12 ++++++------ .../handlers/test_handler_timeout.py | 4 ++-- .../smart_kit_tests/models/test_dialogue_manager.py | 13 ++++++------- 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py index ce63ce03..d223a97a 100644 --- a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py +++ b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py @@ -29,7 +29,7 @@ def run(self, user, text_preprocessing_result, params): class TestTreeScenario(TestCase): - def test_1(self): + async def test_1(self): """ Тест проверяет сценарий из одного узла. Предполагается идеальный случай, когда одно поле и мы смогли его заполнить. @@ -84,11 +84,11 @@ def test_1(self): scenario = TreeScenario(items, 1) - result = scenario.run(text_preprocessing_result, user) + result = await scenario.run(text_preprocessing_result, user) self.assertIsNone(current_node_mock.current_node) context_forms.new.assert_called_once_with(form_type) - def test_breake(self): + async def test_breake(self): """ Тест проверяет выход из сценария если сработает флаг break_scenario """ @@ -145,7 +145,7 @@ def test_breake(self): scenario = TreeScenario(items, 1) - result = scenario.run(text_preprocessing_result, user) + result = await scenario.run(text_preprocessing_result, user) self.assertFalse(scenario.actions[0].called) self.assertEqual(result[0].name, "break action result") diff --git a/tests/smart_kit_tests/handlers/test_handle_close_app.py b/tests/smart_kit_tests/handlers/test_handle_close_app.py index e221e2fc..a2ad1245 100644 --- a/tests/smart_kit_tests/handlers/test_handle_close_app.py +++ b/tests/smart_kit_tests/handlers/test_handle_close_app.py @@ -35,7 +35,7 @@ def test_handler_close_app_init(self): self.assertIsNotNone(obj.KAFKA_KEY) self.assertIsNotNone(obj._clear_current_scenario) - def test_handler_close_app_run(self): + async def test_handler_close_app_run(self): self.assertIsNotNone(handle_close_app.log_const.KEY_NAME) obj = handle_close_app.HandlerCloseApp(app_name=self.app_name) - self.assertIsNone(obj.run(self.test_payload, self.test_user)) + self.assertIsNone(await obj.run(self.test_payload, self.test_user)) diff --git a/tests/smart_kit_tests/handlers/test_handle_respond.py b/tests/smart_kit_tests/handlers/test_handle_respond.py index 02967c98..5cdbc08d 100644 --- a/tests/smart_kit_tests/handlers/test_handle_respond.py +++ b/tests/smart_kit_tests/handlers/test_handle_respond.py @@ -60,7 +60,7 @@ def test_handler_respond_get_action_params(self): self.assertTrue(obj.get_action_params("any data", self.test_user2) == self.callback11_action_params) self.assertTrue(obj.get_action_params(None, self.test_user2) == self.callback11_action_params) - def test_handler_respond_run(self): + async def test_handler_respond_run(self): self.assertIsNotNone(handle_respond.TextPreprocessingResult(self.test_payload.get("message", {}))) self.assertIsNotNone(handle_respond.log_const.KEY_NAME) self.assertIsNotNone(handle_respond.log_const.NORMALIZED_TEXT_VALUE) @@ -68,5 +68,5 @@ def test_handler_respond_run(self): obj1 = handle_respond.HandlerRespond(app_name=self.app_name) obj2 = handle_respond.HandlerRespond(self.app_name, "any action name") with self.assertRaises(KeyError): - obj1.run(self.test_payload, self.test_user1) - self.assertTrue(obj2.run(self.test_payload, self.test_user2) == 10) + await obj1.run(self.test_payload, self.test_user1) + self.assertTrue(await obj2.run(self.test_payload, self.test_user2) == 10) diff --git a/tests/smart_kit_tests/handlers/test_handler_text.py b/tests/smart_kit_tests/handlers/test_handler_text.py index a6ea0906..2f7ea5eb 100644 --- a/tests/smart_kit_tests/handlers/test_handler_text.py +++ b/tests/smart_kit_tests/handlers/test_handler_text.py @@ -37,15 +37,15 @@ def test_handler_text_init(self): self.assertIsNotNone(handler_text.log_const.STARTUP_VALUE) self.assertIsNotNone(obj1.__class__.__name__) - def test_handler_text_handle_base(self): + async def test_handler_text_handle_base(self): obj1 = handler_text.HandlerText(self.app_name, self.test_dialog_manager1) obj2 = handler_text.HandlerText(self.app_name, self.test_dialog_manager2) - self.assertTrue(obj1._handle_base(self.test_text_preprocessing_result, self.test_user) == "TestAnswer") - self.assertTrue(obj2._handle_base(self.test_text_preprocessing_result, self.test_user) == []) + self.assertTrue(await obj1._handle_base(self.test_text_preprocessing_result, self.test_user) == "TestAnswer") + self.assertTrue(await obj2._handle_base(self.test_text_preprocessing_result, self.test_user) == []) - def test_handler_text_run(self): + async def test_handler_text_run(self): self.assertIsNotNone(handler_text.log_const.NORMALIZED_TEXT_VALUE) obj1 = handler_text.HandlerText(self.app_name, self.test_dialog_manager1) obj2 = handler_text.HandlerText(self.app_name, self.test_dialog_manager2) - self.assertTrue(obj1.run(self.test_payload, self.test_user) == "TestAnswer") - self.assertTrue(obj2.run(self.test_payload, self.test_user) == []) + self.assertTrue(await obj1.run(self.test_payload, self.test_user) == "TestAnswer") + self.assertTrue(await obj2.run(self.test_payload, self.test_user) == []) diff --git a/tests/smart_kit_tests/handlers/test_handler_timeout.py b/tests/smart_kit_tests/handlers/test_handler_timeout.py index 44e3c368..6ac6ceee 100644 --- a/tests/smart_kit_tests/handlers/test_handler_timeout.py +++ b/tests/smart_kit_tests/handlers/test_handler_timeout.py @@ -25,8 +25,8 @@ def setUp(self): self.test_user.behaviors.get_callback_action_params = lambda *x, **y: {} self.test_payload = Mock('payload') - def test_handler_timeout(self): + async def test_handler_timeout(self): obj = handler_timeout.HandlerTimeout(self.app_name) self.assertIsNotNone(obj.KAFKA_KEY) self.assertIsNotNone(handler_timeout.log_const.KEY_NAME) - self.assertTrue(obj.run(self.test_payload, self.test_user) == 120) + self.assertTrue(await obj.run(self.test_payload, self.test_user) == 120) diff --git a/tests/smart_kit_tests/models/test_dialogue_manager.py b/tests/smart_kit_tests/models/test_dialogue_manager.py index 7518b6fd..b8334a22 100644 --- a/tests/smart_kit_tests/models/test_dialogue_manager.py +++ b/tests/smart_kit_tests/models/test_dialogue_manager.py @@ -69,7 +69,7 @@ def test_dialogue_manager_found_action(self): with self.assertRaises(TypeError): obj2._nothing_found_action() - def test_dialogue_manager_run(self): + async def test_dialogue_manager_run(self): obj1 = dialogue_manager.DialogueManager({'scenarios': self.test_scenarios, 'external_actions': {'nothing_found_action': self.TestAction}}, self.app_name) @@ -77,18 +77,17 @@ def test_dialogue_manager_run(self): 'external_actions': {}}, self.app_name) # путь по умолчанию без выполнения условий - self.assertTrue(obj1.run(self.test_text_preprocessing_result, self.test_user1) == ("TestNameResult", True)) - self.assertTrue(obj2.run(self.test_text_preprocessing_result, self.test_user1) == ("TestNameResult", True)) + self.assertTrue(await obj1.run(self.test_text_preprocessing_result, self.test_user1) == ("TestNameResult", True)) + self.assertTrue(await obj2.run(self.test_text_preprocessing_result, self.test_user1) == ("TestNameResult", True)) # случай когда срабатоли оба условия self.assertTrue(obj1.run(self.test_text_preprocessing_result, self.test_user2) == ("TestNameResult", True)) # случай, когда 2-е условие не выполнено self.assertTrue(obj2.run(self.test_text_preprocessing_result, self.test_user3) == ('TestNameResult', True)) - def test_dialogue_manager_run_scenario(self): + async def test_dialogue_manager_run_scenario(self): obj = dialogue_manager.DialogueManager({'scenarios': self.test_scenarios, 'external_actions': {'nothing_found_action': self.TestAction}}, self.app_name) - self.assertTrue(obj.run_scenario(1, self.test_text_preprocessing_result, self.test_user1) == "ResultTestName") - self.assertTrue(obj.run_scenario(2, self.test_text_preprocessing_result, self.test_user1) == "TestNameResult") - + self.assertTrue(await obj.run_scenario(1, self.test_text_preprocessing_result, self.test_user1) == "ResultTestName") + self.assertTrue(await obj.run_scenario(2, self.test_text_preprocessing_result, self.test_user1) == "TestNameResult") From 68ba60917941588ff0424c2ba6a79ebc2e29d57c Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 2 Nov 2021 02:57:30 +0300 Subject: [PATCH 027/116] fix run in HandlerText --- smart_kit/handlers/handler_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart_kit/handlers/handler_text.py b/smart_kit/handlers/handler_text.py index 6142d6ab..49c53e43 100644 --- a/smart_kit/handlers/handler_text.py +++ b/smart_kit/handlers/handler_text.py @@ -26,7 +26,7 @@ async def run(self, payload, user): log("text preprocessing result", user, {log_const.KEY_NAME: log_const.NORMALIZED_TEXT_VALUE, "tpr_str": str(text_preprocessing_result.raw)}) - answer = self._handle_base(text_preprocessing_result, user) + answer = await self._handle_base(text_preprocessing_result, user) return answer async def _handle_base(self, text_preprocessing_result, user): From c78ebd4f27cc322710687a503d4f804082588635 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Tue, 2 Nov 2021 09:50:43 +0300 Subject: [PATCH 028/116] make actions async --- core/basic_models/actions/basic_actions.py | 52 +++++------ scenarios/actions/action.py | 104 ++++++++++----------- 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/core/basic_models/actions/basic_actions.py b/core/basic_models/actions/basic_actions.py index b3424146..d0d180a6 100644 --- a/core/basic_models/actions/basic_actions.py +++ b/core/basic_models/actions/basic_actions.py @@ -24,8 +24,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): self.id = id self.version = items.get("version", -1) - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: raise NotImplementedError def on_run_error(self, text_preprocessing_result, user): @@ -50,9 +50,9 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): self.request_type = items.get("request_type") or self.DEFAULT_REQUEST_TYPE self.request_data = items.get("request_data") - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: - super(CommandAction, self).run(user, text_preprocessing_result, params) + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + await super(CommandAction, self).run(user, text_preprocessing_result, params) return None @@ -65,8 +65,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(DoingNothingAction, self).__init__(items, id) self.nodes = items.get("nodes") or {} - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: commands = [Command(self.command, self.nodes, self.id, request_type=self.request_type, request_data=self.request_data)] return commands @@ -97,11 +97,11 @@ def build_requirement(self): def build_internal_item(self): return self._item - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: result = None if self.requirement.check(text_preprocessing_result, user, params): - result = self.internal_item.run(user, text_preprocessing_result, params) + result = await self.internal_item.run(user, text_preprocessing_result, params) return result @@ -133,18 +133,18 @@ def build_items(self): def build_else_item(self): return self._else_item - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: result = None choice_is_made = False for item in self.items: checked = item.requirement.check(text_preprocessing_result, user, params) if checked: - result = item.internal_item.run(user, text_preprocessing_result, params) + result = await item.internal_item.run(user, text_preprocessing_result, params) choice_is_made = True break if not choice_is_made and self._else_item: - result = self.else_item.run(user, text_preprocessing_result, params) + result = await self.else_item.run(user, text_preprocessing_result, params) return result @@ -181,13 +181,13 @@ def build_item(self): def build_else_item(self): return self._else_item - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Optional[Dict[str, Union[str, float, int]]]] = None) -> Optional[List[Command]]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Optional[Dict[str, Union[str, float, int]]]] = None) -> Optional[List[Command]]: result = None if self.requirement.check(text_preprocessing_result, user, params): - result = self.item.run(user, text_preprocessing_result, params) + result = await self.item.run(user, text_preprocessing_result, params) elif self._else_item: - result = self.else_item.run(user, text_preprocessing_result, params) + result = await self.else_item.run(user, text_preprocessing_result, params) return result @@ -204,11 +204,11 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): def build_actions(self): return self._actions - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: commands = [] for action in self.actions: - action_result = action.run(user, text_preprocessing_result, params) + action_result = await action.run(user, text_preprocessing_result, params) if action_result: commands += action_result return commands @@ -224,8 +224,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): self._actions_count = len(items["actions"]) self._last_action_ids_storage = items["last_action_ids_storage"] - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: last_ids = user.last_action_ids[self._last_action_ids_storage] all_indexes = list(range(self._actions_count)) max_last_ids_count = self._actions_count - 1 @@ -235,7 +235,7 @@ def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingRe action_index = random.choice(available_indexes) action = self.actions[action_index] last_ids.add(action_index) - result = action.run(user, text_preprocessing_result, params) + result = await action.run(user, text_preprocessing_result, params) return result @@ -250,8 +250,8 @@ def __init__(self, items, id=None): def build_actions(self): return self._raw_actions - def run(self, user, text_preprocessing_result, params=None): + async def run(self, user, text_preprocessing_result, params=None): pos = random.randint(0, len(self._raw_actions) - 1) action = self.actions[pos] - command_list = action.run(user, text_preprocessing_result, params=params) + command_list = await action.run(user, text_preprocessing_result, params=params) return command_list diff --git a/scenarios/actions/action.py b/scenarios/actions/action.py index 31f33e9d..fafa66ca 100644 --- a/scenarios/actions/action.py +++ b/scenarios/actions/action.py @@ -38,8 +38,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(ClearFormAction, self).__init__(items, id) self.form = items["form"] - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: user.forms.remove_item(self.form) @@ -53,8 +53,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(ClearInnerFormAction, self).__init__(items, id) self.inner_form = items["inner_form"] - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: form = user.forms[self.form] if form: form.forms.remove_item(self.inner_form) @@ -71,8 +71,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): self.form = items["form"] self.field = items["field"] - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: form = user.forms[self.form] form.fields.remove_item(self.field) @@ -88,8 +88,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(RemoveCompositeFormFieldAction, self).__init__(items, id) self.inner_form = items["inner_form"] - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: form = user.forms[self.form] inner_form = form.forms[self.inner_form] inner_form.fields.remove_item(self.field) @@ -102,16 +102,16 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(BreakScenarioAction, self).__init__(items, id) self.scenario_id = items.get("scenario_id") - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: scenario_id = self.scenario_id if self.scenario_id is not None else user.last_scenarios.last_scenario_name user.scenario_models[scenario_id].set_break() class AskAgainAction(Action): - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: last_scenario_id = user.last_scenarios.last_scenario_name scenario = user.descriptions["scenarios"].get(last_scenario_id) return scenario.get_ask_again_question_result(text_preprocessing_result, user, params) @@ -128,8 +128,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): self.behavior = items["behavior"] self.check_scenario = items.get("check_scenario", True) - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: scenario_id = None if self.check_scenario: scenario_id = user.last_scenarios.last_scenario_name @@ -160,15 +160,15 @@ def command_action(self) -> StringAction: def _check(self, user): return not user.behaviors.check_got_saved_id(self.behavior_action.behavior) - def _run(self, user, text_preprocessing_result, params=None): - self.behavior_action.run(user, text_preprocessing_result, params) - command_action_result = self.command_action.run(user, text_preprocessing_result, params) or [] + async def _run(self, user, text_preprocessing_result, params=None): + await self.behavior_action.run(user, text_preprocessing_result, params) + command_action_result = await self.command_action.run(user, text_preprocessing_result, params) or [] return command_action_result - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: if self._check(user): - return self._run(user, text_preprocessing_result, params) + return await self._run(user, text_preprocessing_result, params) class BaseSetVariableAction(Action): @@ -187,8 +187,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): def _set(self, user, value): raise NotImplemented - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: params = user.parametrizer.collect(text_preprocessing_result) try: # if path is wrong, it may fail with UndefinedError @@ -221,7 +221,7 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(SetVariableAction, self).__init__(items, id) self.ttl: int = items.get("ttl") - def _set(self, user, value): + async def _set(self, user, value): user.variables.set(self.key, value, self.ttl) @@ -233,8 +233,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(DeleteVariableAction, self).__init__(items, id) self.key: str = items["key"] - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: user.variables.delete(self.key) @@ -244,8 +244,8 @@ class ClearVariablesAction(Action): def __init__(self, items: Dict[str, Any] = None, id: Optional[str] = None): super(ClearVariablesAction, self).__init__(items, id) - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: user.variables.clear() @@ -271,8 +271,8 @@ def _fill(self, user, data): def _get_data(self, params): return self.template.render(params) - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: params = user.parametrizer.collect(text_preprocessing_result) data = self._get_data(params) self._fill(user, data) @@ -301,8 +301,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(RunScenarioAction, self).__init__(items, id) self.scenario: UnifiedTemplate = UnifiedTemplate(items["scenario"]) - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: if params is None: params = user.parametrizer.collect(text_preprocessing_result) else: @@ -314,8 +314,8 @@ def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult class RunLastScenarioAction(Action): - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: last_scenario_id = user.last_scenarios.last_scenario_name scenario = user.descriptions["scenarios"].get(last_scenario_id) if scenario: @@ -348,20 +348,20 @@ def build_requirement_items(self): def build_else_item(self): return self._else_item - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: result = None choice_is_made = False for scenario, requirement in zip(self._scenarios, self.requirement_items): check_res = requirement.check(text_preprocessing_result, user, params) if check_res: - result = RunScenarioAction(items=scenario).run(user, text_preprocessing_result, params) + result = await RunScenarioAction(items=scenario).run(user, text_preprocessing_result, params) choice_is_made = True break if not choice_is_made and self._else_item: - result = self.else_item.run(user, text_preprocessing_result, params) + result = await self.else_item.run(user, text_preprocessing_result, params) return result @@ -373,8 +373,8 @@ def _clear_scenario(self, user, scenario_id): user.last_scenarios.delete(scenario_id) user.forms.remove_item(scenario.form_type) - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: last_scenario_id = user.last_scenarios.last_scenario_name if last_scenario_id: self._clear_scenario(user, last_scenario_id) @@ -389,8 +389,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(ClearScenarioByIdAction, self).__init__(items, id) self.scenario_id = items.get("scenario_id") - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: if self.scenario_id: self._clear_scenario(user, self.scenario_id) @@ -399,7 +399,7 @@ class ClearCurrentScenarioFormAction(Action): def __init__(self, items, id=None): super().__init__(items, id) - def run(self, user, text_preprocessing_result, params=None): + async def run(self, user, text_preprocessing_result, params=None): last_scenario_id = user.last_scenarios.last_scenario_name if last_scenario_id: user.forms.clear_form(last_scenario_id) @@ -410,7 +410,7 @@ def __init__(self, items, id=None): super().__init__(items, id) self.node_id = items.get('node_id', None) - def run(self, user, text_preprocessing_result, params=None): + async def run(self, user, text_preprocessing_result, params=None): last_scenario_id = user.last_scenarios.last_scenario_name if last_scenario_id: user.scenario_models[last_scenario_id].current_node = self.node_id @@ -430,8 +430,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): for k, v in self.event_content.items(): self.event_content[k] = UnifiedTemplate(v) - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: last_scenario_id = user.last_scenarios.last_scenario_name scenario = user.descriptions["scenarios"].get(last_scenario_id) if scenario: @@ -455,8 +455,8 @@ def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult class EmptyAction(Action): - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: log("%(class_name)s.run: action do nothing.", params={log_const.KEY_NAME: "empty_action", "class_name": self.__class__.__name__}, user=user) return None @@ -464,8 +464,8 @@ def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult class RunScenarioByProjectNameAction(Action): - def run(self, user: User, text_preprocessing_result: TextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: + async def run(self, user: User, text_preprocessing_result: TextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: scenario_id = user.message.project_name scenario = user.descriptions["scenarios"].get(scenario_id) if scenario: @@ -478,8 +478,8 @@ def run(self, user: User, text_preprocessing_result: TextPreprocessingResult, class ProcessBehaviorAction(Action): - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: callback_id = user.message.callback_id log("%(class_name)s.run: got callback_id %(callback_id)s.", @@ -512,7 +512,7 @@ def __init__(self, items, id=None): self.rewrite_saved_messages = items.get("rewrite_saved_messages", False) self._check_scenario: bool = items.get("check_scenario", True) - def _run(self, user, text_preprocessing_result, params=None): + async def _run(self, user, text_preprocessing_result, params=None): action_params = copy.copy(params or {}) From 41a736d0dcf99c5d1c664ce457fa4067f2e2262f Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Tue, 2 Nov 2021 10:10:26 +0300 Subject: [PATCH 029/116] async deep actions --- core/basic_models/actions/counter_actions.py | 20 +++++++------- core/basic_models/actions/external_actions.py | 6 ++--- core/basic_models/actions/string_actions.py | 27 ++++++++++--------- core/basic_models/actions/variable_actions.py | 12 ++++----- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/core/basic_models/actions/counter_actions.py b/core/basic_models/actions/counter_actions.py index 5aeb8b25..c5d4260b 100644 --- a/core/basic_models/actions/counter_actions.py +++ b/core/basic_models/actions/counter_actions.py @@ -18,20 +18,20 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): self.value = items.get("value", 1) self.lifetime = items.get("lifetime") - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: user.counters[self.key].inc(self.value, self.lifetime) class CounterDecrementAction(CounterIncrementAction): - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: user.counters[self.key].dec(-self.value, self.lifetime) class CounterClearAction(CounterIncrementAction): - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: user.counters.clear(self.key) @@ -48,8 +48,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): self.reset_time = items.get("reset_time", False) self.time_shift = items.get("time_shift", 0) - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: user.counters[self.key].set(self.value, self.reset_time, self.time_shift) @@ -61,7 +61,7 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): self.reset_time = items.get("reset_time", False) self.time_shift = items.get("time_shift", 0) - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: value = user.counters[self.src].value user.counters[self.dst].set(value, self.reset_time, self.time_shift) diff --git a/core/basic_models/actions/external_actions.py b/core/basic_models/actions/external_actions.py index b4cddca0..e96495c0 100644 --- a/core/basic_models/actions/external_actions.py +++ b/core/basic_models/actions/external_actions.py @@ -24,8 +24,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(ExternalAction, self).__init__(items, id) self._action_key = items["action"] - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: action = user.descriptions["external_actions"][self._action_key] - commands = action.run(user, text_preprocessing_result, params) + commands = await action.run(user, text_preprocessing_result, params) return commands diff --git a/core/basic_models/actions/string_actions.py b/core/basic_models/actions/string_actions.py index 3c5f4a93..9ac2b150 100644 --- a/core/basic_models/actions/string_actions.py +++ b/core/basic_models/actions/string_actions.py @@ -80,8 +80,8 @@ def _get_rendered_tree_recursive(self, value, params, no_empty=False): result = value return result - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: raise NotImplementedError @@ -101,11 +101,12 @@ class StringAction(NodeAction): } } """ + def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(StringAction, self).__init__(items, id) - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: # Example: Command("ANSWER_TO_USER", {"answer": {"key1": "string1", "keyN": "stringN"}}) command_params = dict() params = params or {} @@ -141,8 +142,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(AfinaAnswerAction, self).__init__(items, id) self.command: str = ANSWER_TO_USER - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: params = user.parametrizer.collect(text_preprocessing_result, filter_params={"command": self.command}) answer_params = dict() result = [] @@ -212,13 +213,15 @@ class SDKAnswer(NodeAction): карточки на андроиде требуют sdk_version не ниже "20.03.0.0" """ INDEX_WILDCARD = "*index*" - RANDOM_PATH = [['items', INDEX_WILDCARD, 'bubble', 'text'], ['pronounceText'], ['suggestions', 'buttons', INDEX_WILDCARD, 'title']] + RANDOM_PATH = [['items', INDEX_WILDCARD, 'bubble', 'text'], ['pronounceText'], + ['suggestions', 'buttons', INDEX_WILDCARD, 'title']] def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(SDKAnswer, self).__init__(items, id) self.command: str = ANSWER_TO_USER if self._nodes == {}: - self._nodes = {i: items.get(i) for i in items if i not in ['random_paths', 'same_ans', 'type', 'support_templates', 'no_empty_nodes']} + self._nodes = {i: items.get(i) for i in items if + i not in ['random_paths', 'same_ans', 'type', 'support_templates', 'no_empty_nodes']} # функция идет по RANDOM_PATH, числа в нем считает индексами массива, # INDEX_WILDCARD - произвольным индексом массива, прочее - ключами словаря @@ -239,8 +242,8 @@ def random_by_path(self, input_dict, nested_key): return last_dict[k] = random.choice(last_dict[k]) - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: result = [] params = user.parametrizer.collect(text_preprocessing_result, filter_params={"command": self.command}) rendered = self._get_rendered_tree(self.nodes, params, self.no_empty_nodes) @@ -394,8 +397,8 @@ def build_suggests(self): def build_root(self): return self._root - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: result = [] params = user.parametrizer.collect(text_preprocessing_result, filter_params={self.COMMAND: self.command}) diff --git a/core/basic_models/actions/variable_actions.py b/core/basic_models/actions/variable_actions.py index 580d4c56..4bf79e18 100644 --- a/core/basic_models/actions/variable_actions.py +++ b/core/basic_models/actions/variable_actions.py @@ -32,8 +32,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): value: str = items["value"] self.template: UnifiedTemplate = UnifiedTemplate(value) - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: params = user.parametrizer.collect(text_preprocessing_result) try: # if path is wrong, it may fail with UndefinedError @@ -61,8 +61,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(DeleteVariableAction, self).__init__(items, id) self.key: str = items["key"] - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: user.variables.delete(self.key) @@ -72,6 +72,6 @@ class ClearVariablesAction(Action): def __init__(self, items: Dict[str, Any] = None, id: Optional[str] = None): super(ClearVariablesAction, self).__init__(items, id) - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: user.variables.clear() From 2b49e3e3019d4b19eddf92dd06a5bceb26a1950f Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Wed, 3 Nov 2021 09:16:06 +0300 Subject: [PATCH 030/116] make half of runs of actions async --- core/basic_models/scenarios/base_scenario.py | 4 +- scenarios/behaviors/behaviors.py | 17 ++- .../field/composite_fillers.py | 8 +- smart_kit/action/http.py | 10 +- smart_kit/handlers/handle_close_app.py | 2 +- smart_kit/handlers/handle_respond.py | 2 +- smart_kit/models/dialogue_manager.py | 2 +- smart_kit/models/smartapp_model.py | 4 +- .../system_answers/nothing_found_action.py | 6 +- .../app/basic_entities/actions.py-tpl | 4 +- .../action_test/test_action.py | 108 +++++++++++------ .../action_test/test_random_action.py | 7 +- .../actions_test/test_action.py | 114 ++++++++++++------ 13 files changed, 185 insertions(+), 103 deletions(-) diff --git a/core/basic_models/scenarios/base_scenario.py b/core/basic_models/scenarios/base_scenario.py index a72ee76a..09d1023e 100644 --- a/core/basic_models/scenarios/base_scenario.py +++ b/core/basic_models/scenarios/base_scenario.py @@ -62,7 +62,7 @@ def get_no_commands_action(self, user, text_preprocessing_result, params: Dict[s scenarios_log_const.CHOSEN_ACTION_VALUE: self._empty_answer} log(scenarios_log_const.CHOSEN_ACTION_MESSAGE, user, log_params) try: - empty_answer = self.empty_answer.run(user, text_preprocessing_result, params) or [] + empty_answer = await self.empty_answer.run(user, text_preprocessing_result, params) or [] except KeyError: log_params = {log_const.KEY_NAME: scenarios_log_const.CHOSEN_ACTION_VALUE} log("Scenario has empty answer, but empty_answer action isn't defined", @@ -74,7 +74,7 @@ def get_action_results(self, user, text_preprocessing_result, actions: List[Action], params: Dict[str, Any] = None) -> List[Command]: results = [] for action in actions: - result = action.run(user, text_preprocessing_result, params) + result = await action.run(user, text_preprocessing_result, params) log_params = self._log_params() log_params["class"] = action.__class__.__name__ log("called action: %(class)s", user, log_params) diff --git a/scenarios/behaviors/behaviors.py b/scenarios/behaviors/behaviors.py index 27de1d06..580e8f28 100644 --- a/scenarios/behaviors/behaviors.py +++ b/scenarios/behaviors/behaviors.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio from time import time from collections import namedtuple from typing import Dict @@ -133,7 +134,9 @@ def success(self, callback_id: str): callback_action_params, ) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - result = behavior.success_action.run(self._user, text_preprocessing_result, callback_action_params) + loop = asyncio.get_running_loop() + result = loop.run_until_complete( + behavior.success_action.run(self._user, text_preprocessing_result, callback_action_params)) self._delete(callback_id) return result @@ -152,7 +155,9 @@ def fail(self, callback_id: str): smart_kit_metrics.counter_behavior_fail, "fail", callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - result = behavior.fail_action.run(self._user, text_preprocessing_result, callback_action_params) + loop = asyncio.get_running_loop() + result = loop.run_until_complete( + behavior.fail_action.run(self._user, text_preprocessing_result, callback_action_params)) self._delete(callback_id) return result @@ -171,7 +176,9 @@ def timeout(self, callback_id: str): smart_kit_metrics.counter_behavior_timeout, "timeout", callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - result = behavior.timeout_action.run(self._user, text_preprocessing_result, callback_action_params) + loop = asyncio.get_running_loop() + result = loop.run_until_complete( + behavior.timeout_action.run(self._user, text_preprocessing_result, callback_action_params)) self._delete(callback_id) return result @@ -190,7 +197,9 @@ def misstate(self, callback_id: str): smart_kit_metrics.counter_behavior_misstate, "misstate", callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - result = behavior.misstate_action.run(self._user, text_preprocessing_result, callback_action_params) + loop = asyncio.get_running_loop() + result = loop.run_until_complete( + behavior.misstate_action.run(self._user, text_preprocessing_result, callback_action_params)) self._delete(callback_id) return result diff --git a/scenarios/scenario_models/field/composite_fillers.py b/scenarios/scenario_models/field/composite_fillers.py index 76210d58..c07f49ab 100644 --- a/scenarios/scenario_models/field/composite_fillers.py +++ b/scenarios/scenario_models/field/composite_fillers.py @@ -29,8 +29,8 @@ def on_extract_error(self, text_preprocessing_result, user, params=None): @exc_handler(on_error_obj_method_name="on_extract_error") def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, - user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: - return self.run(user, text_preprocessing_result, params) + user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: + return await self.run(user, text_preprocessing_result, params) class ChoiceFiller(ChoiceAction): @@ -62,7 +62,7 @@ def on_extract_error(self, text_preprocessing_result, user, params=None): @exc_handler(on_error_obj_method_name="on_extract_error") def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: - return self.run(user, text_preprocessing_result, params) + return await self.run(user, text_preprocessing_result, params) class ElseFiller(ElseAction): @@ -94,4 +94,4 @@ def on_extract_error(self, text_preprocessing_result, user, params=None): @exc_handler(on_error_obj_method_name="on_extract_error") def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: - return self.run(user, text_preprocessing_result, params) + return await self.run(user, text_preprocessing_result, params) diff --git a/smart_kit/action/http.py b/smart_kit/action/http.py index 96e20540..94bc8de2 100644 --- a/smart_kit/action/http.py +++ b/smart_kit/action/http.py @@ -24,8 +24,8 @@ def _check_headers_validity(headers: Dict[str, Any]) -> Dict[str, str]: headers[header_name] = str(header_value) return headers - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: behavior_description = user.descriptions["behaviors"][self.behavior] request_params = self.params @@ -47,8 +47,8 @@ def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingRe with requests.request(**request_parameters) as response: response.raise_for_status() user.variables.set(self.store, response.json()) - return behavior_description.success_action.run(user, text_preprocessing_result, None) + return await behavior_description.success_action.run(user, text_preprocessing_result, None) except requests.exceptions.Timeout: - return behavior_description.timeout_action.run(user, text_preprocessing_result, None) + return await behavior_description.timeout_action.run(user, text_preprocessing_result, None) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError): - return behavior_description.fail_action.run(user, text_preprocessing_result, None) + return await behavior_description.fail_action.run(user, text_preprocessing_result, None) diff --git a/smart_kit/handlers/handle_close_app.py b/smart_kit/handlers/handle_close_app.py index d13c6a62..fe803c8c 100644 --- a/smart_kit/handlers/handle_close_app.py +++ b/smart_kit/handlers/handle_close_app.py @@ -18,5 +18,5 @@ async def run(self, payload, user): log_const.KEY_NAME: "HandlerCloseApp", "tpr_str": str(text_preprocessing_result.raw) } - self._clear_current_scenario.run(user, text_preprocessing_result) + await self._clear_current_scenario.run(user, text_preprocessing_result) log("HandlerCloseApp with text preprocessing result", user, params) diff --git a/smart_kit/handlers/handle_respond.py b/smart_kit/handlers/handle_respond.py index 3174075a..edb50837 100644 --- a/smart_kit/handlers/handle_respond.py +++ b/smart_kit/handlers/handle_respond.py @@ -53,7 +53,7 @@ async def run(self, payload, user): log("text preprocessing result: '%(normalized_text)s'", user, params, level="DEBUG") action = user.descriptions["external_actions"][action_name] - return action.run(user, text_preprocessing_result, action_params) + return await action.run(user, text_preprocessing_result, action_params) @staticmethod def get_processing_time(user): diff --git a/smart_kit/models/dialogue_manager.py b/smart_kit/models/dialogue_manager.py index 5c88cc00..92ac38f4 100644 --- a/smart_kit/models/dialogue_manager.py +++ b/smart_kit/models/dialogue_manager.py @@ -47,7 +47,7 @@ async def run(self, text_preprocessing_result, user): smart_kit_metrics.counter_nothing_found(self.app_name, scenario_key, user) - return self._nothing_found_action.run(user, text_preprocessing_result), False + return await self._nothing_found_action.run(user, text_preprocessing_result), False return await self.run_scenario(scenario_key, text_preprocessing_result, user), True diff --git a/smart_kit/models/smartapp_model.py b/smart_kit/models/smartapp_model.py index c9afb824..b2f320e6 100644 --- a/smart_kit/models/smartapp_model.py +++ b/smart_kit/models/smartapp_model.py @@ -82,6 +82,6 @@ def on_answer_error(self, message, user): if user.settings["template_settings"].get("debug_info"): set_debug_info(self.app_name, callback_action_params, error) exception_action = user.descriptions["external_actions"]["exception_action"] - commands = exception_action.run(user=user, text_preprocessing_result=None, - params=callback_action_params) + commands = await exception_action.run(user=user, text_preprocessing_result=None, + params=callback_action_params) return commands diff --git a/smart_kit/system_answers/nothing_found_action.py b/smart_kit/system_answers/nothing_found_action.py index e1986337..9c0947ac 100644 --- a/smart_kit/system_answers/nothing_found_action.py +++ b/smart_kit/system_answers/nothing_found_action.py @@ -18,6 +18,6 @@ def __init__(self, items: Dict[str, Any] = None, id: Optional[str] = None): super(NothingFoundAction, self).__init__(items, id) self._action = StringAction({"command": NOTHING_FOUND}) - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: - return self._action.run(user, text_preprocessing_result, params=params) + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + return await self._action.run(user, text_preprocessing_result, params=params) diff --git a/smart_kit/template/app/basic_entities/actions.py-tpl b/smart_kit/template/app/basic_entities/actions.py-tpl index cceafe97..9a67afad 100644 --- a/smart_kit/template/app/basic_entities/actions.py-tpl +++ b/smart_kit/template/app/basic_entities/actions.py-tpl @@ -18,7 +18,7 @@ class CustomAction(Action): items = items or {} self.test_param = items.get("test_param") - def run(self, user: User, text_preprocessing_result: TextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: + async def run(self, user: User, text_preprocessing_result: TextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: print("Test Action") return None diff --git a/tests/core_tests/basic_scenario_models_test/action_test/test_action.py b/tests/core_tests/basic_scenario_models_test/action_test/test_action.py index 9706182d..ccad7f34 100644 --- a/tests/core_tests/basic_scenario_models_test/action_test/test_action.py +++ b/tests/core_tests/basic_scenario_models_test/action_test/test_action.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio import unittest from unittest.mock import Mock, MagicMock, patch @@ -39,7 +40,7 @@ def __init__(self, items=None): items = items or {} self.result = items.get("result") - def run(self, user, text_preprocessing_result, params=None): + async def run(self, user, text_preprocessing_result, params=None): return self.result or ["test action run"] @@ -49,7 +50,7 @@ def __init__(self, items=None): self.result = items.get("result") self.done = False - def run(self, user, text_preprocessing_result, params=None): + async def run(self, user, text_preprocessing_result, params=None): self.done = True @@ -89,7 +90,8 @@ def test_base(self): items = {"nodes": "test"} action = Action(items) try: - action.run(None, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(None, None)) result = False except NotImplementedError: result = True @@ -100,12 +102,14 @@ def test_external(self): action = ExternalAction(items) user = Mock() user.descriptions = {"external_actions": {"test_action_key": MockAction()}} - self.assertEqual(action.run(user, None), ["test action run"]) + loop = asyncio.get_event_loop() + self.assertEqual(loop.run_until_complete(action.run(user, None)), ["test action run"]) def test_doing_nothing_action(self): items = {"nodes": {"answer": "test"}, "command": "test_name"} action = DoingNothingAction(items) - result = action.run(None, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(None, None)) self.assertIsInstance(result, list) command = result[0] self.assertIsInstance(command, Command) @@ -121,10 +125,11 @@ def test_requirement_action(self): action = RequirementAction(items) self.assertIsInstance(action.requirement, MockRequirement) self.assertIsInstance(action.internal_item, MockAction) - self.assertEqual(action.run(None, None), ["test action run"]) + loop = asyncio.get_event_loop() + self.assertEqual(loop.run_until_complete(action.run(None, None)), ["test action run"]) items = {"requirement": {"type": "test", "result": False}, "action": {"type": "test"}} action = RequirementAction(items) - result = action.run(None, None) + result = loop.run_until_complete(action.run(None, None)) self.assertIsNone(result) def test_requirement_choice(self): @@ -135,7 +140,8 @@ def test_requirement_choice(self): choice_action = ChoiceAction(items) self.assertIsInstance(choice_action.items, list) self.assertIsInstance(choice_action.items[0], RequirementAction) - result = choice_action.run(None, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(choice_action.run(None, None)) self.assertEqual(result, "action2") def test_requirement_choice_else(self): @@ -149,7 +155,8 @@ def test_requirement_choice_else(self): choice_action = ChoiceAction(items) self.assertIsInstance(choice_action.items, list) self.assertIsInstance(choice_action.items[0], RequirementAction) - result = choice_action.run(None, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(choice_action.run(None, None)) self.assertEqual(result, "action3") def test_string_action(self): @@ -164,7 +171,8 @@ def test_string_action(self): "nodes": {"item": "template", "params": "{{params}}"}} action = StringAction(items) - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertEqual(expected[0].name, result[0].name) self.assertEqual(expected[0].payload, result[0].payload) @@ -180,7 +188,8 @@ def test_else_action_if(self): "else_action": {"type": "test", "result": "else_action"} } action = ElseAction(items) - self.assertEqual(action.run(user, None), "main_action") + loop = asyncio.get_event_loop() + self.assertEqual(loop.run_until_complete(action.run(user, None)), "main_action") def test_else_action_else(self): registered_factories[Requirement] = requirement_factory @@ -194,7 +203,8 @@ def test_else_action_else(self): "else_action": {"type": "test", "result": "else_action"} } action = ElseAction(items) - self.assertEqual(action.run(user, None), "else_action") + loop = asyncio.get_event_loop() + self.assertEqual(loop.run_until_complete(action.run(user, None)), "else_action") def test_else_action_no_else_if(self): registered_factories[Requirement] = requirement_factory @@ -207,7 +217,8 @@ def test_else_action_no_else_if(self): "action": {"type": "test", "result": "main_action"}, } action = ElseAction(items) - self.assertEqual(action.run(user, None), "main_action") + loop = asyncio.get_event_loop() + self.assertEqual(loop.run_until_complete(action.run(user, None)), "main_action") def test_else_action_no_else_else(self): registered_factories[Requirement] = requirement_factory @@ -220,7 +231,8 @@ def test_else_action_no_else_else(self): "action": {"type": "test", "result": "main_action"}, } action = ElseAction(items) - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertIsNone(result) def test_composite_action(self): @@ -234,7 +246,8 @@ def test_composite_action(self): ] } action = CompositeAction(items) - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertEqual(['test action run', 'test action run'], result) def test_node_action_support_templates(self): @@ -261,7 +274,8 @@ def test_node_action_support_templates(self): self.assertIsInstance(template, UnifiedTemplate) user = MagicMock() user.parametrizer = MockSimpleParametrizer(user, {"data": params}) - output = action.run(user=user, text_preprocessing_result=None)[0].payload["answer"] + loop = asyncio.get_event_loop() + output = loop.run_until_complete(action.run(user=user, text_preprocessing_result=None))[0].payload["answer"] self.assertEqual(output, expected) def test_string_action_support_templates(self): @@ -286,7 +300,8 @@ def test_string_action_support_templates(self): action = StringAction(items) user = MagicMock() user.parametrizer = MockSimpleParametrizer(user, {"data": params}) - output = action.run(user=user, text_preprocessing_result=None)[0].payload + loop = asyncio.get_event_loop() + output = loop.run_until_complete(action.run(user=user, text_preprocessing_result=None))[0].payload self.assertEqual(output, expected) @@ -306,13 +321,15 @@ def setUp(self): def test_run_available_indexes(self): self.user.last_action_ids["last_action_ids_storage"].get_list.side_effect = [[0]] - result = self.action.run(self.user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(self.action.run(self.user, None)) self.user.last_action_ids["last_action_ids_storage"].add.assert_called_once() self.assertEqual(result, self.expected1) def test_run_no_available_indexes(self): self.user.last_action_ids["last_action_ids_storage"].get_list.side_effect = [[0, 1]] - result = self.action.run(self.user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(self.action.run(self.user, None)) self.assertEqual(result, self.expected) @@ -324,7 +341,8 @@ def test_run(self): user.counters = {"test": counter} items = {"key": "test"} action = CounterIncrementAction(items) - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) user.counters["test"].inc.assert_called_once() @@ -336,7 +354,8 @@ def test_run(self): user.counters = {"test": counter} items = {"key": "test"} action = CounterDecrementAction(items) - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) user.counters["test"].dec.assert_called_once() @@ -347,7 +366,8 @@ def test_run(self): user.counters.inc = Mock() items = {"key": "test"} action = CounterClearAction(items) - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) user.counters.clear.assert_called_once() @@ -360,7 +380,8 @@ def test_run(self): user.counters = counters items = {"key": "test"} action = CounterSetAction(items) - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) user.counters["test"].set.assert_called_once() @@ -373,7 +394,8 @@ def test_run(self): user.counters = {"src": counter_src, "dst": counter_dst} items = {"source": "src", "destination": "dst"} action = CounterCopyAction(items) - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) user.counters["dst"].set.assert_called_once_with(user.counters["src"].value, action.reset_time, action.time_shift) @@ -390,8 +412,8 @@ def test_typical_answer(self): } } action = AfinaAnswerAction(items) - - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertEqual(expected[0]._name, result[0].name) self.assertEqual(expected[0].raw, result[0].raw) @@ -410,8 +432,8 @@ def test_typical_answer_with_other(self): } } action = AfinaAnswerAction(items) - - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertEqual(expected[0]._name, result[0].name) self.assertEqual(expected[0].raw, result[0].raw) @@ -424,7 +446,8 @@ def test_typical_answer_with_pers_info(self): user.message.payload = {"personInfo": {"name": "Ivan Ivanov"}} items = {"nodes": {"answer": ["{{payload.personInfo.name}}"]}} action = AfinaAnswerAction(items) - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertEqual(expected[0]._name, result[0].name) self.assertEqual(expected[0].raw, result[0].raw) @@ -436,7 +459,8 @@ def test_items_empty(self): user.descriptions = {"render_templates": template} items = None action = AfinaAnswerAction(items) - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertEqual(result, []) def test__items_empty_dict(self): @@ -447,7 +471,8 @@ def test__items_empty_dict(self): user.descriptions = {"render_templates": template} items = {} action = AfinaAnswerAction(items) - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertEqual(result, []) @@ -501,9 +526,10 @@ def test_typical_answer(self): exp3 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText1', 'items': [{'bubble': {'text': 'Text1'}}, {'card': {'type': 'simple_list', 'header': '1 доллар США ', 'items': [{'title': 'Купить', 'body': '67.73 RUR'}, {'title': 'Продать', 'body': '64.56 RUR'}], 'footer': 'Ivan Ivanov Сбербанк Онлайн на сегодня 17:53 при обмене до 1000 USD'}}], 'suggestions': {'buttons': [{'title': 'Отделения', 'action': {'text': 'Где ближайщие отделения сбера?', 'type': 'text'}}]}}}" exp4 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText1', 'items': [{'bubble': {'text': 'Text2'}}, {'card': {'type': 'simple_list', 'header': '1 доллар США ', 'items': [{'title': 'Купить', 'body': '67.73 RUR'}, {'title': 'Продать', 'body': '64.56 RUR'}], 'footer': 'Ivan Ivanov Сбербанк Онлайн на сегодня 17:53 при обмене до 1000 USD'}}], 'suggestions': {'buttons': [{'title': 'Отделения', 'action': {'text': 'Где ближайщие отделения сбера?', 'type': 'text'}}]}}}" expect_arr = [exp1, exp2, exp3, exp4] + loop = asyncio.get_event_loop() for i in range(10): action = SDKAnswer(items) - result = action.run(user, None) + result = loop.run_until_complete(action.run(user, None)) self.assertEqual("ANSWER_TO_USER", result[0].name) self.assertTrue(str(result[0].raw) in expect_arr) @@ -524,9 +550,10 @@ def test_typical_answer_without_items(self): exp3 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText2'}}" exp4 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText2'}}" exp_list = [exp1, exp2, exp3, exp4] + loop = asyncio.get_event_loop() for i in range(10): action = SDKAnswer(items) - result = action.run(user, None) + result = loop.run_until_complete(action.run(user, None)) self.assertEqual("ANSWER_TO_USER", result[0].name) self.assertTrue(str(result[0].raw) in exp_list) @@ -562,9 +589,10 @@ def test_typical_answer_without_nodes(self): exp3 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText1', 'suggestions': {'buttons': [{'title': 'отделения2', 'action': {'text': 'отделения', 'type': 'text'}}, {'title': 'кредит1', 'action': {'text': 'кредит', 'type': 'text'}}]}}}" exp4 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText1', 'suggestions': {'buttons': [{'title': 'отделения2', 'action': {'text': 'отделения', 'type': 'text'}}, {'title': 'кредит2', 'action': {'text': 'кредит', 'type': 'text'}}]}}}" expect_arr = [exp1, exp2, exp3, exp4] + loop = asyncio.get_event_loop() for i in range(10): action = SDKAnswer(items) - result = action.run(user, None) + result = loop.run_until_complete(action.run(user, None)) self.assertEqual("ANSWER_TO_USER", result[0].name) self.assertTrue(str(result[0].raw) in expect_arr) @@ -655,8 +683,9 @@ def test_SDKItemAnswer_full(self): exp2 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'items': [{'bubble': {'text': 'Ivan Ivanov', 'markdown': False}}, {'card': {'cards_params': 'a lot of params'}}], 'suggestions': {'buttons': [{'title': 'p1', 'action': {'text': 'Ivan Ivanov', 'type': 'text'}}, {'title': 'p1', 'action': {'text': 'Ivan Ivanov', 'type': 'text'}}, {'title': 'p1', 'action': {'deep_link': 'www.ww.w', 'type': 'deep_link'}}]}, 'pronounceText': 'p1'}}" action = SDKAnswerToUser(items) + loop = asyncio.get_event_loop() for i in range(3): - result = action.run(user, None) + result = loop.run_until_complete(action.run(user, None)) self.assertTrue(str(result[0].raw) in [exp1, exp2]) def test_SDKItemAnswer_root(self): @@ -704,8 +733,9 @@ def test_SDKItemAnswer_root(self): exp2 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'p2'}}" action = SDKAnswerToUser(items) + loop = asyncio.get_event_loop() for i in range(3): - result = action.run(user, None) + result = loop.run_until_complete(action.run(user, None)) self.assertTrue(str(result[0].raw) in [exp1, exp2]) def test_SDKItemAnswer_simple(self): @@ -726,7 +756,8 @@ def test_SDKItemAnswer_simple(self): ] } action = SDKAnswerToUser(items) - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertDictEqual(result[0].raw, {'messageName': 'ANSWER_TO_USER', 'payload': {'items': [{'bubble': {'text': '42', 'markdown': True}}]}}) def test_SDKItemAnswer_suggestions_template(self): @@ -748,7 +779,8 @@ def test_SDKItemAnswer_suggestions_template(self): } } action = SDKAnswerToUser(items) - result = action.run(user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, None)) self.assertDictEqual( result[0].raw, { diff --git a/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py b/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py index 22d03a5e..4155a8ef 100644 --- a/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py +++ b/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py @@ -1,3 +1,4 @@ +import asyncio from unittest import TestCase from core.basic_models.actions.basic_actions import Action, action_factory, actions, DoingNothingAction, RandomAction @@ -32,7 +33,8 @@ def test_1(self): ] } action = RandomAction(items, 5) - result = action.run(None, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(None, None)) self.assertIsNotNone(result) def test_2(self): @@ -48,5 +50,6 @@ def test_2(self): ] } action = RandomAction(items, 5) - result = action.run(None, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(None, None)) self.assertIsNotNone(result) diff --git a/tests/scenarios_tests/actions_test/test_action.py b/tests/scenarios_tests/actions_test/test_action.py index c041760e..72dac6d3 100644 --- a/tests/scenarios_tests/actions_test/test_action.py +++ b/tests/scenarios_tests/actions_test/test_action.py @@ -1,3 +1,4 @@ +import asyncio import unittest from typing import Dict, Any, Union, Optional from unittest.mock import MagicMock, Mock, ANY @@ -31,7 +32,7 @@ def __init__(self, items=None): self.items = items or {} self.result = items.get("result") - def run(self, user, text_preprocessing_result, params=None): + async def run(self, user, text_preprocessing_result, params=None): return self.result or ["test action run"] @@ -56,7 +57,8 @@ class ClearFormIdActionTest(unittest.TestCase): def test_run(self): action = ClearFormAction({"form": "form"}) user = MagicMock() - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) user.forms.remove_item.assert_called_once_with("form") @@ -65,7 +67,8 @@ def test_run(self): action = ClearInnerFormAction({"form": "form", "inner_form": "inner_form"}) user, form = MagicMock(), MagicMock() user.forms.__getitem__.return_value = form - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) form.forms.remove_item.assert_called_once_with("inner_form") @@ -77,7 +80,8 @@ def test_run_1(self): scenario_model = MagicMock() scenario_model.set_break = Mock(return_value=None) user.scenario_models = {scenario_id: scenario_model} - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) user.scenario_models[scenario_id].set_break.assert_called_once() def test_run_2(self): @@ -88,7 +92,8 @@ def test_run_2(self): scenario_model = MagicMock() scenario_model.set_break = Mock(return_value=None) user.scenario_models = {scenario_id: scenario_model} - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) user.scenario_models[scenario_id].set_break.assert_called_once() @@ -103,7 +108,8 @@ def test_run(self): scenarios = {last_scenario_name: scenario} user.descriptions = {"scenarios": scenarios} action = AskAgainAction(items) - tesult = action.run(user, None) + loop = asyncio.get_event_loop() + tesult = loop.run_until_complete(action.run(user, None)) self.assertEqual(tesult, "test_result") @@ -112,7 +118,8 @@ def test_run(self): action = RemoveFormFieldAction({"form": "form", "field": "field"}) user, form = MagicMock(), MagicMock() user.forms.__getitem__.return_value = form - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) form.fields.remove_item.assert_called_once_with("field") @@ -122,7 +129,8 @@ def test_run(self): user, inner_form, form = MagicMock(), MagicMock(), MagicMock() form.forms.__getitem__.return_value = inner_form user.forms.__getitem__.return_value = form - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) inner_form.fields.remove_item.assert_called_once_with("field") @@ -145,7 +153,8 @@ def test_save_behavior_scenario_name(self): action = SaveBehaviorAction(data) tpr = Mock() tpr_raw = tpr.raw - action.run(self.user, tpr) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, tpr)) self.user.behaviors.add.assert_called_once_with(self.user.message.generate_new_callback_id(), "test", self.user.last_scenarios.last_scenario_name, tpr_raw, action_params=None) @@ -158,7 +167,8 @@ def test_save_behavior_without_scenario_name(self): action = SaveBehaviorAction(data) text_preprocessing_result_raw = Mock() text_preprocessing_result = Mock(raw=text_preprocessing_result_raw) - action.run(self.user, text_preprocessing_result, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, text_preprocessing_result, None)) self.user.behaviors.add.assert_called_once_with(self.user.message.generate_new_callback_id(), "test", None, text_preprocessing_result_raw, action_params=None) @@ -186,7 +196,8 @@ def test_action_1(self): action = SelfServiceActionWithState(data) text_preprocessing_result_raw = Mock() text_preprocessing_result = Mock(raw=text_preprocessing_result_raw) - result = action.run(self.user, text_preprocessing_result, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(self.user, text_preprocessing_result, None)) behavior.check_got_saved_id.assert_called_once() behavior.add.assert_called_once() self.assertEqual(result[0].name, "cmd_id") @@ -202,7 +213,8 @@ def test_action_2(self): self.user.behaviors = behavior behavior.check_got_saved_id = Mock(return_value=True) action = SelfServiceActionWithState(data) - result = action.run(self.user, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(self.user, None)) behavior.add.assert_not_called() self.assertIsNone(result) @@ -231,7 +243,8 @@ def test_action_3(self): action = SelfServiceActionWithState(data) text_preprocessing_result_raw = Mock() text_preprocessing_result = Mock(raw=text_preprocessing_result_raw) - result = action.run(self.user, text_preprocessing_result, None) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(self.user, text_preprocessing_result, None)) behavior.check_got_saved_id.assert_called_once() behavior.add.assert_called_once() self.assertEqual(result[0].name, "cmd_id") @@ -258,19 +271,22 @@ def setUp(self): def test_action(self): action = SetVariableAction({"key": "some_key", "value": "some_value"}) - action.run(self.user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, None)) self.user.variables.set.assert_called_with("some_key", "some_value", None) def test_action_jinja_key_default(self): self.user.message.payload = {"some_value": "some_value_test"} action = SetVariableAction({"key": "some_key", "value": "{{payload.some_value}}"}) - action.run(self.user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, None)) self.user.variables.set.assert_called_with("some_key", "some_value_test", None) def test_action_jinja_no_key(self): self.user.message.payload = {"some_value": "some_value_test"} action = SetVariableAction({"key": "some_key", "value": "{{payload.no_key}}"}) - action.run(self.user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, None)) self.user.variables.set.assert_called_with("some_key", "", None) @@ -288,7 +304,8 @@ def setUp(self): def test_action(self): action = DeleteVariableAction({"key": "some_key_1"}) - action.run(self.user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, None)) self.user.variables.delete.assert_called_with("some_key_1") @@ -310,7 +327,8 @@ def setUp(self): def test_action(self): action = ClearVariablesAction() - action.run(self.user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, None)) self.user.variables.clear.assert_called_with() @@ -326,7 +344,8 @@ def test_fill_field(self): field = Mock() field.fill = Mock() user.forms["test_form"].fields = {"test_field": field} - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) field.fill.assert_called_once_with(params["test_field"]) @@ -346,7 +365,8 @@ def test_fill_field(self): field = Mock() field.fill = Mock() user.forms["test_form"].forms["test_internal_form"].fields = {"test_field": field} - action.run(user, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(user, None)) field.fill.assert_called_once_with(params["test_field"]) @@ -359,7 +379,8 @@ def test_scenario_action(self): scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"test": scen}} - result = action.run(user, Mock()) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, Mock())) self.assertEqual(result, scen_result) def test_scenario_action_with_jinja_good(self): @@ -373,7 +394,8 @@ def test_scenario_action_with_jinja_good(self): scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"ANNA.pipeline.scenario": scen}} - result = action.run(user, Mock()) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, Mock())) self.assertEqual(result, scen_result) def test_scenario_action_no_scenario(self): @@ -384,7 +406,8 @@ def test_scenario_action_no_scenario(self): scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"next_scenario": scen}} - result = action.run(user, Mock()) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, Mock())) self.assertEqual(result, None) def test_scenario_action_without_jinja(self): @@ -395,7 +418,8 @@ def test_scenario_action_without_jinja(self): scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"next_scenario": scen}} - result = action.run(user, Mock()) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, Mock())) self.assertEqual(result, scen_result) @@ -412,7 +436,8 @@ def test_scenario_action(self): last_scenario_name = "test" user.last_scenarios.scenarios_names = [last_scenario_name] user.last_scenarios.last_scenario_name = last_scenario_name - result = action.run(user, Mock()) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, Mock())) self.assertEqual(result, scen_result) @@ -428,7 +453,8 @@ def mock_and_perform_action(test_items: Dict[str, Any], expected_result: Optiona scen.run.return_value = expected_result if expected_scen: user.descriptions = {"scenarios": {expected_scen: scen}} - return action.run(user, Mock()) + loop = asyncio.get_event_loop() + return loop.run_until_complete(action.run(user, Mock())) def test_choice_scenario_action(self): # Проверяем, что запустили нужный сценарий, в случае если выполнился его requirement @@ -505,7 +531,8 @@ def test_action(self): user.descriptions = {"scenarios": {scenario_name: scenario}} action = ClearCurrentScenarioAction({}) - result = action.run(user, {}, {}) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, {}, {})) self.assertIsNone(result) user.last_scenarios.delete.assert_called_once() user.forms.remove_item.assert_called_once() @@ -518,7 +545,8 @@ def test_action_with_empty_scenarios_names(self): user.last_scenarios.delete = Mock() action = ClearCurrentScenarioAction({}) - result = action.run(user, {}, {}) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, {}, {})) self.assertIsNone(result) user.last_scenarios.delete.assert_not_called() user.forms.remove_item.assert_not_called() @@ -537,7 +565,8 @@ def test_action(self): user.descriptions = {"scenarios": {scenario_name: scenario}} action = ClearScenarioByIdAction({"scenario_id": scenario_name}) - result = action.run(user, {}, {}) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, {}, {})) self.assertIsNone(result) user.last_scenarios.delete.assert_called_once() user.forms.remove_item.assert_called_once() @@ -549,7 +578,8 @@ def test_action_with_empty_scenarios_names(self): user.last_scenarios.last_scenario_name = "test_scenario" action = ClearScenarioByIdAction({}) - result = action.run(user, {}, {}) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, {}, {})) self.assertIsNone(result) user.last_scenarios.delete.assert_not_called() user.forms.remove_item.assert_not_called() @@ -569,7 +599,8 @@ def test_action(self): user.descriptions = {"scenarios": {scenario_name: scenario}} action = ClearCurrentScenarioFormAction({}) - result = action.run(user, {}, {}) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, {}, {})) self.assertIsNone(result) user.forms.clear_form.assert_called_once() @@ -586,7 +617,8 @@ def test_action_with_empty_last_scenario(self): user.descriptions = {"scenarios": {scenario_name: scenario}} action = ClearCurrentScenarioFormAction({}) - result = action.run(user, {}, {}) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, {}, {})) self.assertIsNone(result) user.forms.remove_item.assert_not_called() @@ -601,7 +633,8 @@ def test_action(self): user.scenario_models = {'test_scenario': scenario_model} action = ResetCurrentNodeAction({}) - result = action.run(user, {}, {}) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, {}, {})) self.assertIsNone(result) self.assertIsNone(user.scenario_models['test_scenario'].current_node) @@ -614,7 +647,8 @@ def test_action_with_empty_last_scenario(self): user.scenario_models = {'test_scenario': scenario_model} action = ResetCurrentNodeAction({}) - result = action.run(user, {}, {}) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, {}, {})) self.assertIsNone(result) self.assertEqual('some_node', user.scenario_models['test_scenario'].current_node) @@ -630,7 +664,8 @@ def test_specific_target(self): 'node_id': 'another_node' } action = ResetCurrentNodeAction(items) - result = action.run(user, {}, {}) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(action.run(user, {}, {})) self.assertIsNone(result) self.assertEqual('another_node', user.scenario_models['test_scenario'].current_node) @@ -669,7 +704,8 @@ def test_action_with_non_empty_scenario(self): ) action = AddHistoryEventAction(items) - action.run(self.user, None, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, None, None)) self.user.history.add_event.assert_called_once() self.user.history.add_event.assert_called_once_with(expected) @@ -683,7 +719,8 @@ def test_action_with_empty_scenario(self): } action = AddHistoryEventAction(items) - action.run(self.user, None, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, None, None)) self.user.history.add_event.assert_not_called() @@ -706,7 +743,8 @@ def test_action_with_jinja(self): ) action = AddHistoryEventAction(items) - action.run(self.user, None, None) + loop = asyncio.get_event_loop() + loop.run_until_complete(action.run(self.user, None, None)) self.user.history.add_event.assert_called_once() self.user.history.add_event.assert_called_once_with(expected) From 0d9768848b14cfdbf611a6d373766b7b20f29c98 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 8 Nov 2021 04:37:10 +0300 Subject: [PATCH 031/116] async fillers --- .../form_filling_scenario.py | 19 ++-- .../tree_scenario/tree_scenario.py | 2 +- scenarios/scenario_models/field/field.py | 5 +- .../field/field_filler_description.py | 96 +++++++++---------- 4 files changed, 63 insertions(+), 59 deletions(-) diff --git a/scenarios/scenario_descriptions/form_filling_scenario.py b/scenarios/scenario_descriptions/form_filling_scenario.py index 12ba79e7..a7e3e7ec 100644 --- a/scenarios/scenario_descriptions/form_filling_scenario.py +++ b/scenarios/scenario_descriptions/form_filling_scenario.py @@ -61,7 +61,7 @@ def _find_field(self, form, text_preprocessing_result, user, params): field = form.fields[field_name] if not field.valid and field.description.has_requests and \ field.description.requirement.check( - text_preprocessing_result, user, params + text_preprocessing_result, user, params ): return field @@ -75,7 +75,7 @@ def get_fields_data(self, form, form_key): def _clean_key(self, key: str): return key.replace(" ", "") - def _extract_by_field_filler(self, field_key, field_descr, text_normalization_result, user, params): + async def _extract_by_field_filler(self, field_key, field_descr, text_normalization_result, user, params): result = {} check = field_descr.requirement.check(text_normalization_result, user, params) log_params = self._log_params() @@ -85,7 +85,7 @@ def _extract_by_field_filler(self, field_key, field_descr, text_normalization_re message = "FormFillingScenario.extract: field %(field_key)s requirement %(requirement)s return value: %(check)s" log(message, user, log_params) if check: - result[field_key] = field_descr.filler.run(user, text_normalization_result, params) + result[field_key] = await (field_descr.filler.run(user, text_normalization_result, params)) event = Event(type=HistoryConstants.types.FIELD_EVENT, scenario=self.root_id, content={HistoryConstants.content_fields.FIELD: field_key}, @@ -93,7 +93,7 @@ def _extract_by_field_filler(self, field_key, field_descr, text_normalization_re user.history.add_event(event) return result - def _extract_data(self, form, text_normalization_result, user, params): + async def _extract_data(self, form, text_normalization_result, user, params): result = {} callback_id = user.message.callback_id @@ -103,14 +103,15 @@ def _extract_data(self, form, text_normalization_result, user, params): field = form.fields[request_field["id"]] field_descr = form.description.fields[request_field["id"]] if field.available: - result.update(self._extract_by_field_filler(request_field["id"], field_descr, text_normalization_result, - user, params)) + result.update(await self._extract_by_field_filler(request_field["id"], field_descr, + text_normalization_result, + user, params)) else: for field_key, field_descr in form.description.fields.items(): field = form.fields[field_key] if field.available and isinstance(field, QuestionField): - result.update(self._extract_by_field_filler(field_key, field_descr, - text_normalization_result, user, params)) + result.update(await self._extract_by_field_filler(field_key, field_descr, + text_normalization_result, user, params)) return result def _validate_extracted_data(self, user, text_preprocessing_result, form, data_extracted, params): @@ -180,7 +181,7 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No user.last_scenarios.add(self.id, text_preprocessing_result) user.preprocessing_messages_for_scenarios.add(text_preprocessing_result) - data_extracted = self._extract_data(form, text_preprocessing_result, user, params) + data_extracted = await self._extract_data(form, text_preprocessing_result, user, params) logging_params = {"data_extracted_str": str(data_extracted)} logging_params.update(self._log_params()) log("Extracted data=%(data_extracted_str)s", user, logging_params) diff --git a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py index 28c7a4b6..f1e6aca7 100644 --- a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py +++ b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py @@ -107,7 +107,7 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No for field_key, field_descr in internal_form.description.fields.items(): field = internal_form.fields[field_key] if field.available: - extracted = field_descr.filler.run(user, text_preprocessing_result, params) + extracted = await field_descr.filler.run(user, text_preprocessing_result, params) if extracted is not None: event = Event(type=HistoryConstants.types.FIELD_EVENT, scenario=self.root_id, diff --git a/scenarios/scenario_models/field/field.py b/scenarios/scenario_models/field/field.py index 5db7a514..c44e9335 100644 --- a/scenarios/scenario_models/field/field.py +++ b/scenarios/scenario_models/field/field.py @@ -1,4 +1,6 @@ # coding: utf-8 +import asyncio + from core.logging.logger_utils import log from core.model.registered import Registered @@ -37,9 +39,10 @@ def can_be_updated(self): return self.value is not None def check_can_be_filled(self, text_preprocessing_result, user): + loop = asyncio.get_event_loop() return ( self.description.requirement.check(text_preprocessing_result, user) and - self.description.filler.run(user, text_preprocessing_result) is not None + loop.run_until_complete(self.description.filler.run(user, text_preprocessing_result)) is not None ) @property diff --git a/scenarios/scenario_models/field/field_filler_description.py b/scenarios/scenario_models/field/field_filler_description.py index 5a237117..27d6a892 100644 --- a/scenarios/scenario_models/field/field_filler_description.py +++ b/scenarios/scenario_models/field/field_filler_description.py @@ -43,7 +43,7 @@ def _log_params(self): "filler": self.__class__.__name__ } - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, params: Dict[str, Any] = None) -> None: return None @@ -53,9 +53,9 @@ def on_extract_error(self, text_preprocessing_result, user, params=None): level="ERROR", exc_info=True) return None - def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Any]] = None) -> None: - return self.extract(text_preprocessing_result, user, params) + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Any]] = None) -> None: + return await self.extract(text_preprocessing_result, user, params) def _postprocessing(self, user: User, item: str) -> None: last_scenario_name = user.last_scenarios.last_scenario_name @@ -70,10 +70,10 @@ def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> self.filler = items.get("filler") @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, - user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, + user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: filler = user.descriptions["external_field_fillers"][self.filler] - return filler.run(user, text_preprocessing_result, params) + return await filler.run(user, text_preprocessing_result, params) class CompositeFiller(FieldFillerDescription): @@ -89,8 +89,8 @@ def build_fillers(self): return self._fillers @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, - user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, + user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: extracted = None for filler in self.fillers: extracted = filler.extract(text_preprocessing_result, user, params) @@ -111,8 +111,8 @@ def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> self.template: UnifiedTemplate = UnifiedTemplate(value) @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, - user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, + user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: params = params or {} collected = user.parametrizer.collect(text_preprocessing_result) params.update(collected) @@ -136,8 +136,8 @@ def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, class FirstNumberFiller(FieldFillerDescription): @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[int]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[int]: numbers = text_preprocessing_result.num_token_values if numbers: log_params = self._log_params() @@ -150,8 +150,8 @@ def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: class FirstCurrencyFiller(FieldFillerDescription): @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: currencies = text_preprocessing_result.ccy_token_values if currencies: log_params = self._log_params() @@ -164,8 +164,8 @@ def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: class FirstOrgFiller(FieldFillerDescription): @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: orgs = text_preprocessing_result.org_token_values if orgs: log_params = self._log_params() @@ -178,8 +178,8 @@ def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: class FirstGeoFiller(FieldFillerDescription): @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: geos = text_preprocessing_result.geo_token_values if geos: log_params = self._log_params() @@ -199,8 +199,8 @@ def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> self.delimiter = items.get("delimiter", ",") @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: original_text = text_preprocessing_result.original_text match = re.findall(self.regexp, original_text) if match: @@ -232,15 +232,16 @@ def _operation(self, original_text, typeOp, amount): return func(original_text, amount) if amount else func(original_text) @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: original_text = text_preprocessing_result.original_text if self.operations: for op in self.operations: original_text = self._operation(original_text, op["type"], op.get("amount")) text_preprocessing_result_copy = pickle_deepcopy(text_preprocessing_result) text_preprocessing_result_copy.original_text = original_text - return super(RegexpAndStringOperationsFieldFiller, self).extract(text_preprocessing_result_copy, user, params) + return await super(RegexpAndStringOperationsFieldFiller, self).extract(text_preprocessing_result_copy, + user, params) class AllRegexpsFieldFiller(FieldFillerDescription): @@ -256,8 +257,8 @@ def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> self.original_text_lower = items.get("original_text_lower") or False @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: original_text = text_preprocessing_result.original_text if self.original_text_lower: original_text = original_text.lower() @@ -276,8 +277,8 @@ def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: class FirstPersonFiller(FieldFillerDescription): @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: persons = text_preprocessing_result.person_token_values if persons: log_params = self._log_params() @@ -302,19 +303,19 @@ def build_filler(self): return self._filler @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: result = self.filler.extract(text_preprocessing_result, user, params) if result is None: - result = self._try_extract_last_messages(user, params) + result = await self._try_extract_last_messages(user, params) return result - def _try_extract_last_messages(self, user, params): + async def _try_extract_last_messages(self, user, params): processed_items = user.preprocessing_messages_for_scenarios.processed_items count = self.count - 1 if self.count else len(processed_items) for preprocessing_result_raw in islice(processed_items, 0, count): preprocessing_result = TextPreprocessingResult(preprocessing_result_raw) - result = self.filler.extract(preprocessing_result, user, params) + result = await self.filler.extract(preprocessing_result, user, params) if result is not None: return result @@ -322,8 +323,8 @@ def _try_extract_last_messages(self, user, params): class UserIdFiller(FieldFillerDescription): @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: result = user.message.uuid.get('userId') return result @@ -355,8 +356,8 @@ def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> self.normalized_cases.append((key, tokens_list)) @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: tpr_tokenized_set = {norm.get("lemma") for norm in text_preprocessing_result.tokenized_elements_list_pymorphy if norm.get("token_type") != "SENTENCE_ENDPOINT_TOKEN"} for key, tokens_list in self.normalized_cases: @@ -390,8 +391,8 @@ def _check_exceptions(self, key, tpr_original_set): return False @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + async def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[str]: tpr_original_set = {*text_preprocessing_result.original_text.split()} for key, tokens_list in self.original_cases: for tokens in tokens_list: @@ -428,8 +429,8 @@ def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> } @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[bool]: + async def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Optional[bool]: if text_preprocessing_result.tokenized_string in self.yes_words_normalized: params = self._log_params() params["tokenized_string"] = text_preprocessing_result.tokenized_string @@ -451,9 +452,8 @@ def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User class ApproveRawTextFiller(ApproveFiller): @exc_handler(on_error_obj_method_name="on_extract_error") - def extract( - self, text_preprocessing_result: TextPreprocessingResult, user: User, params: Dict[str, Any] = None - ) -> Optional[bool]: + async def extract(self, text_preprocessing_result: TextPreprocessingResult, + user: User, params: Dict[str, Any] = None) -> Optional[bool]: original_text = ' '.join(text_preprocessing_result.original_text.split()).lower().rstrip('!.)') if original_text in self.set_yes_words: params = self._log_params() @@ -493,8 +493,8 @@ def _get_result(self, answers: List[Dict[str, Union[str, float, bool]]]) -> str: return answers[0][self._cls_const_answer_key] @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Union[str, None, List[Dict[str, Union[str, float, bool]]]]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> Union[str, None, List[Dict[str, Union[str, float, bool]]]]: result = None classifier = self.classifier with StatsTimer() as timer: @@ -515,6 +515,6 @@ def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: class ClassifierFillerMeta(ClassifierFiller): - def _get_result(self, answers: List[Dict[str, Union[str, float, bool]]]) -> List[ - Dict[str, Union[str, float, bool]]]: + def _get_result(self, answers: List[Dict[str, Union[str, float, bool]]])\ + -> List[Dict[str, Union[str, float, bool]]]: return answers From 0afbc8b3b413e736772dcea2a6ebb56040e1d1e0 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Mon, 8 Nov 2021 09:24:34 +0300 Subject: [PATCH 032/116] done async actions --- .../behaviors_test/test_behavior_description.py | 4 ++-- tests/scenarios_tests/scenarios_test/test_tree_scenario.py | 4 ++-- tests/smart_kit_tests/handlers/test_handle_respond.py | 4 ++-- tests/smart_kit_tests/models/test_dialogue_manager.py | 4 ++-- .../system_answers/test_nothing_found_action.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/scenarios_tests/behaviors_test/test_behavior_description.py b/tests/scenarios_tests/behaviors_test/test_behavior_description.py index ac40b3f8..c6339dc8 100644 --- a/tests/scenarios_tests/behaviors_test/test_behavior_description.py +++ b/tests/scenarios_tests/behaviors_test/test_behavior_description.py @@ -7,7 +7,7 @@ class MockAction(Action): - def run(self, user, text_preprocessing_result, params=None): + async def run(self, user, text_preprocessing_result, params=None): return [] @@ -16,7 +16,7 @@ def __init__(self): self.id = '123-456-789' self.version = 21 # какая-то версия - def run(self, a, b, c): + async def run(self, a, b, c): return 123 # некий результат diff --git a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py index d223a97a..8b48732a 100644 --- a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py +++ b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py @@ -12,7 +12,7 @@ def __init__(self, items=None, command_name=None): self.called = False self.command_name = command_name - def run(self, user, text_preprocessing_result, params): + async def run(self, user, text_preprocessing_result, params): self.called = True if self.command_name: return [Command(self.command_name)] @@ -22,7 +22,7 @@ class BreakAction: def __init__(self, items=None): pass - def run(self, user, text_preprocessing_result, params): + async def run(self, user, text_preprocessing_result, params): user.scenario_models["some_id"].break_scenario = True return [] diff --git a/tests/smart_kit_tests/handlers/test_handle_respond.py b/tests/smart_kit_tests/handlers/test_handle_respond.py index 5cdbc08d..531d612c 100644 --- a/tests/smart_kit_tests/handlers/test_handle_respond.py +++ b/tests/smart_kit_tests/handlers/test_handle_respond.py @@ -21,8 +21,8 @@ def setUp(self): self.test_user1.message.app_info = {} self.test_user1.behaviors = MagicMock() - self.test_action = Mock('action') - self.test_action.run = lambda x, y, z: 10 # пусть что то возвращает + self.test_action = Mock('action') # должно быть async? + self.test_action.run = lambda x, y, z: 10 # пусть что то возвращает. Должно быть async? self.test_user2 = MagicMock('user') self.test_user2.id = '123-345-678' # пусть чему-то равняется self.test_user2.descriptions = {'external_actions': {'any action name': self.test_action}} diff --git a/tests/smart_kit_tests/models/test_dialogue_manager.py b/tests/smart_kit_tests/models/test_dialogue_manager.py index b8334a22..80bacaa2 100644 --- a/tests/smart_kit_tests/models/test_dialogue_manager.py +++ b/tests/smart_kit_tests/models/test_dialogue_manager.py @@ -37,9 +37,9 @@ def setUp(self): self.test_scenario2.text_fits = lambda x, y: True self.test_scenario2.run = lambda x, y: y.name + x.name self.test_scenarios = TestScenarioDesc({1: self.test_scenario1, 2: self.test_scenario2}) - self.TestAction = Mock() + self.TestAction = Mock() # должно быть async? self.TestAction.description = "test_function" - self.TestAction.run = lambda x, y: x.name + y.name + self.TestAction.run = lambda x, y: x.name + y.name # должно быть async? self.app_name = "test" def test_log_const(self): diff --git a/tests/smart_kit_tests/system_answers/test_nothing_found_action.py b/tests/smart_kit_tests/system_answers/test_nothing_found_action.py index 1f7835a9..f93f2d41 100644 --- a/tests/smart_kit_tests/system_answers/test_nothing_found_action.py +++ b/tests/smart_kit_tests/system_answers/test_nothing_found_action.py @@ -28,5 +28,5 @@ def test_system_answers_nothing_found_action_init(self): def test_system_answer_nothing_found_action_run(self): obj1 = nothing_found_action.NothingFoundAction() obj2 = nothing_found_action.NothingFoundAction(self.test_items1, self.test_id) - self.assertTrue(isinstance(obj1.run(self.test_user1, self.test_text_preprocessing_result).pop(), Command)) - self.assertTrue(isinstance(obj2.run(self.test_user1, self.test_text_preprocessing_result).pop(), Command)) + self.assertTrue(isinstance((await obj1.run(self.test_user1, self.test_text_preprocessing_result)).pop(), Command)) + self.assertTrue(isinstance((await obj2.run(self.test_user1, self.test_text_preprocessing_result)).pop(), Command)) From 1c2dac7a90140bd0005866bcea30cc0a70fb3457 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 8 Nov 2021 20:49:24 +0300 Subject: [PATCH 033/116] async fillers --- core/basic_models/requirement/basic_requirements.py | 4 ++-- core/basic_models/scenarios/base_scenario.py | 2 +- scenarios/scenario_descriptions/form_filling_scenario.py | 8 ++++---- scenarios/scenario_models/field/field.py | 5 ++--- .../scenario_models/field/field_filler_description.py | 4 ++-- smart_kit/models/dialogue_manager.py | 2 +- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/core/basic_models/requirement/basic_requirements.py b/core/basic_models/requirement/basic_requirements.py index 31f0c78e..545b87f1 100644 --- a/core/basic_models/requirement/basic_requirements.py +++ b/core/basic_models/requirement/basic_requirements.py @@ -226,14 +226,14 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: id, ) - def check( + async def check( self, text_preprocessing_result: TextPreprocessingResult, user: User, params: Dict[str, Any] = None ) -> bool: result = bool( - self.filler.extract(text_preprocessing_result, user, params), + await self.filler.extract(text_preprocessing_result, user, params), ) return result diff --git a/core/basic_models/scenarios/base_scenario.py b/core/basic_models/scenarios/base_scenario.py index 09d1023e..ea6250dc 100644 --- a/core/basic_models/scenarios/base_scenario.py +++ b/core/basic_models/scenarios/base_scenario.py @@ -54,7 +54,7 @@ def check_available(self, text_preprocessing_result, user): def _log_params(self): return {log_const.KEY_NAME: log_const.SCENARIO_VALUE} - def text_fits(self, text_preprocessing_result, user): + async def text_fits(self, text_preprocessing_result, user): return False def get_no_commands_action(self, user, text_preprocessing_result, params: Dict[str, Any] = None): diff --git a/scenarios/scenario_descriptions/form_filling_scenario.py b/scenarios/scenario_descriptions/form_filling_scenario.py index a7e3e7ec..6830e52e 100644 --- a/scenarios/scenario_descriptions/form_filling_scenario.py +++ b/scenarios/scenario_descriptions/form_filling_scenario.py @@ -26,8 +26,8 @@ def _get_form(self, user): form.refresh() return form - def text_fits(self, text_preprocessing_result, user): - return self._check_field(text_preprocessing_result, user, None) + async def text_fits(self, text_preprocessing_result, user): + return await self._check_field(text_preprocessing_result, user, None) def check_ask_again_requests(self, text_preprocessing_result, user, params): form = user.forms[self.form_type] @@ -48,10 +48,10 @@ def ask_again(self, text_preprocessing_result, user, params): return question.run(user, text_preprocessing_result, params) - def _check_field(self, text_preprocessing_result, user, params): + async def _check_field(self, text_preprocessing_result, user, params): form = user.forms[self.form_type] field = self._field(form, text_preprocessing_result, user, params) - return field.check_can_be_filled(text_preprocessing_result, user) if field else False + return await field.check_can_be_filled(text_preprocessing_result, user) if field else False def _field(self, form, text_preprocessing_result, user, params): return self._find_field(form, text_preprocessing_result, user, params) diff --git a/scenarios/scenario_models/field/field.py b/scenarios/scenario_models/field/field.py index c44e9335..34d012f9 100644 --- a/scenarios/scenario_models/field/field.py +++ b/scenarios/scenario_models/field/field.py @@ -38,11 +38,10 @@ def available(self): def can_be_updated(self): return self.value is not None - def check_can_be_filled(self, text_preprocessing_result, user): - loop = asyncio.get_event_loop() + async def check_can_be_filled(self, text_preprocessing_result, user): return ( self.description.requirement.check(text_preprocessing_result, user) and - loop.run_until_complete(self.description.filler.run(user, text_preprocessing_result)) is not None + await self.description.filler.run(user, text_preprocessing_result) is not None ) @property diff --git a/scenarios/scenario_models/field/field_filler_description.py b/scenarios/scenario_models/field/field_filler_description.py index 27d6a892..3dc4ffc7 100644 --- a/scenarios/scenario_models/field/field_filler_description.py +++ b/scenarios/scenario_models/field/field_filler_description.py @@ -93,7 +93,7 @@ async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: extracted = None for filler in self.fillers: - extracted = filler.extract(text_preprocessing_result, user, params) + extracted = await filler.extract(text_preprocessing_result, user, params) if extracted is not None: break return extracted @@ -305,7 +305,7 @@ def build_filler(self): @exc_handler(on_error_obj_method_name="on_extract_error") async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, params: Dict[str, Any] = None) -> Optional[str]: - result = self.filler.extract(text_preprocessing_result, user, params) + result = await self.filler.extract(text_preprocessing_result, user, params) if result is None: result = await self._try_extract_last_messages(user, params) return result diff --git a/smart_kit/models/dialogue_manager.py b/smart_kit/models/dialogue_manager.py index 92ac38f4..26ae9358 100644 --- a/smart_kit/models/dialogue_manager.py +++ b/smart_kit/models/dialogue_manager.py @@ -38,7 +38,7 @@ async def run(self, text_preprocessing_result, user): if is_form_filling: params = user.parametrizer.collect(text_preprocessing_result) - if not scenario.text_fits(text_preprocessing_result, user): + if not await scenario.text_fits(text_preprocessing_result, user): if scenario.check_ask_again_requests(text_preprocessing_result, user, params): reply = scenario.ask_again(text_preprocessing_result, user, params) From d38246a40df24cb93335944f53ce8f2d501edb25 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 8 Nov 2021 20:53:31 +0300 Subject: [PATCH 034/116] remove useless import --- scenarios/scenario_models/field/field.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scenarios/scenario_models/field/field.py b/scenarios/scenario_models/field/field.py index 34d012f9..5ec8f865 100644 --- a/scenarios/scenario_models/field/field.py +++ b/scenarios/scenario_models/field/field.py @@ -1,6 +1,4 @@ # coding: utf-8 -import asyncio - from core.logging.logger_utils import log from core.model.registered import Registered From 4ad7dd073ea0d3f854e526ebd48165b3fd83faa2 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 9 Nov 2021 05:27:40 +0300 Subject: [PATCH 035/116] async tests for fillers --- core/basic_models/scenarios/base_scenario.py | 4 +-- .../field/composite_fillers.py | 10 +++--- smart_kit/models/smartapp_model.py | 2 +- .../actions_test/test_action.py | 15 ++++----- tests/scenarios_tests/fillers/test_approve.py | 16 +++++----- .../fillers/test_available_info_filler.py | 28 ++++++++-------- .../fillers/test_classifier_filler.py | 16 +++++----- .../fillers/test_composite_filler.py | 12 +++---- .../fillers/test_external_filler.py | 4 +-- .../fillers/test_first_meeting.py | 16 +++++----- .../fillers/test_geo_token_filler.py | 12 +++---- .../fillers/test_intersection.py | 32 +++++++++---------- .../fillers/test_org_token_filler.py | 12 +++---- .../fillers/test_person_filler.py | 12 +++---- .../fillers/test_previous_messages_filler.py | 8 ++--- ...est_regexp_and_string_operations_filler.py | 20 ++++++------ .../fillers/test_regexp_filler.py | 20 ++++++------ .../fillers/test_regexps_filler.py | 12 +++---- .../test_run_scenario_by_project_name.py | 6 ++-- .../test_nothing_found_action.py | 2 +- 20 files changed, 128 insertions(+), 131 deletions(-) diff --git a/core/basic_models/scenarios/base_scenario.py b/core/basic_models/scenarios/base_scenario.py index ea6250dc..4866a03d 100644 --- a/core/basic_models/scenarios/base_scenario.py +++ b/core/basic_models/scenarios/base_scenario.py @@ -57,7 +57,7 @@ def _log_params(self): async def text_fits(self, text_preprocessing_result, user): return False - def get_no_commands_action(self, user, text_preprocessing_result, params: Dict[str, Any] = None): + async def get_no_commands_action(self, user, text_preprocessing_result, params: Dict[str, Any] = None): log_params = {log_const.KEY_NAME: scenarios_log_const.CHOSEN_ACTION_VALUE, scenarios_log_const.CHOSEN_ACTION_VALUE: self._empty_answer} log(scenarios_log_const.CHOSEN_ACTION_MESSAGE, user, log_params) @@ -70,7 +70,7 @@ def get_no_commands_action(self, user, text_preprocessing_result, params: Dict[s empty_answer = [] return empty_answer - def get_action_results(self, user, text_preprocessing_result, + async def get_action_results(self, user, text_preprocessing_result, actions: List[Action], params: Dict[str, Any] = None) -> List[Command]: results = [] for action in actions: diff --git a/scenarios/scenario_models/field/composite_fillers.py b/scenarios/scenario_models/field/composite_fillers.py index c07f49ab..80244142 100644 --- a/scenarios/scenario_models/field/composite_fillers.py +++ b/scenarios/scenario_models/field/composite_fillers.py @@ -28,7 +28,7 @@ def on_extract_error(self, text_preprocessing_result, user, params=None): return None @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: return await self.run(user, text_preprocessing_result, params) @@ -60,8 +60,8 @@ def on_extract_error(self, text_preprocessing_result, user, params=None): return None @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, - user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, + user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: return await self.run(user, text_preprocessing_result, params) @@ -92,6 +92,6 @@ def on_extract_error(self, text_preprocessing_result, user, params=None): return None @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, - user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: + async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, + user: User, params: Dict[str, Any] = None) -> Optional[Union[int, float, str, bool, List, Dict]]: return await self.run(user, text_preprocessing_result, params) diff --git a/smart_kit/models/smartapp_model.py b/smart_kit/models/smartapp_model.py index b2f320e6..ecc74c1f 100644 --- a/smart_kit/models/smartapp_model.py +++ b/smart_kit/models/smartapp_model.py @@ -66,7 +66,7 @@ async def answer(self, message, user): return commands - def on_answer_error(self, message, user): + async def on_answer_error(self, message, user): user.do_not_save = True smart_kit_metrics.counter_exception(self.app_name) params = {log_const.KEY_NAME: log_const.DIALOG_ERROR_VALUE, diff --git a/tests/scenarios_tests/actions_test/test_action.py b/tests/scenarios_tests/actions_test/test_action.py index 72dac6d3..afcbaac2 100644 --- a/tests/scenarios_tests/actions_test/test_action.py +++ b/tests/scenarios_tests/actions_test/test_action.py @@ -269,24 +269,21 @@ def setUp(self): user.variables.set = Mock() self.user = user - def test_action(self): + async def test_action(self): action = SetVariableAction({"key": "some_key", "value": "some_value"}) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, None)) + await action.run(self.user, None) self.user.variables.set.assert_called_with("some_key", "some_value", None) - def test_action_jinja_key_default(self): + async def test_action_jinja_key_default(self): self.user.message.payload = {"some_value": "some_value_test"} action = SetVariableAction({"key": "some_key", "value": "{{payload.some_value}}"}) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, None)) + await action.run(self.user, None) self.user.variables.set.assert_called_with("some_key", "some_value_test", None) - def test_action_jinja_no_key(self): + async def test_action_jinja_no_key(self): self.user.message.payload = {"some_value": "some_value_test"} action = SetVariableAction({"key": "some_key", "value": "{{payload.no_key}}"}) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, None)) + await action.run(self.user, None) self.user.variables.set.assert_called_with("some_key", "", None) diff --git a/tests/scenarios_tests/fillers/test_approve.py b/tests/scenarios_tests/fillers/test_approve.py index 4fdf623c..e0ddb412 100644 --- a/tests/scenarios_tests/fillers/test_approve.py +++ b/tests/scenarios_tests/fillers/test_approve.py @@ -23,7 +23,7 @@ def patch_get_app_config(mock_get_app_config): class TestApproveFiller(TestCase): @patch('smart_kit.configs.get_app_config') - def test_1(self, mock_get_app_config): + async def test_1(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) items = { 'yes_words': [ @@ -49,24 +49,24 @@ def test_1(self, mock_get_app_config): user_phrase = 'даю' text_pre_result = TextPreprocessingResult(normalizer(user_phrase)) - result = filler.extract(text_pre_result, None) + result = await filler.extract(text_pre_result, None) self.assertTrue(result) user_phrase = 'да нет' text_pre_result = TextPreprocessingResult(normalizer(user_phrase)) - result = filler.extract(text_pre_result, None) + result = await filler.extract(text_pre_result, None) self.assertFalse(result) user_phrase = 'даю добро' text_pre_result = TextPreprocessingResult(normalizer(user_phrase)) - result = filler.extract(text_pre_result, None) + result = await filler.extract(text_pre_result, None) self.assertIsNone(result) class TestApproveRawTextFiller(TestCase): @patch('smart_kit.configs.get_app_config') - def test_1(self, mock_get_app_config): + async def test_1(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) items = { 'yes_words': [ @@ -92,15 +92,15 @@ def test_1(self, mock_get_app_config): user_phrase = 'конечно' text_pre_result = TextPreprocessingResult(normalizer(user_phrase)) - result = filler.extract(text_pre_result, None) + result = await filler.extract(text_pre_result, None) self.assertTrue(result) user_phrase = 'да нет' text_pre_result = TextPreprocessingResult(normalizer(user_phrase)) - result = filler.extract(text_pre_result, None) + result = await filler.extract(text_pre_result, None) self.assertFalse(result) user_phrase = 'даю' text_pre_result = TextPreprocessingResult(normalizer(user_phrase)) - result = filler.extract(text_pre_result, None) + result = await filler.extract(text_pre_result, None) self.assertIsNone(result) diff --git a/tests/scenarios_tests/fillers/test_available_info_filler.py b/tests/scenarios_tests/fillers/test_available_info_filler.py index 5544eb4d..b1fe8621 100644 --- a/tests/scenarios_tests/fillers/test_available_info_filler.py +++ b/tests/scenarios_tests/fillers/test_available_info_filler.py @@ -39,7 +39,7 @@ def setUp(self): user.descriptions = {"render_templates": template} self.user = user - def test_getting_person_info_value(self): + async def test_getting_person_info_value(self): name = "Name!" surname = "Surname!" self.user.person_info.raw = Mock() @@ -47,34 +47,34 @@ def test_getting_person_info_value(self): person_info_items = {'value': '{{person_info.full_name.surname}}'} person_info_filler = AvailableInfoFiller(person_info_items) - result = person_info_filler.extract(None, self.user) + result = await person_info_filler.extract(None, self.user) self.assertEqual(result, surname) - def test_getting_payload_value(self): + async def test_getting_payload_value(self): self.user.message.payload = {"sf_answer": {"address": self.address}} - result = self.payload_filler.extract(None, self.user) + result = await self.payload_filler.extract(None, self.user) self.assertEqual(result, self.address) - def test_getting_uuid_value(self): + async def test_getting_uuid_value(self): uuid = "15" self.user.message.uuid = {"chatId": uuid} uuid_items = {'value': '{{uuid.chatId}}'} uuid_filler = AvailableInfoFiller(uuid_items) - result = uuid_filler.extract(None, self.user) + result = await uuid_filler.extract(None, self.user) self.assertEqual(result, uuid) - def test_not_failing_on_wrong_path(self): + async def test_not_failing_on_wrong_path(self): self.user.message.payload = {"other_answer": {"address": self.address}} - result = self.payload_filler.extract(None, self.user) + result = await self.payload_filler.extract(None, self.user) self.assertIsNone(result) - def test_return_empty_value(self): + async def test_return_empty_value(self): self.user.message.payload = {"sf_answer": '1'} - result = self.payload_filler.extract(None, self.user) + result = await self.payload_filler.extract(None, self.user) self.assertEqual("", result) - def test_filter(self): + async def test_filter(self): template = Mock() template.get_template = Mock(return_value=["payload.personInfo.identityCard"]) self.user.parametrizer = MockParametrizer(self.user, {"filter": True}) @@ -82,10 +82,10 @@ def test_filter(self): self.user.descriptions = {"render_templates": template} payload_items = {'value': '{{filter}}'} filler = AvailableInfoFiller(payload_items) - result = filler.extract(None, self.user) + result = await filler.extract(None, self.user) self.assertEqual("filter_out", result) - def test_getting_payload_parsed_value(self): + async def test_getting_payload_parsed_value(self): data = [ { "id": 1, @@ -143,5 +143,5 @@ def test_getting_payload_parsed_value(self): "cacheGuid": "FHGDDASDHDAKSGFLAK", "data": data } - result = payload_filler.extract(None, self.user) + result = await payload_filler.extract(None, self.user) self.assertEqual(result, data) diff --git a/tests/scenarios_tests/fillers/test_classifier_filler.py b/tests/scenarios_tests/fillers/test_classifier_filler.py index 1db00599..f1066596 100644 --- a/tests/scenarios_tests/fillers/test_classifier_filler.py +++ b/tests/scenarios_tests/fillers/test_classifier_filler.py @@ -25,16 +25,16 @@ def setUp(self): "find_best_answer", return_value=[{"answer": "нет", "score": 0.7, "other": False}, {"answer": "да", "score": 0.3, "other": False}] ) - def test_filler_extract(self, mock_classifier_model): + async def test_filler_extract(self, mock_classifier_model): """Тест кейз проверяет что поле заполнено наиболее вероятным значением, которое вернула модель.""" expected_res = "нет" - actual_res = self.filler.extract(self.mock_text_preprocessing_result, self.mock_user) + actual_res = await self.filler.extract(self.mock_text_preprocessing_result, self.mock_user) self.assertEqual(expected_res, actual_res) @patch.object(ExternalClassifier, "find_best_answer", return_value=[]) - def test_filler_extract_if_no_model_answer(self, mock_classifier_model): + async def test_filler_extract_if_no_model_answer(self, mock_classifier_model): """Тест кейз проверяет что поле осталось не заполненным те результат None, если модель не выдала ответ.""" - actual_res = self.filler.extract(self.mock_text_preprocessing_result, self.mock_user) + actual_res = await self.filler.extract(self.mock_text_preprocessing_result, self.mock_user) self.assertIsNone(actual_res) @@ -54,14 +54,14 @@ def setUp(self): "external_classifiers": ["read_book_or_not_classifier", "hello_scenario_classifier"]} @patch.object(ExternalClassifier, "find_best_answer", return_value=[{"answer": "нет", "score": 1.0, "other": False}]) - def test_filler_extract(self, mock_classifier_model): + async def test_filler_extract(self, mock_classifier_model): """Тест кейз проверяет что мы получаем тот же самый ответ, что вернула модель.""" expected_res = [{"answer": "нет", "score": 1.0, "other": False}] - actual_res = self.filler_meta.extract(self.mock_text_preprocessing_result, self.mock_user) + actual_res = await self.filler_meta.extract(self.mock_text_preprocessing_result, self.mock_user) self.assertEqual(expected_res, actual_res) @patch.object(ExternalClassifier, "find_best_answer", return_value=[]) - def test_filler_extract_if_no_model_answer(self, mock_classifier_model): + async def test_filler_extract_if_no_model_answer(self, mock_classifier_model): """Тест кейз проверяет результат None, если модель не выдала ответ.""" - actual_res = self.filler_meta.extract(self.mock_text_preprocessing_result, self.mock_user) + actual_res = await self.filler_meta.extract(self.mock_text_preprocessing_result, self.mock_user) self.assertIsNone(actual_res) diff --git a/tests/scenarios_tests/fillers/test_composite_filler.py b/tests/scenarios_tests/fillers/test_composite_filler.py index 2d9a11c7..fe9e2b49 100644 --- a/tests/scenarios_tests/fillers/test_composite_filler.py +++ b/tests/scenarios_tests/fillers/test_composite_filler.py @@ -23,7 +23,7 @@ def setUpClass(cls): field_filler_description["mock_filler"] = MockFiller TestCompositeFiller.user = Mock() - def test_first_filler(self): + async def test_first_filler(self): expected = "first" items = { "fillers": [ @@ -32,10 +32,10 @@ def test_first_filler(self): ] } filler = CompositeFiller(items) - result = filler.extract(None, self.user) + result = await filler.extract(None, self.user) self.assertEqual(expected, result) - def test_second_filler(self): + async def test_second_filler(self): expected = "second" items = { "fillers": [ @@ -44,10 +44,10 @@ def test_second_filler(self): ] } filler = CompositeFiller(items) - result = filler.extract(None, self.user) + result = await filler.extract(None, self.user) self.assertEqual(expected, result) - def test_not_fit(self): + async def test_not_fit(self): items = { "fillers": [ {"type": "mock_filler"}, @@ -55,5 +55,5 @@ def test_not_fit(self): ] } filler = CompositeFiller(items) - result = filler.extract(None, self.user) + result = await filler.extract(None, self.user) self.assertIsNone(result) \ No newline at end of file diff --git a/tests/scenarios_tests/fillers/test_external_filler.py b/tests/scenarios_tests/fillers/test_external_filler.py index 7886c1e3..892a660f 100644 --- a/tests/scenarios_tests/fillers/test_external_filler.py +++ b/tests/scenarios_tests/fillers/test_external_filler.py @@ -6,7 +6,7 @@ class TestExternalFieldFillerDescription(TestCase): - def test_1(self): + async def test_1(self): expected = 5 items = {"filler": "my_key"} mock_filler = Mock() @@ -16,6 +16,6 @@ def test_1(self): mock_user.descriptions = {"external_field_fillers": {"my_key": mock_filler}} filler = ExternalFieldFillerDescription(items) - result = filler.extract(None, mock_user) + result = await filler.extract(None, mock_user) self.assertEqual(expected, result) diff --git a/tests/scenarios_tests/fillers/test_first_meeting.py b/tests/scenarios_tests/fillers/test_first_meeting.py index daa08512..98e89c52 100644 --- a/tests/scenarios_tests/fillers/test_first_meeting.py +++ b/tests/scenarios_tests/fillers/test_first_meeting.py @@ -5,46 +5,46 @@ class TestFirstNumberFiller(TestCase): - def test_1(self): + async def test_1(self): expected = "5" items = {} text_preprocessing_result = Mock() text_preprocessing_result.num_token_values = [expected] filler = FirstNumberFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) - def test_2(self): + async def test_2(self): items = {} text_preprocessing_result = Mock() text_preprocessing_result.num_token_values = [] filler = FirstNumberFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertIsNone(result) class TestFirstCurrencyFiller(TestCase): - def test_1(self): + async def test_1(self): expected = "ru" items = {} text_preprocessing_result = Mock() text_preprocessing_result.ccy_token_values = [expected] filler = FirstCurrencyFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) - def test_2(self): + async def test_2(self): items = {} text_preprocessing_result = Mock() text_preprocessing_result.ccy_token_values = [] filler = FirstCurrencyFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertIsNone(result) diff --git a/tests/scenarios_tests/fillers/test_geo_token_filler.py b/tests/scenarios_tests/fillers/test_geo_token_filler.py index e67daab4..4f81e378 100644 --- a/tests/scenarios_tests/fillers/test_geo_token_filler.py +++ b/tests/scenarios_tests/fillers/test_geo_token_filler.py @@ -9,28 +9,28 @@ def setUp(self): items = {} self.filler = FirstGeoFiller(items) - def test_1(self): + async def test_1(self): expected = "москва" text_preprocessing_result = Mock() text_preprocessing_result.geo_token_values = ["москва"] - result = self.filler.extract(text_preprocessing_result, None) + result = await self.filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) - def test_2(self): + async def test_2(self): expected = "москва" text_preprocessing_result = Mock() text_preprocessing_result.geo_token_values = ["москва", "питер", "казань"] - result = self.filler.extract(text_preprocessing_result, None) + result = await self.filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) - def test_3(self): + async def test_3(self): text_preprocessing_result = Mock() text_preprocessing_result.geo_token_values = [] - result = self.filler.extract(text_preprocessing_result, None) + result = await self.filler.extract(text_preprocessing_result, None) self.assertIsNone(result) diff --git a/tests/scenarios_tests/fillers/test_intersection.py b/tests/scenarios_tests/fillers/test_intersection.py index cbab76b3..1ff84222 100644 --- a/tests/scenarios_tests/fillers/test_intersection.py +++ b/tests/scenarios_tests/fillers/test_intersection.py @@ -22,7 +22,7 @@ def patch_get_app_config(mock_get_app_config): class TestIntersectionFieldFiller(TestCase): @patch('smart_kit.configs.get_app_config') - def test_1(self, mock_get_app_config): + async def test_1(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) expected = 'лосось' items = { @@ -46,12 +46,12 @@ def test_1(self, mock_get_app_config): ] filler = IntersectionFieldFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) @patch('smart_kit.configs.get_app_config') - def test_2(self, mock_get_app_config): + async def test_2(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) items = { 'strict': True, @@ -75,12 +75,12 @@ def test_2(self, mock_get_app_config): ] filler = IntersectionFieldFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertIsNone(result) @patch('smart_kit.configs.get_app_config') - def test_3(self, mock_get_app_config): + async def test_3(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) expected = 'лосось' items = { @@ -101,24 +101,24 @@ def test_3(self, mock_get_app_config): ] filler = IntersectionFieldFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) @patch('smart_kit.configs.get_app_config') - def test_4(self, mock_get_app_config): + async def test_4(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) items = {} text_preprocessing_result = Mock() text_preprocessing_result.tokenized_elements_list_pymorphy = [] filler = IntersectionFieldFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertIsNone(result) @patch('smart_kit.configs.get_app_config') - def test_5(self, mock_get_app_config): + async def test_5(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) expected = 'дефолтный тунец' items = { @@ -142,14 +142,14 @@ def test_5(self, mock_get_app_config): ] filler = IntersectionFieldFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) class TestIntersectionOriginalTextFiller(TestCase): @patch('smart_kit.configs.get_app_config') - def test_1(self, mock_get_app_config): + async def test_1(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) items = { 'cases': { @@ -165,12 +165,12 @@ def test_1(self, mock_get_app_config): text_preprocessing_result.original_text = 'всего хорошего и спасибо за рыбу' filler = IntersectionOriginalTextFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertIsNone(result) @patch('smart_kit.configs.get_app_config') - def test_2(self, mock_get_app_config): + async def test_2(self, mock_get_app_config): expected = 'лосось' patch_get_app_config(mock_get_app_config) items = { @@ -187,12 +187,12 @@ def test_2(self, mock_get_app_config): text_preprocessing_result.original_text = 'всего хорошая и спасибо за рыба' filler = IntersectionOriginalTextFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) @patch('smart_kit.configs.get_app_config') - def test_3(self, mock_get_app_config): + async def test_3(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) items = { 'cases': { @@ -211,6 +211,6 @@ def test_3(self, mock_get_app_config): text_preprocessing_result.original_text = 'не это хорошая рыба' filler = IntersectionOriginalTextFiller(items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertIsNone(result) diff --git a/tests/scenarios_tests/fillers/test_org_token_filler.py b/tests/scenarios_tests/fillers/test_org_token_filler.py index 8fc6ed8e..27690030 100644 --- a/tests/scenarios_tests/fillers/test_org_token_filler.py +++ b/tests/scenarios_tests/fillers/test_org_token_filler.py @@ -9,28 +9,28 @@ def setUp(self): items = {} self.filler = FirstOrgFiller(items) - def test_1(self): + async def test_1(self): expected = "тинькофф" text_preprocessing_result = Mock() text_preprocessing_result.org_token_values = ["тинькофф"] - result = self.filler.extract(text_preprocessing_result, None) + result = await self.filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) - def test_2(self): + async def test_2(self): expected = "тинькофф" text_preprocessing_result = Mock() text_preprocessing_result.org_token_values = ["тинькофф", "втб", "мегафон"] - result = self.filler.extract(text_preprocessing_result, None) + result = await self.filler.extract(text_preprocessing_result, None) self.assertEqual(expected, result) - def test_3(self): + async def test_3(self): text_preprocessing_result = Mock() text_preprocessing_result.org_token_values = [] - result = self.filler.extract(text_preprocessing_result, None) + result = await self.filler.extract(text_preprocessing_result, None) self.assertIsNone(result) diff --git a/tests/scenarios_tests/fillers/test_person_filler.py b/tests/scenarios_tests/fillers/test_person_filler.py index ad64cebc..a35a91b7 100644 --- a/tests/scenarios_tests/fillers/test_person_filler.py +++ b/tests/scenarios_tests/fillers/test_person_filler.py @@ -9,28 +9,28 @@ def setUp(self): items = {} self.filler = FirstPersonFiller(items) - def test_1(self): + async def test_1(self): expected = {"name": "иван"} text_preprocessing_result = Mock() text_preprocessing_result.person_token_values = [{"name": "иван"}] - result = self.filler.extract(text_preprocessing_result, None) + result = await self.filler.extract(text_preprocessing_result, None) self.assertDictEqual(expected, result) - def test_2(self): + async def test_2(self): expected = {"name": "иван"} text_preprocessing_result = Mock() text_preprocessing_result.person_token_values = [{"name": "иван"}, {"name": "иван", "patronymic": "иванович"}] - result = self.filler.extract(text_preprocessing_result, None) + result = await self.filler.extract(text_preprocessing_result, None) self.assertDictEqual(expected, result) - def test_3(self): + async def test_3(self): text_preprocessing_result = Mock() text_preprocessing_result.person_token_values = [] - result = self.filler.extract(text_preprocessing_result, None) + result = await self.filler.extract(text_preprocessing_result, None) self.assertIsNone(result) \ No newline at end of file diff --git a/tests/scenarios_tests/fillers/test_previous_messages_filler.py b/tests/scenarios_tests/fillers/test_previous_messages_filler.py index d0e16a5a..ba4dabd3 100644 --- a/tests/scenarios_tests/fillers/test_previous_messages_filler.py +++ b/tests/scenarios_tests/fillers/test_previous_messages_filler.py @@ -15,7 +15,7 @@ def extract(self, text_preprocessing_result, user, params): class PreviousMessagesFillerTest(unittest.TestCase): - def test_fill_1(self): + async def test_fill_1(self): registered_factories[FieldFillerDescription] = field_filler_factory field_filler_description["mock_filler"] = MockFiller expected = "first" @@ -24,10 +24,10 @@ def test_fill_1(self): user.preprocessing_messages_for_scenarios = Mock() user.preprocessing_messages_for_scenarios.processed_items = [{}, {}, {}] filler = PreviousMessagesFiller(items) - filler.extract(None, user) + await filler.extract(None, user) self.assertEqual(filler.filler.count, 4) - def test_fill_2(self): + async def test_fill_2(self): registered_factories[FieldFillerDescription] = field_filler_factory field_filler_description["mock_filler"] = MockFiller expected = "first" @@ -36,5 +36,5 @@ def test_fill_2(self): user.preprocessing_messages_for_scenarios = Mock() user.preprocessing_messages_for_scenarios.processed_items = [{}, {}, {}] filler = PreviousMessagesFiller(items) - filler.extract(None, user) + await filler.extract(None, user) self.assertEqual(filler.filler.count, 2) diff --git a/tests/scenarios_tests/fillers/test_regexp_and_string_operations_filler.py b/tests/scenarios_tests/fillers/test_regexp_and_string_operations_filler.py index 80312eba..40e95fb8 100644 --- a/tests/scenarios_tests/fillers/test_regexp_and_string_operations_filler.py +++ b/tests/scenarios_tests/fillers/test_regexp_and_string_operations_filler.py @@ -35,38 +35,38 @@ def test_operation_amount(self): result = self._test_operation(field_value, type_op, amount) self.assertEqual(field_value.lstrip(amount), result) - def _test_extract(self, field_value): + async def _test_extract(self, field_value): text_preprocessing_result = PickableMock() text_preprocessing_result.original_text = field_value filler = RegexpAndStringOperationsFieldFiller(self.items) - return filler.extract(text_preprocessing_result, None) + return await filler.extract(text_preprocessing_result, None) - def test_extract_upper(self): + async def test_extract_upper(self): field_value = "1-rsar09a" self.items["operations"] = [{"type":"upper"}] - result = self._test_extract(field_value) + result = await self._test_extract(field_value) self.assertEqual(field_value.upper(), result) - def test_extract_rstrip(self): + async def test_extract_rstrip(self): field_value = "1-RSAR09A !)" self.items["operations"] = [{"type":"rstrip", "amount": "!) "}] - result = self._test_extract(field_value) + result = await self._test_extract(field_value) self.assertEqual(field_value.rstrip("!) "), result) - def test_extract_upper_rstrip(self): + async def test_extract_upper_rstrip(self): field_value = "1-rsar09a !)" self.items["operations"] = [ {"type":"upper"}, {"type":"rstrip", "amount": "!) "} ] - result = self._test_extract(field_value) + result = await self._test_extract(field_value) self.assertEqual(field_value.upper().rstrip("!) "), result) - def test_extract_no_operations(self): + async def test_extract_no_operations(self): field_value = "1-rsar09a !)" self.items["operations"] = [] - result = self._test_extract(field_value) + result = await self._test_extract(field_value) self.assertIsNone(result) diff --git a/tests/scenarios_tests/fillers/test_regexp_filler.py b/tests/scenarios_tests/fillers/test_regexp_filler.py index 7d43960e..e7c0e188 100644 --- a/tests/scenarios_tests/fillers/test_regexp_filler.py +++ b/tests/scenarios_tests/fillers/test_regexp_filler.py @@ -13,46 +13,46 @@ def setUp(self): def test_no_exp_init(self): self.assertRaises(KeyError, RegexpFieldFiller, {}) - def test_no_exp(self): + async def test_no_exp(self): field_value = "1-RSAR09A" text_preprocessing_result = Mock() text_preprocessing_result.original_text = field_value filler = RegexpFieldFiller(self.items) filler.regexp = None - self.assertIsNone(filler.extract(text_preprocessing_result, self.user)) + self.assertIsNone(await filler.extract(text_preprocessing_result, self.user)) - def test_extract(self): + async def test_extract(self): field_value = "1-RSAR09A" text_preprocessing_result = Mock() text_preprocessing_result.original_text = field_value filler = RegexpFieldFiller(self.items) - result = filler.extract(text_preprocessing_result, self.user) + result = await filler.extract(text_preprocessing_result, self.user) self.assertEqual(field_value, result) - def test_extract_no_match(self): + async def test_extract_no_match(self): text_preprocessing_result = Mock() text_preprocessing_result.original_text = "text" filler = RegexpFieldFiller(self.items) - result = filler.extract(text_preprocessing_result, self.user) + result = await filler.extract(text_preprocessing_result, self.user) self.assertIsNone(result) - def test_extract_mult_match_default_delimiter(self): + async def test_extract_mult_match_default_delimiter(self): field_value = "1-RSAR09A пустой тест 1-RSAR02A" res = ",".join(['1-RSAR09A', '1-RSAR02A']) text_preprocessing_result = Mock() text_preprocessing_result.original_text = field_value filler = RegexpFieldFiller(self.items) - result = filler.extract(text_preprocessing_result, self.user) + result = await filler.extract(text_preprocessing_result, self.user) self.assertEqual(res, result) - def test_extract_mult_match_custom_delimiter(self): + async def test_extract_mult_match_custom_delimiter(self): field_value = "1-RSAR09A пустой тест 1-RSAR02B" self.items["delimiter"] = ";" res = self.items["delimiter"].join(['1-RSAR09A', '1-RSAR02B']) @@ -60,6 +60,6 @@ def test_extract_mult_match_custom_delimiter(self): text_preprocessing_result.original_text = field_value filler = RegexpFieldFiller(self.items) - result = filler.extract(text_preprocessing_result, self.user) + result = await filler.extract(text_preprocessing_result, self.user) self.assertEqual(res, result) diff --git a/tests/scenarios_tests/fillers/test_regexps_filler.py b/tests/scenarios_tests/fillers/test_regexps_filler.py index 2ae53b6a..40b75b64 100644 --- a/tests/scenarios_tests/fillers/test_regexps_filler.py +++ b/tests/scenarios_tests/fillers/test_regexps_filler.py @@ -16,29 +16,29 @@ def setUpClass(cls): cls.filler = AllRegexpsFieldFiller(cls.items) - def test_extract_1(self): + async def test_extract_1(self): field_value = "Просим отозвать платежное поручение 14 от 23.01.19 на сумму 3500 и вернуть деньги на расчетный счет." text_preprocessing_result = Mock() text_preprocessing_result.original_text = field_value filler = AllRegexpsFieldFiller(self.items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertEqual('14', result) - def test_extract_2(self): + async def test_extract_2(self): field_value = "поручение12 поручение14 #1 n3 п/п70 n33" text_preprocessing_result = Mock() text_preprocessing_result.original_text = field_value filler = AllRegexpsFieldFiller(self.items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertEqual("3|33|1|12|14|70", result) - def test_extract_no_match(self): + async def test_extract_no_match(self): text_preprocessing_result = Mock() text_preprocessing_result.original_text = "текст без искомых номеров" filler = AllRegexpsFieldFiller(self.items) - result = filler.extract(text_preprocessing_result, None) + result = await filler.extract(text_preprocessing_result, None) self.assertIsNone(result) diff --git a/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py b/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py index 616ac188..be29a6c1 100644 --- a/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py +++ b/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py @@ -32,13 +32,13 @@ def setUp(self): self.test_text_preprocessing_result = Mock('TextPreprocessingResult') self.items = {"any_key": "any value"} - def test_run_scenario_by_project_name_run(self): + async def test_run_scenario_by_project_name_run(self): obj1 = RunScenarioByProjectNameAction(self.items) # без оглядки на аннотации из PEP 484 - self.assertTrue(obj1.run(self.test_user1, self.test_text_preprocessing_result, {'any_attr': {'any_data'}}) == + self.assertTrue(await obj1.run(self.test_user1, self.test_text_preprocessing_result, {'any_attr': {'any_data'}}) == 'result to run scenario') obj2 = RunScenarioByProjectNameAction(self.items) - self.assertIsNone(obj2.run(self.test_user2, self.test_text_preprocessing_result)) + self.assertIsNone(await obj2.run(self.test_user2, self.test_text_preprocessing_result)) def test_run_scenario_by_project_name_log_vars(self): obj = RunScenarioByProjectNameAction(self.items) diff --git a/tests/smart_kit_tests/system_answers/test_nothing_found_action.py b/tests/smart_kit_tests/system_answers/test_nothing_found_action.py index f93f2d41..28fd67d2 100644 --- a/tests/smart_kit_tests/system_answers/test_nothing_found_action.py +++ b/tests/smart_kit_tests/system_answers/test_nothing_found_action.py @@ -25,7 +25,7 @@ def test_system_answers_nothing_found_action_init(self): self.assertTrue(isinstance(obj1._action, StringAction)) self.assertTrue(obj1._action.command == NOTHING_FOUND) - def test_system_answer_nothing_found_action_run(self): + async def test_system_answer_nothing_found_action_run(self): obj1 = nothing_found_action.NothingFoundAction() obj2 = nothing_found_action.NothingFoundAction(self.test_items1, self.test_id) self.assertTrue(isinstance((await obj1.run(self.test_user1, self.test_text_preprocessing_result)).pop(), Command)) From 193257b11d87c550ee4c758ebd2f035ffc67f1f7 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Tue, 9 Nov 2021 10:32:29 +0300 Subject: [PATCH 036/116] make all requirements and their usages async --- core/basic_models/actions/basic_actions.py | 53 +++- core/basic_models/actions/string_actions.py | 192 ++++++++++++- .../requirement/basic_requirements.py | 84 +++--- .../requirement/counter_requirements.py | 8 +- .../requirement/device_requirements.py | 28 +- .../requirement/external_requirements.py | 6 +- .../requirement/project_requirements.py | 4 +- .../requirement/user_text_requirements.py | 20 +- core/basic_models/scenarios/base_scenario.py | 2 +- scenarios/actions/action.py | 48 +++- scenarios/requirements/requirements.py | 16 +- .../form_filling_scenario.py | 13 +- scenarios/scenario_models/field/field.py | 8 +- .../field_requirements/field_requirements.py | 39 ++- .../last_scenarios_description.py | 8 +- .../app/basic_entities/requirements.py-tpl | 4 +- .../action_test/test_action.py | 2 +- .../requirements_test/test_requirements.py | 129 ++++++--- .../requirements_test/test_requirements.py | 255 +++++++++--------- .../user_models/test_is_int_value.py | 13 +- .../test_token_part_in_set_requirement.py | 28 +- .../requirement/test_device_requirements.py | 6 +- 22 files changed, 676 insertions(+), 290 deletions(-) diff --git a/core/basic_models/actions/basic_actions.py b/core/basic_models/actions/basic_actions.py index d0d180a6..611d0595 100644 --- a/core/basic_models/actions/basic_actions.py +++ b/core/basic_models/actions/basic_actions.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio import random from typing import Union, Dict, List, Any, Optional @@ -100,11 +101,56 @@ def build_internal_item(self): async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: result = None - if self.requirement.check(text_preprocessing_result, user, params): + if await self.requirement.check(text_preprocessing_result, user, params): result = await self.internal_item.run(user, text_preprocessing_result, params) return result +class GatherChoiceAction(Action): + version: Optional[int] + requirement_actions: RequirementAction + else_action: Action + + FIELD_REQUIREMENT_KEY = "requirement_actions" + FIELD_ELSE_KEY = "else_action" + + def __init__(self, items: Dict[str, Any], id: Optional[str] = None): + super(GatherChoiceAction, self).__init__(items, id) + self._requirement_items = items[self.FIELD_REQUIREMENT_KEY] + self._else_item = items.get(self.FIELD_ELSE_KEY) + + self.items = self.build_items() + + if self._else_item: + self.else_item = self.build_else_item() + else: + self.else_item = None + + @list_factory(RequirementAction) + def build_items(self): + return self._requirement_items + + @factory(Action) + def build_else_item(self): + return self._else_item + + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + result = None + choice_is_made = False + check_results = await asyncio.gather( + item.requirement.check(text_preprocessing_result, user, params) for item in self.items) + for i, checked in enumerate(check_results): + if checked: + item = self.items[i] + result = await item.internal_item.run(user, text_preprocessing_result, params) + choice_is_made = True + break + if not choice_is_made and self._else_item: + result = await self.else_item.run(user, text_preprocessing_result, params) + return result + + class ChoiceAction(Action): version: Optional[int] requirement_actions: RequirementAction @@ -138,8 +184,7 @@ async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreproces result = None choice_is_made = False for item in self.items: - checked = item.requirement.check(text_preprocessing_result, user, params) - if checked: + if await item.requirement.check(text_preprocessing_result, user, params): result = await item.internal_item.run(user, text_preprocessing_result, params) choice_is_made = True break @@ -184,7 +229,7 @@ def build_else_item(self): async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, params: Optional[Optional[Dict[str, Union[str, float, int]]]] = None) -> Optional[List[Command]]: result = None - if self.requirement.check(text_preprocessing_result, user, params): + if await self.requirement.check(text_preprocessing_result, user, params): result = await self.item.run(user, text_preprocessing_result, params) elif self._else_item: result = await self.else_item.run(user, text_preprocessing_result, params) diff --git a/core/basic_models/actions/string_actions.py b/core/basic_models/actions/string_actions.py index 9ac2b150..87676711 100644 --- a/core/basic_models/actions/string_actions.py +++ b/core/basic_models/actions/string_actions.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio import random from copy import copy from typing import Union, Dict, List, Any, Optional @@ -262,6 +263,191 @@ async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreproces return result +class GatherSDKAnswerToUser(NodeAction): + """ + Example: + { + "type": "sdk_answer_to_user", + "root": + [ + { + "type": "pronounce_text", + "text": "ans" + } + ], + "static": { + "ios_card": { + "type": "list_card", + "cells": [ + { + "ios_params": "ios" + } + ] + }, + "android_card": { + "type": "list_card", + "cells": [ + { + "android_params": "android" + } + ] + }, + "tittle1": "static tittle1", + "tittle2": "static tittle2", + "sg_dl": "www.www.www", + "sg_text": "static suggest text" + }, + "random_choice": [ + { + "ans": "random text1" + }, + { + "ans": "random text2" + } + ], + "items": [ + { + "type": "item_card", + "value": "ios_card", + "requirement": { + "type": "external", + "requirement": "OCTOBER_iOS" + } + }, + { + "type": "item_card", + "value": "android_card", + "requirement": { + "type": "external", + "requirement": "OCTOBER_android" + } + }, + { + "type": "bubble_text", + "text": "ans" + } + ], + "suggestions": [ + { + "type": "suggest_text", + "text": "sg_text", + "title": "tittle1" + }, + { + "type": "suggest_deeplink", + "text": "sg_text", + "deep_link": "tittle1" + } + ] + } + + Output: + { + "items": + [{ + "card": {"type": "list_card", "cells": [{"ios_params": "ios"}]}}, + {"bubble": {"text": "random texti", "markdown": True} + }], + "suggestions": + {"buttons": + [ + {"title": "static tittle1", "action": {"text": "static suggest text", "type": "text"}}, + {"title": "static tittle2", "action": {"deep_link": "www.www.www", "type": "deep_link"}} + ] + }, + "pronounceText": "random texti" + } + ответ c карточками с случайным выбором текстов из random_choice + карточки на андроиде требуют sdk_version не ниже "20.03.0.0" + """ + + ITEMS = "items" + SUGGESTIONS = "suggestions" + SUGGESTIONS_TEMPLATE = "suggestions_template" + BUTTONS = "buttons" + STATIC = "static" + RANDOM_CHOICE = "random_choice" + COMMAND = "command" + ROOT = "root" + + def __init__(self, items: Dict[str, Any], id: Optional[str] = None): + super(GatherSDKAnswerToUser, self).__init__(items, id) + self.command: str = ANSWER_TO_USER + self._nodes[self.STATIC] = items.get(self.STATIC, {}) + self._nodes[self.RANDOM_CHOICE] = items.get(self.RANDOM_CHOICE, {}) + self._nodes[self.SUGGESTIONS] = items.get(self.SUGGESTIONS, {}) + self._nodes[self.SUGGESTIONS_TEMPLATE] = items.get(self.SUGGESTIONS_TEMPLATE, {}) + self._items = items.get(self.ITEMS, {}) + self._suggests = items.get(self.SUGGESTIONS, {}) + self._suggests_template = items.get(self.SUGGESTIONS_TEMPLATE) + self._root = items.get(self.ROOT, {}) + + self.items = self.build_items() + self.suggests = self.build_suggests() + self.root = self.build_root() + + @list_factory(SdkAnswerItem) + def build_items(self): + return self._items + + @list_factory(SdkAnswerItem) + def build_suggests(self): + return self._suggests + + @list_factory(SdkAnswerItem) + def build_root(self): + return self._root + + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]: + + result = [] + params = user.parametrizer.collect(text_preprocessing_result, filter_params={self.COMMAND: self.command}) + rendered = self._get_rendered_tree(self.nodes[self.STATIC], params, self.no_empty_nodes) + if self._nodes[self.RANDOM_CHOICE]: + random_node = random.choice(self.nodes[self.RANDOM_CHOICE]) + rendered_random = self._get_rendered_tree(random_node, params, self.no_empty_nodes) + rendered.update(rendered_random) + out = {} + check_results = await asyncio.gather( + item.requirement.check(text_preprocessing_result, user, params) for item in self.items) + for i, check in enumerate(check_results): + item = self.items[i] + if check: + out.setdefault(self.ITEMS, []).append(item.render(rendered)) + + if self._suggests_template is not None: + out[self.SUGGESTIONS] = self._get_rendered_tree(self.nodes[self.SUGGESTIONS_TEMPLATE], params, + self.no_empty_nodes) + else: + check_results = await asyncio.gather( + suggest.requirement.check(text_preprocessing_result, user, params) for suggest in self.suggests) + for i, check in enumerate(check_results): + suggest = self.suggests[i] + if check: + data_dict = out.setdefault(self.SUGGESTIONS, {self.BUTTONS: []}) + buttons = data_dict[self.BUTTONS] + rendered_text = suggest.render(rendered) + buttons.append(rendered_text) + check_results = await asyncio.gather( + part.requirement.check(text_preprocessing_result, user) for part in self.root) + for i, check in enumerate(check_results): + part = self.root[i] + if check: + out.update(part.render(rendered)) + if rendered or not self.no_empty_nodes: + result = [ + Command( + self.command, + out, + self.id, + request_type=self.request_type, + request_data=self.request_data, + ) + ] + return result + + class SDKAnswerToUser(NodeAction): """ Example: @@ -409,7 +595,7 @@ async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreproces rendered.update(rendered_random) out = {} for item in self.items: - if item.requirement.check(text_preprocessing_result, user, params): + if await item.requirement.check(text_preprocessing_result, user, params): out.setdefault(self.ITEMS, []).append(item.render(rendered)) if self._suggests_template is not None: @@ -417,13 +603,13 @@ async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreproces self.no_empty_nodes) else: for suggest in self.suggests: - if suggest.requirement.check(text_preprocessing_result, user, params): + if await suggest.requirement.check(text_preprocessing_result, user, params): data_dict = out.setdefault(self.SUGGESTIONS, {self.BUTTONS: []}) buttons = data_dict[self.BUTTONS] rendered_text = suggest.render(rendered) buttons.append(rendered_text) for part in self.root: - if part.requirement.check(text_preprocessing_result, user): + if await part.requirement.check(text_preprocessing_result, user): out.update(part.render(rendered)) if rendered or not self.no_empty_nodes: result = [ diff --git a/core/basic_models/requirement/basic_requirements.py b/core/basic_models/requirement/basic_requirements.py index 31f0c78e..1106162f 100644 --- a/core/basic_models/requirement/basic_requirements.py +++ b/core/basic_models/requirement/basic_requirements.py @@ -1,3 +1,4 @@ +import asyncio import hashlib from datetime import datetime, timezone from random import random @@ -38,8 +39,8 @@ def _log_params(self): "requirement": self.__class__.__name__ } - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: return True def on_check_error(self, text_preprocessing_result, user): @@ -63,20 +64,42 @@ def build_requirements(self): return self._requirements +class GatherAndRequirement(CompositeRequirement): + + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: + check_results = await asyncio.gather( + requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) + for requirement in self.requirements) + return all(check_results) + + class AndRequirement(CompositeRequirement): - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: - return all(requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) - for requirement in self.requirements) + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: + return all( + await requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) + for requirement in self.requirements) + + +class GatherOrRequirement(CompositeRequirement): + + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: + check_results = await asyncio.gather( + requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) + for requirement in self.requirements) + return any(check_results) class OrRequirement(CompositeRequirement): - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: - return any(requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) - for requirement in self.requirements) + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: + return any( + await requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) + for requirement in self.requirements) class NotRequirement(Requirement): @@ -91,9 +114,10 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: def build_requirement(self): return self._requirement - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: - return not self.requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: + return not await self.requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, + params=params) class ComparisonRequirement(Requirement): @@ -116,8 +140,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: super(RandomRequirement, self).__init__(items, id) self.percent = items["percent"] - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: result = random() * 100 return result < self.percent @@ -129,8 +153,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: super(TopicRequirement, self).__init__(items, id) self.topics = items["topics"] - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: return user.message.topic_key in self.topics @@ -139,8 +163,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: super(TemplateRequirement, self).__init__(items, id) self._template = UnifiedTemplate(items["template"]) - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: params = params or {} collected = user.parametrizer.collect(text_preprocessing_result) params.update(collected) @@ -160,8 +184,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: super(RollingRequirement, self).__init__(items, id) self.percent = items["percent"] - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: id = user.id s = id.encode('utf-8') hash = int(hashlib.sha256(s).hexdigest(), 16) @@ -173,7 +197,7 @@ class TimeRequirement(ComparisonRequirement): def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: super().__init__(items, id) - def check( + async def check( self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, @@ -199,7 +223,7 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: super().__init__(items, id) self.match_cron = items['match_cron'] - def check( + async def check( self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, @@ -226,7 +250,7 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: id, ) - def check( + async def check( self, text_preprocessing_result: TextPreprocessingResult, user: User, @@ -252,8 +276,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: def classifier(self) -> Classifier: return ExternalClassifier(self._classifier) - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: check_res = True classifier = self.classifier with StatsTimer() as timer: @@ -279,8 +303,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: self.field_name = items["field_name"] self.value = items["value"] - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> bool: return user.forms[self.form_name].fields[self.field_name].value == self.value @@ -300,6 +324,6 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: # Если среда исполнения задана, то проверям, что среда в списке возможных значений для сценария, иначе - False self.check_result = self.environment in self.values if self.environment else False - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: return self.check_result diff --git a/core/basic_models/requirement/counter_requirements.py b/core/basic_models/requirement/counter_requirements.py index 1619a8b8..dce4a532 100644 --- a/core/basic_models/requirement/counter_requirements.py +++ b/core/basic_models/requirement/counter_requirements.py @@ -18,8 +18,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: items = items or {} self.key = items["key"] - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: counter = user.counters[self.key] return self.operator.compare(counter) @@ -34,7 +34,7 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: self.key = items["key"] self.fallback_value = items.get("fallback_value") or False - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: _time = user.counters[self.key].update_time return self.operator.compare(time() - _time) if _time else self.fallback_value diff --git a/core/basic_models/requirement/device_requirements.py b/core/basic_models/requirement/device_requirements.py index 8323b89d..2ba2f71d 100644 --- a/core/basic_models/requirement/device_requirements.py +++ b/core/basic_models/requirement/device_requirements.py @@ -23,8 +23,8 @@ def descr_to_check_in(self): def get_field(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser): return NotImplementedError - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: return self.get_field(text_preprocessing_result, user) in self.descr_to_check_in @@ -51,8 +51,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: items = items or {} self.platfrom_type = items["platfrom_type"] - def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: return user.message.device.platform_type == self.platfrom_type @@ -67,8 +67,8 @@ def build_operator(self): class PlatformVersionRequirement(BasicVersionRequirement): - def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: platform_version = convert_version_to_list_of_int(user.message.device.platform_version) return self.operator.compare(platform_version) if platform_version is not None else False @@ -80,15 +80,15 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: items = items or {} self.surface = items["surface"] - def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: return user.message.device.surface == self.surface class SurfaceVersionRequirement(BasicVersionRequirement): - def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: surface_version = convert_version_to_list_of_int(user.message.device.surface_version) return self.operator.compare(surface_version) if surface_version is not None else False @@ -100,8 +100,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: items = items or {} self.app_type = items["app_type"] - def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: return self.app_type in user.message.device.features.get("appTypes", []) @@ -112,6 +112,6 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: items = items or {} self.property_type = items["property_type"] - def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: TextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: return user.message.device.capabilities.get(self.property_type, {}).get("available", False) diff --git a/core/basic_models/requirement/external_requirements.py b/core/basic_models/requirement/external_requirements.py index e5aace78..a0660ce0 100644 --- a/core/basic_models/requirement/external_requirements.py +++ b/core/basic_models/requirement/external_requirements.py @@ -20,7 +20,7 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: super(ExternalRequirement, self).__init__(items, id) self.requirement = items["requirement"] - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: requirement = user.descriptions["external_requirements"][self.requirement] - return requirement.check(text_preprocessing_result, user, params) + return await requirement.check(text_preprocessing_result, user, params) diff --git a/core/basic_models/requirement/project_requirements.py b/core/basic_models/requirement/project_requirements.py index 3114ce1d..5f1203b5 100644 --- a/core/basic_models/requirement/project_requirements.py +++ b/core/basic_models/requirement/project_requirements.py @@ -14,6 +14,6 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: self._key = items["key"] self._value = items["value"] - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: return user.settings[self._config][self._key] == self._value diff --git a/core/basic_models/requirement/user_text_requirements.py b/core/basic_models/requirement/user_text_requirements.py index 8cbbf415..dec25d58 100644 --- a/core/basic_models/requirement/user_text_requirements.py +++ b/core/basic_models/requirement/user_text_requirements.py @@ -18,8 +18,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: super(AnySubstringInLoweredTextRequirement, self).__init__(items, id) self.substrings = self.items["substrings"] - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: lowered_text = text_preprocessing_result.lower() if isinstance(text_preprocessing_result, str) \ else text_preprocessing_result.raw["original_text"].lower() return any(s.lower() in lowered_text for s in self.substrings) @@ -49,8 +49,8 @@ class IntersectionWithTokensSetRequirement(NormalizedInputWordsRequirement): Слова из input_words также проходят нормализацию перед сравнением. """ - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: words_normalized_set = set([ token["lemma"] for token in text_preprocessing_result.raw["tokenized_elements_list_pymorphy"] if not token.get("token_type") == "SENTENCE_ENDPOINT_TOKEN" @@ -71,8 +71,8 @@ class NormalizedTextInSetRequirement(NormalizedInputWordsRequirement): нормализованных строк из input_words, иначе - False. """ - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: normalized_text = text_preprocessing_result.raw["normalized_text"].replace(".", "").strip() result = normalized_text in self.normalized_input_words if result: @@ -94,8 +94,8 @@ class PhoneNumberNumberRequirement(ComparisonRequirement): def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: super().__init__(items, id) - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, + params: Dict[str, Any] = None) -> bool: len_phone_number_token = len(text_preprocessing_result.get_token_values_by_type("PHONE_NUMBER_TOKEN")) result = self.operator.compare(len_phone_number_token) if result: @@ -114,7 +114,7 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: self.min_num = float(items["min_num"]) self.max_num = float(items["max_num"]) - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> bool: num = float(text_preprocessing_result.num_token_values) return self.min_num <= num <= self.max_num if num else False diff --git a/core/basic_models/scenarios/base_scenario.py b/core/basic_models/scenarios/base_scenario.py index 09d1023e..19f2d101 100644 --- a/core/basic_models/scenarios/base_scenario.py +++ b/core/basic_models/scenarios/base_scenario.py @@ -48,7 +48,7 @@ def build_available_requirement(self): def check_available(self, text_preprocessing_result, user): if not self.switched_off: - return self.available_requirement.check(text_preprocessing_result, user) + return await self.available_requirement.check(text_preprocessing_result, user) return False def _log_params(self): diff --git a/scenarios/actions/action.py b/scenarios/actions/action.py index fafa66ca..e01326f0 100644 --- a/scenarios/actions/action.py +++ b/scenarios/actions/action.py @@ -1,3 +1,4 @@ +import asyncio import collections import copy import json @@ -354,7 +355,52 @@ async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessing choice_is_made = False for scenario, requirement in zip(self._scenarios, self.requirement_items): - check_res = requirement.check(text_preprocessing_result, user, params) + check_res = await requirement.check(text_preprocessing_result, user, params) + if check_res: + result = await RunScenarioAction(items=scenario).run(user, text_preprocessing_result, params) + choice_is_made = True + break + + if not choice_is_made and self._else_item: + result = await self.else_item.run(user, text_preprocessing_result, params) + + return result + + +class GatherChoiceScenarioAction(Action): + FIELD_SCENARIOS_KEY = "scenarios" + FIELD_ELSE_KEY = "else_action" + FIELD_REQUIREMENT_KEY = "requirement" + + def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: + super(GatherChoiceScenarioAction, self).__init__(items, id) + self._else_item = items.get(self.FIELD_ELSE_KEY) + self._scenarios = items[self.FIELD_SCENARIOS_KEY] + self._requirements = [scenario.pop(self.FIELD_REQUIREMENT_KEY) for scenario in self._scenarios] + + self.requirement_items = self.build_requirement_items() + + if self._else_item: + self.else_item = self.build_else_item() + else: + self.else_item = None + + @list_factory(Requirement) + def build_requirement_items(self): + return self._requirements + + @factory(Action) + def build_else_item(self): + return self._else_item + + async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Union[None, str, List[Command]]: + result = None + choice_is_made = False + + check_results = await asyncio.gather(requirement.check(text_preprocessing_result, user, params) + for requirement in self.requirement_items) + for scenario, check_res in zip(self._scenarios, check_results): if check_res: result = await RunScenarioAction(items=scenario).run(user, text_preprocessing_result, params) choice_is_made = True diff --git a/scenarios/requirements/requirements.py b/scenarios/requirements/requirements.py index 70c9a62a..b41efb1e 100644 --- a/scenarios/requirements/requirements.py +++ b/scenarios/requirements/requirements.py @@ -10,8 +10,8 @@ class AskAgainExistRequirement(Requirement): - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> bool: last_scenario_id = user.last_scenarios.last_scenario_name scenario = user.descriptions["scenarios"].get(last_scenario_id) return scenario.check_ask_again_question(text_preprocessing_result, user, params) @@ -23,8 +23,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: self._template = UnifiedTemplate(items["template"]) self._items = set(items["items"]) - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> bool: params = params or {} collected = user.parametrizer.collect(text_preprocessing_result) params.update(collected) @@ -38,8 +38,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: self._template = UnifiedTemplate(items["template"]) self._items = set(items["items"]) - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> bool: params = params or {} collected = user.parametrizer.collect(text_preprocessing_result) params.update(collected) @@ -56,8 +56,8 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: self._template = UnifiedTemplate(items["template"]) self._regexp = re.compile(items["regexp"], re.S | re.M) - def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, + params: Dict[str, Any] = None) -> bool: params = params or {} collected = user.parametrizer.collect(text_preprocessing_result) params.update(collected) diff --git a/scenarios/scenario_descriptions/form_filling_scenario.py b/scenarios/scenario_descriptions/form_filling_scenario.py index a7e3e7ec..34f21060 100644 --- a/scenarios/scenario_descriptions/form_filling_scenario.py +++ b/scenarios/scenario_descriptions/form_filling_scenario.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio from typing import Dict, Any from core.basic_models.scenarios.base_scenario import BaseScenario @@ -59,10 +60,10 @@ def _field(self, form, text_preprocessing_result, user, params): def _find_field(self, form, text_preprocessing_result, user, params): for field_name in form.fields.descriptions: field = form.fields[field_name] + loop = asyncio.get_event_loop() if not field.valid and field.description.has_requests and \ - field.description.requirement.check( - text_preprocessing_result, user, params - ): + loop.run_until_complete( + field.description.requirement.check(text_preprocessing_result, user, params)): return field def get_fields_data(self, form, form_key): @@ -77,7 +78,7 @@ def _clean_key(self, key: str): async def _extract_by_field_filler(self, field_key, field_descr, text_normalization_result, user, params): result = {} - check = field_descr.requirement.check(text_normalization_result, user, params) + check = await field_descr.requirement.check(text_normalization_result, user, params) log_params = self._log_params() log_params["requirement"] = field_descr.requirement.__class__.__name__, log_params["check"] = check @@ -118,8 +119,10 @@ def _validate_extracted_data(self, user, text_preprocessing_result, form, data_e error_msgs = [] for field_key, field in form.description.fields.items(): value = data_extracted.get(field_key) + loop = asyncio.get_event_loop() # is not None is necessary, because 0 and False should be checked, None - shouldn't fill - if value is not None and not field.field_validator.requirement.check(value, params): + if value is not None and \ + not loop.run_until_complete(field.field_validator.requirement.check(value, params)): log_params = { log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE, "field_key": field_key diff --git a/scenarios/scenario_models/field/field.py b/scenarios/scenario_models/field/field.py index c44e9335..ea37837d 100644 --- a/scenarios/scenario_models/field/field.py +++ b/scenarios/scenario_models/field/field.py @@ -40,10 +40,10 @@ def can_be_updated(self): def check_can_be_filled(self, text_preprocessing_result, user): loop = asyncio.get_event_loop() - return ( - self.description.requirement.check(text_preprocessing_result, user) and - loop.run_until_complete(self.description.filler.run(user, text_preprocessing_result)) is not None - ) + check, run = loop.run_until_complete(asyncio.gather( + self.description.requirement.check(text_preprocessing_result, user), + self.description.filler.run(user, text_preprocessing_result))) + return check and run is not None @property def valid(self): diff --git a/scenarios/scenario_models/field_requirements/field_requirements.py b/scenarios/scenario_models/field_requirements/field_requirements.py index 7759491c..e4b5aaeb 100644 --- a/scenarios/scenario_models/field_requirements/field_requirements.py +++ b/scenarios/scenario_models/field_requirements/field_requirements.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio from typing import Dict, List, Optional, Any, Set from core.basic_models.operators.operators import Operator @@ -14,7 +15,7 @@ class FieldRequirement: def __init__(self, items: Optional[Dict[str, Any]]) -> None: pass - def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: return True @@ -32,13 +33,27 @@ def build_requirements(self): class AndFieldRequirement(CompositeFieldRequirement): - def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: - return all(requirement.check(field_value=field_value, params=params) for requirement in self.requirements) + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + return all(await requirement.check(field_value=field_value, params=params) for requirement in self.requirements) + + +class GatherAndFieldRequirement(CompositeFieldRequirement): + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + check_results = await asyncio.gather(requirement.check(field_value=field_value, params=params) + for requirement in self.requirements) + return all(check_results) class OrFieldRequirement(CompositeFieldRequirement): - def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: - return any(requirement.check(field_value=field_value, params=params) for requirement in self.requirements) + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + return any(await requirement.check(field_value=field_value, params=params) for requirement in self.requirements) + + +class GatherOrFieldRequirement(CompositeFieldRequirement): + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + check_results = await asyncio.gather(requirement.check(field_value=field_value, params=params) + for requirement in self.requirements) + return any(check_results) class NotFieldRequirement(FieldRequirement): @@ -53,8 +68,8 @@ def __init__(self, items: Optional[Dict[str, Any]]) -> None: def build_requirement(self): return self._requirement - def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: - return not self.requirement.check(field_value=field_value, params=params) + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + return not await self.requirement.check(field_value=field_value, params=params) class ComparisonFieldRequirement(FieldRequirement): @@ -69,7 +84,7 @@ def __init__(self, items: Optional[Dict[str, Any]]) -> None: def build_operator(self): return self._operator - def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: return self.operator.compare(field_value) @@ -77,7 +92,7 @@ class IsIntFieldRequirement(FieldRequirement): def __init__(self, items: Optional[Dict[str, Any]]) -> None: super(IsIntFieldRequirement, self).__init__(items) - def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: try: int(field_value) return True @@ -92,7 +107,7 @@ def __init__(self, items: Optional[Dict[str, Any]]) -> None: super(ValueInSetRequirement, self).__init__(items) self.symbols: Set = set(items["symbols"]) - def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: return field_value in self.symbols @@ -102,7 +117,7 @@ def __init__(self, items: Optional[Dict[str, Any]]) -> None: self.part = items['part'] self.values = items['values'] - def check(self, field_value: dict, params: Dict[str, Any] = None) -> bool: + async def check(self, field_value: dict, params: Dict[str, Any] = None) -> bool: return field_value[self.part] in self.values @@ -115,5 +130,5 @@ def __init__(self, items: Optional[Dict[str, Any]]) -> None: self.min_field_length = items["min_field_length"] self.max_field_length = items["max_field_length"] - def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: + async def check(self, field_value: str, params: Dict[str, Any] = None) -> bool: return self.min_field_length <= len(field_value) <= self.max_field_length diff --git a/scenarios/user/last_scenarios/last_scenarios_description.py b/scenarios/user/last_scenarios/last_scenarios_description.py index 12638a95..a4b21706 100644 --- a/scenarios/user/last_scenarios/last_scenarios_description.py +++ b/scenarios/user/last_scenarios/last_scenarios_description.py @@ -1,4 +1,6 @@ # coding: utf-8 +import asyncio + from lazy import lazy from core.basic_models.requirement.basic_requirements import Requirement @@ -18,5 +20,7 @@ def requirement(self): return self._requirement def check(self, text_preprocessing_result, user): - return user.message.channel in self._channels and self.requirement.check(text_preprocessing_result, user) if \ - self._channels else self.requirement.check(text_preprocessing_result, user) + loop = asyncio.get_event_loop() + return user.message.channel in self._channels and \ + loop.run_until_complete(self.requirement.check(text_preprocessing_result, user)) if self._channels else \ + loop.run_until_complete(self.requirement.check(text_preprocessing_result, user)) diff --git a/smart_kit/template/app/basic_entities/requirements.py-tpl b/smart_kit/template/app/basic_entities/requirements.py-tpl index 14d0c77c..153743f7 100644 --- a/smart_kit/template/app/basic_entities/requirements.py-tpl +++ b/smart_kit/template/app/basic_entities/requirements.py-tpl @@ -18,6 +18,6 @@ class CustomRequirement(Requirement): items = items or {} self.test_param = items.get("test_param") - def check(self, text_preprocessing_result: TextPreprocessingResult, - user: User, params: Dict[str, Any] = None) -> bool: + async def check(self, text_preprocessing_result: TextPreprocessingResult, + user: User, params: Dict[str, Any] = None) -> bool: return False diff --git a/tests/core_tests/basic_scenario_models_test/action_test/test_action.py b/tests/core_tests/basic_scenario_models_test/action_test/test_action.py index ccad7f34..2ae3a7fe 100644 --- a/tests/core_tests/basic_scenario_models_test/action_test/test_action.py +++ b/tests/core_tests/basic_scenario_models_test/action_test/test_action.py @@ -58,7 +58,7 @@ class MockRequirement: def __init__(self, items): self.result = items.get("result") - def check(self, text_preprocessing_result, user, params): + async def check(self, text_preprocessing_result, user, params): return self.result diff --git a/tests/core_tests/requirements_test/test_requirements.py b/tests/core_tests/requirements_test/test_requirements.py index a2494a9d..9ab03f61 100644 --- a/tests/core_tests/requirements_test/test_requirements.py +++ b/tests/core_tests/requirements_test/test_requirements.py @@ -1,3 +1,4 @@ +import asyncio import os import unittest from time import time @@ -34,7 +35,7 @@ def __init__(self, items=None): items = items or {} self.cond = items.get("cond") or False - def check(self, text_preprocessing_result, user, params): + async def check(self, text_preprocessing_result, user, params): return self.cond @@ -80,7 +81,8 @@ def compare(self, value): class RequirementTest(unittest.TestCase): def test_base(self): requirement = Requirement(None) - assert requirement.check(None, None) + loop = asyncio.get_event_loop() + assert loop.run_until_complete(requirement.check(None, None)) def test_composite(self): registered_factories[Requirement] = MockRequirement @@ -89,7 +91,8 @@ def test_composite(self): {"cond": True} ]}) self.assertEqual(len(requirement.requirements), 2) - self.assertTrue(requirement.check(None, None)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, None))) def test_and_success(self): registered_factories[Requirement] = MockRequirement @@ -97,7 +100,8 @@ def test_and_success(self): {"cond": True}, {"cond": True} ]}) - self.assertTrue(requirement.check(None, None)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, None))) def test_and_fail(self): registered_factories[Requirement] = MockRequirement @@ -105,7 +109,8 @@ def test_and_fail(self): {"cond": True}, {"cond": False} ]}) - self.assertFalse(requirement.check(None, None)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(None, None))) def test_or_success(self): registered_factories[Requirement] = MockRequirement @@ -113,7 +118,8 @@ def test_or_success(self): {"cond": True}, {"cond": False} ]}) - self.assertTrue(requirement.check(None, None)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, None))) def test_or_fail(self): registered_factories[Requirement] = MockRequirement @@ -121,17 +127,20 @@ def test_or_fail(self): {"cond": False}, {"cond": False} ]}) - self.assertFalse(requirement.check(None, None)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(None, None))) def test_not_success(self): registered_factories[Requirement] = MockRequirement requirement = NotRequirement({"requirement": {"cond": False}}) - self.assertTrue(requirement.check(None, None)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, None))) def test_not_fail(self): registered_factories[Requirement] = MockRequirement requirement = NotRequirement({"requirement": {"cond": True}}) - self.assertFalse(requirement.check(None, None)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(None, None))) def test_channel_success(self): user = Mock() @@ -139,7 +148,8 @@ def test_channel_success(self): user.message = message requirement = ChannelRequirement({"channels": ["ch1"]}) text_normalization_result = None - self.assertTrue(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) def test_channel_fail(self): user = Mock() @@ -147,15 +157,18 @@ def test_channel_fail(self): user.message = message requirement = ChannelRequirement({"channels": ["ch1"]}) text_normalization_result = None - self.assertFalse(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) def test_random_requirement_true(self): requirement = RandomRequirement({"percent": 100}) - self.assertTrue(requirement.check(None, None)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, None))) def test_random_requirement_false(self): requirement = RandomRequirement({"percent": 0}) - self.assertFalse(requirement.check(None, None)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(None, None))) def test_topic_requirement(self): requirement = TopicRequirement({"topics": ["test"]}) @@ -163,7 +176,8 @@ def test_topic_requirement(self): message = Mock() message.topic_key = "test" user.message = message - self.assertTrue(requirement.check(None, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, user))) def test_counter_value_requirement(self): registered_factories[Operator] = MockAmountOperator @@ -172,7 +186,8 @@ def test_counter_value_requirement(self): counter.__gt__ = Mock(return_value=True) user.counters = {"test": counter} requirement = CounterValueRequirement({"operator": {"type": "equal", "amount": 2}, "key": "test"}) - self.assertTrue(requirement.check(None, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, user))) def test_counter_time_requirement(self): registered_factories[Operator] = MockAmountOperator @@ -181,7 +196,8 @@ def test_counter_time_requirement(self): counter.update_time = int(time()) - 10 user.counters = {"test": counter} requirement = CounterUpdateTimeRequirement({"operator": {"type": "more_or_equal", "amount": 5}, "key": "test"}) - self.assertTrue(requirement.check(None, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, user))) def test_template_req_true(self): items = { @@ -196,7 +212,8 @@ def test_template_req_true(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - self.assertTrue(requirement.check(None, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, user))) def test_template_req_false(self): items = { @@ -207,7 +224,8 @@ def test_template_req_false(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - self.assertFalse(requirement.check(None, user)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(None, user))) def test_template_req_raise(self): items = { @@ -225,14 +243,16 @@ def test_rolling_requirement_true(self): user.id = "353454" requirement = RollingRequirement({"percent": 100}) text_normalization_result = None - self.assertTrue(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) def test_rolling_requirement_false(self): user = Mock() user.id = "353454" requirement = RollingRequirement({"percent": 0}) text_normalization_result = None - self.assertFalse(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) def test_time_requirement_true(self): user = Mock() @@ -254,7 +274,8 @@ def test_time_requirement_true(self): } ) text_normalization_result = None - self.assertTrue(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) def test_time_requirement_false(self): user = Mock() @@ -276,7 +297,8 @@ def test_time_requirement_false(self): } ) text_normalization_result = None - self.assertFalse(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) def test_datetime_requirement_true(self): user = Mock() @@ -295,7 +317,8 @@ def test_datetime_requirement_true(self): } ) text_normalization_result = None - self.assertTrue(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) def test_datetime_requirement_false(self): user = Mock() @@ -314,7 +337,8 @@ def test_datetime_requirement_false(self): } ) text_normalization_result = None - self.assertFalse(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) @patch('smart_kit.configs.get_app_config') def test_intersection_requirement_true(self, mock_get_app_config): @@ -334,7 +358,8 @@ def test_intersection_requirement_true(self, mock_get_app_config): {'lemma': 'я'}, {'lemma': 'хотеть'}, ] - self.assertTrue(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) @patch('smart_kit.configs.get_app_config') def test_intersection_requirement_false(self, mock_get_app_config): @@ -355,7 +380,8 @@ def test_intersection_requirement_false(self, mock_get_app_config): {'lemma': 'за'}, {'lemma': 'что'}, ] - self.assertFalse(requirement.check(text_normalization_result, user)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) @patch.object(ExternalClassifier, "find_best_answer", return_value=[{"answer": "нет", "score": 1.0, "other": False}]) def test_classifier_requirement_true(self, mock_classifier_model): @@ -366,7 +392,8 @@ def test_classifier_requirement_true(self, mock_classifier_model): classifier_requirement = ClassifierRequirement(test_items) mock_user = Mock() mock_user.descriptions = {"external_classifiers": ["read_book_or_not_classifier", "hello_scenario_classifier"]} - result = classifier_requirement.check(Mock(), mock_user) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(classifier_requirement.check(Mock(), mock_user)) self.assertTrue(result) @patch.object(ExternalClassifier, "find_best_answer", return_value=[]) @@ -376,7 +403,8 @@ def test_classifier_requirement_false(self, mock_classifier_model): classifier_requirement = ClassifierRequirement(test_items) mock_user = Mock() mock_user.descriptions = {"external_classifiers": ["read_book_or_not_classifier", "hello_scenario_classifier"]} - result = classifier_requirement.check(Mock(), mock_user) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(classifier_requirement.check(Mock(), mock_user)) self.assertFalse(result) @patch.object(ExternalClassifier, "find_best_answer", return_value=[{"answer": "other", "score": 1.0, "other": True}]) @@ -386,7 +414,8 @@ def test_classifier_requirement_false_if_class_other(self, mock_classifier_model classifier_requirement = ClassifierRequirement(test_items) mock_user = Mock() mock_user.descriptions = {"external_classifiers": ["read_book_or_not_classifier", "hello_scenario_classifier"]} - result = classifier_requirement.check(Mock(), mock_user) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(classifier_requirement.check(Mock(), mock_user)) self.assertFalse(result) def test_form_field_value_requirement_true(self): @@ -405,7 +434,8 @@ def test_form_field_value_requirement_true(self): user.forms[form_name].fields = {form_field: Mock(), "value": field_value} user.forms[form_name].fields[form_field].value = field_value - result = req_form_field_value.check(Mock(), user) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(req_form_field_value.check(Mock(), user)) self.assertTrue(result) def test_form_field_value_requirement_false(self): @@ -424,7 +454,8 @@ def test_form_field_value_requirement_false(self): user.forms[form_name].fields = {form_field: Mock(), "value": "OTHER_TEST_VAL"} user.forms[form_name].fields[form_field].value = "OTHER_TEST_VAL" - result = req_form_field_value.check(Mock(), user) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(req_form_field_value.check(Mock(), user)) self.assertFalse(result) @patch("smart_kit.configs.get_app_config") @@ -432,14 +463,16 @@ def test_environment_requirement_true(self, mock_get_app_config): """Тест кейз проверяет что условие возвращает True, т.к среда исполнения из числа values.""" patch_get_app_config(mock_get_app_config) environment_req = EnvironmentRequirement({"values": ["ift", "uat"]}) - self.assertTrue(environment_req.check(Mock(), Mock())) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(environment_req.check(Mock(), Mock()))) @patch("smart_kit.configs.get_app_config") def test_environment_requirement_false(self, mock_get_app_config): """Тест кейз проверяет что условие возвращает False, т.к среда исполнения НЕ из числа values.""" patch_get_app_config(mock_get_app_config) environment_req = EnvironmentRequirement({"values": ["uat", "pt"]}) - self.assertFalse(environment_req.check(Mock(), Mock())) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(environment_req.check(Mock(), Mock()))) def test_any_substring_in_lowered_text_requirement_true(self): """Тест кейз проверяет что условие возвращает True, т.к нашлась подстрока из списка substrings, которая @@ -448,7 +481,8 @@ def test_any_substring_in_lowered_text_requirement_true(self): req = AnySubstringInLoweredTextRequirement({"substrings": ["искомая подстрока", "другое знанчение"]}) text_preprocessing_result = Mock() text_preprocessing_result.raw = {"original_text": "КАКОЙ-ТО ТЕКСТ С ИСКОМАЯ ПОДСТРОКА"} - result = req.check(text_preprocessing_result, Mock()) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(req.check(text_preprocessing_result, Mock())) self.assertTrue(result) def test_any_substring_in_lowered_text_requirement_false(self): @@ -458,7 +492,8 @@ def test_any_substring_in_lowered_text_requirement_false(self): req = AnySubstringInLoweredTextRequirement({"substrings": ["искомая подстрока", "другая подстрока"]}) text_preprocessing_result = Mock() text_preprocessing_result.raw = {"original_text": "КАКОЙ-ТО ТЕКСТ"} - result = req.check(text_preprocessing_result, Mock()) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(req.check(text_preprocessing_result, Mock())) self.assertFalse(result) def test_num_in_range_requirement_true(self): @@ -466,28 +501,32 @@ def test_num_in_range_requirement_true(self): req = NumInRangeRequirement({"min_num": "5", "max_num": "10"}) text_preprocessing_result = Mock() text_preprocessing_result.num_token_values = 7 - self.assertTrue(req.check(text_preprocessing_result, Mock())) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) def test_num_in_range_requirement_false(self): """Тест кейз проверяет что условие возвращает False, т.к число НЕ находится в заданном диапазоне.""" req = NumInRangeRequirement({"min_num": "5", "max_num": "10"}) text_preprocessing_result = Mock() text_preprocessing_result.num_token_values = 20 - self.assertFalse(req.check(text_preprocessing_result, Mock())) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) def test_phone_number_number_requirement_true(self): """Тест кейз проверяет что условие возвращает True, т.к кол-во номеров телефонов больше заданного.""" req = PhoneNumberNumberRequirement({"operator": {"type": "more", "amount": 1}}) text_preprocessing_result = Mock() text_preprocessing_result.get_token_values_by_type.return_value = ["89030478799", "89092534523"] - self.assertTrue(req.check(text_preprocessing_result, Mock())) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) def test_phone_number_number_requirement_false(self): """Тест кейз проверяет что условие возвращает False, т.к кол-во номеров телефонов НЕ больше заданного.""" req = PhoneNumberNumberRequirement({"operator": {"type": "more", "amount": 10}}) text_preprocessing_result = Mock() text_preprocessing_result.get_token_values_by_type.return_value = ["89030478799"] - self.assertFalse(req.check(text_preprocessing_result, Mock())) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) @patch("smart_kit.configs.get_app_config") def test_intersection_with_tokens_requirement_true(self, mock_get_app_config): @@ -509,7 +548,8 @@ def test_intersection_with_tokens_requirement_true(self, mock_get_app_config): "part_of_speech": "NOUN"}, "lemma": "погода"} ]} - self.assertTrue(req.check(text_preprocessing_result, Mock())) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) @patch("smart_kit.configs.get_app_config") def test_intersection_with_tokens_requirement_false(self, mock_get_app_config): @@ -531,7 +571,8 @@ def test_intersection_with_tokens_requirement_false(self, mock_get_app_config): "part_of_speech": "NOUN"}, "lemma": "погода"} ]} - self.assertFalse(req.check(text_preprocessing_result, Mock())) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) @patch("smart_kit.configs.get_app_config") def test_normalized_text_in_set_requirement_true(self, mock_get_app_config): @@ -545,7 +586,8 @@ def test_normalized_text_in_set_requirement_true(self, mock_get_app_config): text_preprocessing_result = Mock() text_preprocessing_result.raw = {"normalized_text": "погода ."} - self.assertTrue(req.check(text_preprocessing_result, Mock())) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) @patch("smart_kit.configs.get_app_config") def test_normalized_text_in_set_requirement_false(self, mock_get_app_config): @@ -559,7 +601,8 @@ def test_normalized_text_in_set_requirement_false(self, mock_get_app_config): text_preprocessing_result = Mock() text_preprocessing_result.raw = {"normalized_text": "хотеть узнать ."} - self.assertFalse(req.check(text_preprocessing_result, Mock())) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) if __name__ == '__main__': diff --git a/tests/scenarios_tests/requirements_test/test_requirements.py b/tests/scenarios_tests/requirements_test/test_requirements.py index 242a359b..a8c4c5c6 100644 --- a/tests/scenarios_tests/requirements_test/test_requirements.py +++ b/tests/scenarios_tests/requirements_test/test_requirements.py @@ -1,8 +1,10 @@ # coding: utf-8 +import asyncio import unittest from unittest.mock import Mock -from scenarios.requirements.requirements import TemplateInArrayRequirement, ArrayItemInTemplateRequirement, RegexpInTemplateRequirement +from scenarios.requirements.requirements import TemplateInArrayRequirement, ArrayItemInTemplateRequirement, \ + RegexpInTemplateRequirement class MockRequirement: @@ -10,7 +12,7 @@ def __init__(self, items=None): items = items or {} self.cond = items.get("cond") or False - def check(self, text_preprocessing_result, user): + async def check(self, text_preprocessing_result, user): return self.cond @@ -70,144 +72,145 @@ def test_template_in_array_req_true(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - self.assertTrue(requirement.check(None, user)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, user))) def test_template_in_array_req_true2(self): - items = { - "template": "{{ payload.message.strip() }}", - "items": ["AAA", "BBB", "CCC"] - } - requirement = TemplateInArrayRequirement(items) - params = {"payload": { - "userInfo": { - "tbcode": "32" - }, - "message": " BBB " - }} - user = Mock() - user.parametrizer = Mock() - user.parametrizer.collect = Mock(return_value=params) - self.assertTrue(requirement.check(None, user)) + items = { + "template": "{{ payload.message.strip() }}", + "items": ["AAA", "BBB", "CCC"] + } + requirement = TemplateInArrayRequirement(items) + params = {"payload": { + "userInfo": { + "tbcode": "32" + }, + "message": " BBB " + }} + user = Mock() + user.parametrizer = Mock() + user.parametrizer.collect = Mock(return_value=params) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, user))) def test_template_in_array_req_false(self): - items = { - "template": "{{ payload.message.strip() }}", - "items": ["AAA", "CCC"] - } - requirement = TemplateInArrayRequirement(items) - params = {"payload": { - "userInfo": { - "tbcode": "32", - }, - "message": " BBB " - }} - user = Mock() - user.parametrizer = Mock() - user.parametrizer.collect = Mock(return_value=params) - self.assertFalse(requirement.check(None, user)) - + items = { + "template": "{{ payload.message.strip() }}", + "items": ["AAA", "CCC"] + } + requirement = TemplateInArrayRequirement(items) + params = {"payload": { + "userInfo": { + "tbcode": "32", + }, + "message": " BBB " + }} + user = Mock() + user.parametrizer = Mock() + user.parametrizer.collect = Mock(return_value=params) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(None, user))) def test_array_in_template_req_true(self): - items = { - "template": { - "type": "unified_template", - "template": "{{ payload.userInfo.departcode.split('/')|tojson }}", - "loader": "json" - }, - "items": ["111", "456"] - } - requirement = ArrayItemInTemplateRequirement(items) - params = {"payload": { - "userInfo": { - "tbcode": "32", - "departcode": "123/2345/456" - }, - "message": " BBB " - }} - user = Mock() - user.parametrizer = Mock() - user.parametrizer.collect = Mock(return_value=params) - self.assertTrue(requirement.check(None, user)) - + items = { + "template": { + "type": "unified_template", + "template": "{{ payload.userInfo.departcode.split('/')|tojson }}", + "loader": "json" + }, + "items": ["111", "456"] + } + requirement = ArrayItemInTemplateRequirement(items) + params = {"payload": { + "userInfo": { + "tbcode": "32", + "departcode": "123/2345/456" + }, + "message": " BBB " + }} + user = Mock() + user.parametrizer = Mock() + user.parametrizer.collect = Mock(return_value=params) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, user))) def test_array_in_template_req_true2(self): - items = { - "template": "{{ payload.message.strip() }}", - "items": ["AAA", "BBB"] - } - requirement = ArrayItemInTemplateRequirement(items) - params = {"payload": { - "userInfo": { - "tbcode": "32", - "departcode": "123/2345/456" - }, - "message": " BBB " - }} - user = Mock() - user.parametrizer = Mock() - user.parametrizer.collect = Mock(return_value=params) - self.assertTrue(requirement.check(None, user)) - + items = { + "template": "{{ payload.message.strip() }}", + "items": ["AAA", "BBB"] + } + requirement = ArrayItemInTemplateRequirement(items) + params = {"payload": { + "userInfo": { + "tbcode": "32", + "departcode": "123/2345/456" + }, + "message": " BBB " + }} + user = Mock() + user.parametrizer = Mock() + user.parametrizer.collect = Mock(return_value=params) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, user))) def test_array_in_template_req_false(self): - items = { - "template": { - "type": "unified_template", - "template": "{{ payload.userInfo.departcode.split('/')|tojson }}", - "loader": "json" - }, - "items": ["111", "222"] - } - requirement = ArrayItemInTemplateRequirement(items) - params = {"payload": { - "userInfo": { - "tbcode": "32", - "departcode": "123/2345/456" - }, - "message": " BBB " - }} - user = Mock() - user.parametrizer = Mock() - user.parametrizer.collect = Mock(return_value=params) - self.assertFalse(requirement.check(None, user)) - + items = { + "template": { + "type": "unified_template", + "template": "{{ payload.userInfo.departcode.split('/')|tojson }}", + "loader": "json" + }, + "items": ["111", "222"] + } + requirement = ArrayItemInTemplateRequirement(items) + params = {"payload": { + "userInfo": { + "tbcode": "32", + "departcode": "123/2345/456" + }, + "message": " BBB " + }} + user = Mock() + user.parametrizer = Mock() + user.parametrizer.collect = Mock(return_value=params) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(None, user))) def test_regexp_in_template_req_true(self): - items = { - "template": "{{ payload.message.strip() }}", - "regexp": "(^|\s)[Фф](\.|-)?1(\-)?(у|У)?($|\s)" - } - requirement = RegexpInTemplateRequirement(items) - params = {"payload": { - "userInfo": { - "tbcode": "32", - }, - "message": "карточки ф1у" - }} - user = Mock() - user.parametrizer = Mock() - user.parametrizer.collect = Mock(return_value=params) - self.assertTrue(requirement.check(None, user)) - + items = { + "template": "{{ payload.message.strip() }}", + "regexp": "(^|\s)[Фф](\.|-)?1(\-)?(у|У)?($|\s)" + } + requirement = RegexpInTemplateRequirement(items) + params = {"payload": { + "userInfo": { + "tbcode": "32", + }, + "message": "карточки ф1у" + }} + user = Mock() + user.parametrizer = Mock() + user.parametrizer.collect = Mock(return_value=params) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(None, user))) def test_regexp_in_template_req_false(self): - items = { - "template": "{{ payload.message.strip() }}", - "regexp": "(^|\s)[Фф](\.|-)?1(\-)?(у|У)?($|\s)" - } - requirement = RegexpInTemplateRequirement(items) - params = {"payload": { - "userInfo": { - "tbcode": "32", - }, - "message": "карточки конг фу 1" - }} - user = Mock() - user.parametrizer = Mock() - user.parametrizer.collect = Mock(return_value=params) - self.assertFalse(requirement.check(None, user)) - - + items = { + "template": "{{ payload.message.strip() }}", + "regexp": "(^|\s)[Фф](\.|-)?1(\-)?(у|У)?($|\s)" + } + requirement = RegexpInTemplateRequirement(items) + params = {"payload": { + "userInfo": { + "tbcode": "32", + }, + "message": "карточки конг фу 1" + }} + user = Mock() + user.parametrizer = Mock() + user.parametrizer.collect = Mock(return_value=params) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(None, user))) if __name__ == '__main__': diff --git a/tests/scenarios_tests/user_models/test_is_int_value.py b/tests/scenarios_tests/user_models/test_is_int_value.py index 9636a6e1..19bb99c0 100644 --- a/tests/scenarios_tests/user_models/test_is_int_value.py +++ b/tests/scenarios_tests/user_models/test_is_int_value.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio from unittest import TestCase from scenarios.scenario_models.field_requirements.field_requirements import IsIntFieldRequirement @@ -13,16 +14,20 @@ def setUpClass(cls): def test_is_int_number_string(self): text = "123" - self.assertTrue(self.requirement.check(text)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(self.requirement.check(text))) def test_is_int_float_string(self): text = "1.23" - self.assertFalse(self.requirement.check(text)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(self.requirement.check(text))) def test_is_int_text_string(self): text = "test" - self.assertFalse(self.requirement.check(text)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(self.requirement.check(text))) def test_is_int_empty_string(self): text = "" - self.assertFalse(self.requirement.check(text)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(self.requirement.check(text))) diff --git a/tests/scenarios_tests/user_models/test_token_part_in_set_requirement.py b/tests/scenarios_tests/user_models/test_token_part_in_set_requirement.py index f7a3556a..d2bd095a 100644 --- a/tests/scenarios_tests/user_models/test_token_part_in_set_requirement.py +++ b/tests/scenarios_tests/user_models/test_token_part_in_set_requirement.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio import unittest from scenarios.scenario_models.field_requirements.field_requirements import TokenPartInSet @@ -22,7 +23,8 @@ def test_token_part_in_set_requirement_equal_false(self): 'country': 'Португалия', 'country_hidden': False} requirement = TokenPartInSet(requirement_items) - self.assertFalse(requirement.check(token_val)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(token_val))) def test_token_part_in_set_requirement_equal_true(self): requirement_items ={ @@ -41,7 +43,8 @@ def test_token_part_in_set_requirement_equal_true(self): 'country': 'Португалия', 'country_hidden': False} requirement = TokenPartInSet(requirement_items) - self.assertTrue(requirement.check(token_val)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(token_val))) def test_token_part_in_set_requirement_equal_False_double_empty(self): requirement_items ={ @@ -60,7 +63,8 @@ def test_token_part_in_set_requirement_equal_False_double_empty(self): 'country_hidden': False, 'locality_type': []} requirement = TokenPartInSet(requirement_items) - self.assertFalse(requirement.check(token_val)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(token_val))) def test_token_part_in_set_requirement_equal_False_empty_val_none(self): requirement_items ={ @@ -79,7 +83,8 @@ def test_token_part_in_set_requirement_equal_False_empty_val_none(self): 'country_hidden': False, 'locality_type': None} requirement = TokenPartInSet(requirement_items) - self.assertFalse(requirement.check(token_val)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(token_val))) def test_token_part_in_set_requirement_equal_False_string(self): requirement_items ={ @@ -97,7 +102,8 @@ def test_token_part_in_set_requirement_equal_False_string(self): 'country': 'Португалия', 'country_hidden': False} requirement = TokenPartInSet(requirement_items) - self.assertFalse(requirement.check(token_val)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(token_val))) def test_token_part_in_set_requirement_equal_False(self): requirement_items ={ @@ -115,7 +121,8 @@ def test_token_part_in_set_requirement_equal_False(self): 'country': 'Португалия', 'country_hidden': False} requirement = TokenPartInSet(requirement_items) - self.assertFalse(requirement.check(token_val)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(token_val))) def test_token_part_in_set_requirement_equal_False_val_int(self): requirement_items ={ @@ -133,7 +140,8 @@ def test_token_part_in_set_requirement_equal_False_val_int(self): 'country': 'Португалия', 'country_hidden': False} requirement = TokenPartInSet(requirement_items) - self.assertFalse(requirement.check(token_val)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(token_val))) def test_token_part_in_set_requirement_equal_True_arr(self): requirement_items ={ @@ -151,7 +159,8 @@ def test_token_part_in_set_requirement_equal_True_arr(self): 'country': 'Португалия', 'country_hidden': False} requirement = TokenPartInSet(requirement_items) - self.assertTrue(requirement.check(token_val)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(requirement.check(token_val))) def test_token_part_in_set_requirement_equal_False_arr(self): requirement_items ={ @@ -169,4 +178,5 @@ def test_token_part_in_set_requirement_equal_False_arr(self): 'country': 'Португалия', 'country_hidden': False} requirement = TokenPartInSet(requirement_items) - self.assertFalse(requirement.check(token_val)) + loop = asyncio.get_event_loop() + self.assertFalse(loop.run_until_complete(requirement.check(token_val))) diff --git a/tests/smart_kit_tests/requirement/test_device_requirements.py b/tests/smart_kit_tests/requirement/test_device_requirements.py index f421adef..f92bd2e7 100644 --- a/tests/smart_kit_tests/requirement/test_device_requirements.py +++ b/tests/smart_kit_tests/requirement/test_device_requirements.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio import unittest from unittest.mock import Mock from core.basic_models.requirement import device_requirements @@ -31,8 +32,9 @@ def test_platform_type_requirement_init(self): def test_platform_type_requirement_check(self): obj1 = device_requirements.PlatformTypeRequirement(self.test_items1, self.test_id) - self.assertTrue(obj1.check(self.test_text_processing_result, self.test_user1)) - self.assertTrue(not obj1.check(self.test_text_processing_result, self.test_user2)) + loop = asyncio.get_event_loop() + self.assertTrue(loop.run_until_complete(obj1.check(self.test_text_processing_result, self.test_user1))) + self.assertTrue(not loop.run_until_complete(obj1.check(self.test_text_processing_result, self.test_user2))) class RequirementTest2(unittest.TestCase): From dd8f952d8b8ac4d0b928abce4b8aedcb9b1af6e6 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Tue, 9 Nov 2021 11:11:06 +0300 Subject: [PATCH 037/116] fix asyncio bugs --- core/basic_models/scenarios/base_scenario.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/basic_models/scenarios/base_scenario.py b/core/basic_models/scenarios/base_scenario.py index 4da8f89e..bc1138d5 100644 --- a/core/basic_models/scenarios/base_scenario.py +++ b/core/basic_models/scenarios/base_scenario.py @@ -1,4 +1,5 @@ # coding: utf-8 +import asyncio from typing import Dict, Any, List import core.logging.logger_constants as log_const @@ -48,7 +49,8 @@ def build_available_requirement(self): def check_available(self, text_preprocessing_result, user): if not self.switched_off: - return await self.available_requirement.check(text_preprocessing_result, user) + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.available_requirement.check(text_preprocessing_result, user)) return False def _log_params(self): From 36487129bff20d3b73dc701d8eceb0aad799dfdf Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 9 Nov 2021 21:05:12 +0300 Subject: [PATCH 038/116] fix some async code and async tests --- .../requirement/basic_requirements.py | 10 +- scenarios/actions/action.py | 4 +- scenarios/behaviors/behaviors.py | 18 +- scenarios/scenario_models/field/field.py | 2 +- smart_kit/action/http.py | 2 +- smart_kit/handlers/handler_timeout.py | 2 +- .../_trial_temp/_trial_marker | 0 .../requirements_test/test_requirements.py | 217 +++++++---------- .../actions_test/_trial_temp/_trial_marker | 0 .../actions_test/test_action.py | 224 ++++++++---------- .../behaviors_test/_trial_temp/_trial_marker | 0 .../behaviors_test/test_behavior_model.py | 26 +- .../fillers/test_external_filler.py | 4 +- .../scenarios_test/test_tree_scenario.py | 6 +- .../models/_trial_temp/_trial_marker | 0 .../models/test_dialogue_manager.py | 4 +- 16 files changed, 217 insertions(+), 302 deletions(-) create mode 100755 tests/core_tests/requirements_test/_trial_temp/_trial_marker create mode 100755 tests/scenarios_tests/actions_test/_trial_temp/_trial_marker create mode 100755 tests/scenarios_tests/behaviors_test/_trial_temp/_trial_marker create mode 100755 tests/smart_kit_tests/models/_trial_temp/_trial_marker diff --git a/core/basic_models/requirement/basic_requirements.py b/core/basic_models/requirement/basic_requirements.py index 7879fca8..0c87101f 100644 --- a/core/basic_models/requirement/basic_requirements.py +++ b/core/basic_models/requirement/basic_requirements.py @@ -79,8 +79,9 @@ class AndRequirement(CompositeRequirement): async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, params: Dict[str, Any] = None) -> bool: return all( - await requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) - for requirement in self.requirements) + [await requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) + for requirement in self.requirements] + ) class GatherOrRequirement(CompositeRequirement): @@ -98,8 +99,9 @@ class OrRequirement(CompositeRequirement): async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser, params: Dict[str, Any] = None) -> bool: return any( - await requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) - for requirement in self.requirements) + [await requirement.check(text_preprocessing_result=text_preprocessing_result, user=user, params=params) + for requirement in self.requirements] + ) class NotRequirement(Requirement): diff --git a/scenarios/actions/action.py b/scenarios/actions/action.py index e01326f0..72ab959f 100644 --- a/scenarios/actions/action.py +++ b/scenarios/actions/action.py @@ -541,9 +541,9 @@ async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessing return None if user.message.payload: - return user.behaviors.success(callback_id) + return await user.behaviors.success(callback_id) - return user.behaviors.fail(callback_id) + return await user.behaviors.fail(callback_id) class SelfServiceActionWithState(BasicSelfServiceActionWithState): diff --git a/scenarios/behaviors/behaviors.py b/scenarios/behaviors/behaviors.py index 580e8f28..473ca2f9 100644 --- a/scenarios/behaviors/behaviors.py +++ b/scenarios/behaviors/behaviors.py @@ -115,7 +115,7 @@ def _log_callback(self, callback_id: str, log_name: str, metric, behavior_result user=self._user, params=log_params) - def success(self, callback_id: str): + async def success(self, callback_id: str): log(f"behavior.success started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_SUCCESS_VALUE, @@ -134,13 +134,11 @@ def success(self, callback_id: str): callback_action_params, ) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - loop = asyncio.get_running_loop() - result = loop.run_until_complete( - behavior.success_action.run(self._user, text_preprocessing_result, callback_action_params)) + await behavior.success_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result - def fail(self, callback_id: str): + async def fail(self, callback_id: str): log(f"behavior.fail started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_FAIL_VALUE, @@ -155,13 +153,11 @@ def fail(self, callback_id: str): smart_kit_metrics.counter_behavior_fail, "fail", callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - loop = asyncio.get_running_loop() - result = loop.run_until_complete( - behavior.fail_action.run(self._user, text_preprocessing_result, callback_action_params)) + await behavior.fail_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result - def timeout(self, callback_id: str): + async def timeout(self, callback_id: str): log(f"behavior.timeout started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_TIMEOUT_VALUE, @@ -176,9 +172,7 @@ def timeout(self, callback_id: str): smart_kit_metrics.counter_behavior_timeout, "timeout", callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - loop = asyncio.get_running_loop() - result = loop.run_until_complete( - behavior.timeout_action.run(self._user, text_preprocessing_result, callback_action_params)) + await behavior.timeout_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result diff --git a/scenarios/scenario_models/field/field.py b/scenarios/scenario_models/field/field.py index d9de4ddc..2d2b048e 100644 --- a/scenarios/scenario_models/field/field.py +++ b/scenarios/scenario_models/field/field.py @@ -41,7 +41,7 @@ def can_be_updated(self): async def check_can_be_filled(self, text_preprocessing_result, user): check, run = await asyncio.gather( self.description.requirement.check(text_preprocessing_result, user), - self.description.filler.run(user, text_preprocessing_result)) + await self.description.filler.run(user, text_preprocessing_result)) return check and run is not None @property diff --git a/smart_kit/action/http.py b/smart_kit/action/http.py index 94bc8de2..af67825c 100644 --- a/smart_kit/action/http.py +++ b/smart_kit/action/http.py @@ -29,7 +29,7 @@ async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreproces behavior_description = user.descriptions["behaviors"][self.behavior] request_params = self.params - request_params["timeout"] = behavior_description.timeout(user) + request_params["timeout"] = await behavior_description.timeout(user) params = params or {} collected = user.parametrizer.collect(text_preprocessing_result) diff --git a/smart_kit/handlers/handler_timeout.py b/smart_kit/handlers/handler_timeout.py index da5b3fb7..224bdad9 100644 --- a/smart_kit/handlers/handler_timeout.py +++ b/smart_kit/handlers/handler_timeout.py @@ -28,5 +28,5 @@ async def run(self, payload, user): user, app_info=app_info) callback_id = user.message.callback_id - result = user.behaviors.timeout(callback_id) + result = await user.behaviors.timeout(callback_id) return result diff --git a/tests/core_tests/requirements_test/_trial_temp/_trial_marker b/tests/core_tests/requirements_test/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/core_tests/requirements_test/test_requirements.py b/tests/core_tests/requirements_test/test_requirements.py index 9ab03f61..93aadac4 100644 --- a/tests/core_tests/requirements_test/test_requirements.py +++ b/tests/core_tests/requirements_test/test_requirements.py @@ -1,4 +1,3 @@ -import asyncio import os import unittest from time import time @@ -78,128 +77,112 @@ def compare(self, value): return value == self.amount -class RequirementTest(unittest.TestCase): - def test_base(self): +class RequirementTest(unittest.IsolatedAsyncioTestCase): + async def test_base(self): requirement = Requirement(None) - loop = asyncio.get_event_loop() - assert loop.run_until_complete(requirement.check(None, None)) + assert await requirement.check(None, None) - def test_composite(self): + async def test_composite(self): registered_factories[Requirement] = MockRequirement requirement = CompositeRequirement({"requirements": [ {"cond": True}, {"cond": True} ]}) - self.assertEqual(len(requirement.requirements), 2) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, None))) + self.assertTrue(await requirement.check(None, None)) - def test_and_success(self): + async def test_and_success(self): registered_factories[Requirement] = MockRequirement requirement = AndRequirement({"requirements": [ {"cond": True}, {"cond": True} ]}) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, None))) + self.assertTrue(await requirement.check(None, None)) - def test_and_fail(self): + async def test_and_fail(self): registered_factories[Requirement] = MockRequirement requirement = AndRequirement({"requirements": [ {"cond": True}, {"cond": False} ]}) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(None, None))) + self.assertFalse(await requirement.check(None, None)) - def test_or_success(self): + async def test_or_success(self): registered_factories[Requirement] = MockRequirement requirement = OrRequirement({"requirements": [ {"cond": True}, {"cond": False} ]}) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, None))) + self.assertTrue(await requirement.check(None, None)) - def test_or_fail(self): + async def test_or_fail(self): registered_factories[Requirement] = MockRequirement requirement = OrRequirement({"requirements": [ {"cond": False}, {"cond": False} ]}) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(None, None))) + self.assertFalse(await requirement.check(None, None)) - def test_not_success(self): + async def test_not_success(self): registered_factories[Requirement] = MockRequirement requirement = NotRequirement({"requirement": {"cond": False}}) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, None))) + self.assertTrue(await requirement.check(None, None)) - def test_not_fail(self): + async def test_not_fail(self): registered_factories[Requirement] = MockRequirement requirement = NotRequirement({"requirement": {"cond": True}}) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(None, None))) + self.assertFalse(await requirement.check(None, None)) - def test_channel_success(self): + async def test_channel_success(self): user = Mock() message = Mock(channel="ch1") user.message = message requirement = ChannelRequirement({"channels": ["ch1"]}) text_normalization_result = None - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertTrue(await requirement.check(text_normalization_result, user)) - def test_channel_fail(self): + async def test_channel_fail(self): user = Mock() message = Mock(channel="ch2") user.message = message requirement = ChannelRequirement({"channels": ["ch1"]}) text_normalization_result = None - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertFalse(await requirement.check(text_normalization_result, user)) - def test_random_requirement_true(self): + async def test_random_requirement_true(self): requirement = RandomRequirement({"percent": 100}) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, None))) + self.assertTrue(await requirement.check(None, None)) - def test_random_requirement_false(self): + async def test_random_requirement_false(self): requirement = RandomRequirement({"percent": 0}) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(None, None))) + self.assertFalse(await requirement.check(None, None)) - def test_topic_requirement(self): + async def test_topic_requirement(self): requirement = TopicRequirement({"topics": ["test"]}) user = Mock() message = Mock() message.topic_key = "test" user.message = message - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, user))) + self.assertTrue(await requirement.check(None, user)) - def test_counter_value_requirement(self): + async def test_counter_value_requirement(self): registered_factories[Operator] = MockAmountOperator user = Mock() counter = Mock() counter.__gt__ = Mock(return_value=True) user.counters = {"test": counter} requirement = CounterValueRequirement({"operator": {"type": "equal", "amount": 2}, "key": "test"}) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, user))) + self.assertTrue(await requirement.check(None, user)) - def test_counter_time_requirement(self): + async def test_counter_time_requirement(self): registered_factories[Operator] = MockAmountOperator user = Mock() counter = Mock() counter.update_time = int(time()) - 10 user.counters = {"test": counter} requirement = CounterUpdateTimeRequirement({"operator": {"type": "more_or_equal", "amount": 5}, "key": "test"}) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, user))) + self.assertTrue(await requirement.check(None, user)) - def test_template_req_true(self): + async def test_template_req_true(self): items = { "template": "{{ payload.message.strip() in payload.murexIds }}" } @@ -212,10 +195,9 @@ def test_template_req_true(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, user))) + self.assertTrue(await requirement.check(None, user)) - def test_template_req_false(self): + async def test_template_req_false(self): items = { "template": "{{ payload.groupCode == 'BROKER' }}" } @@ -224,10 +206,9 @@ def test_template_req_false(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(None, user))) + self.assertFalse(await requirement.check(None, user)) - def test_template_req_raise(self): + async def test_template_req_raise(self): items = { "template": "{{ payload.groupCode }}" } @@ -238,23 +219,21 @@ def test_template_req_raise(self): user.parametrizer.collect = Mock(return_value=params) self.assertRaises(TypeError, requirement.check, None, user) - def test_rolling_requirement_true(self): + async def test_rolling_requirement_true(self): user = Mock() user.id = "353454" requirement = RollingRequirement({"percent": 100}) text_normalization_result = None - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertTrue(await requirement.check(text_normalization_result, user)) - def test_rolling_requirement_false(self): + async def test_rolling_requirement_false(self): user = Mock() user.id = "353454" requirement = RollingRequirement({"percent": 0}) text_normalization_result = None - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertFalse(await requirement.check(text_normalization_result, user)) - def test_time_requirement_true(self): + async def test_time_requirement_true(self): user = Mock() user.id = "353454" user.message.payload = { @@ -274,10 +253,9 @@ def test_time_requirement_true(self): } ) text_normalization_result = None - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertTrue(await requirement.check(text_normalization_result, user)) - def test_time_requirement_false(self): + async def test_time_requirement_false(self): user = Mock() user.id = "353454" user.message.payload = { @@ -297,10 +275,9 @@ def test_time_requirement_false(self): } ) text_normalization_result = None - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertFalse(await requirement.check(text_normalization_result, user)) - def test_datetime_requirement_true(self): + async def test_datetime_requirement_true(self): user = Mock() user.id = "353454" user.message.payload = { @@ -317,10 +294,9 @@ def test_datetime_requirement_true(self): } ) text_normalization_result = None - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertTrue(await requirement.check(text_normalization_result, user)) - def test_datetime_requirement_false(self): + async def test_datetime_requirement_false(self): user = Mock() user.id = "353454" user.message.payload = { @@ -337,11 +313,10 @@ def test_datetime_requirement_false(self): } ) text_normalization_result = None - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertFalse(await requirement.check(text_normalization_result, user)) @patch('smart_kit.configs.get_app_config') - def test_intersection_requirement_true(self, mock_get_app_config): + async def test_intersection_requirement_true(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) user = Mock() requirement = IntersectionRequirement( @@ -358,11 +333,10 @@ def test_intersection_requirement_true(self, mock_get_app_config): {'lemma': 'я'}, {'lemma': 'хотеть'}, ] - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertTrue(await requirement.check(text_normalization_result, user)) @patch('smart_kit.configs.get_app_config') - def test_intersection_requirement_false(self, mock_get_app_config): + async def test_intersection_requirement_false(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) user = Mock() requirement = IntersectionRequirement( @@ -380,11 +354,10 @@ def test_intersection_requirement_false(self, mock_get_app_config): {'lemma': 'за'}, {'lemma': 'что'}, ] - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(text_normalization_result, user))) + self.assertFalse(await requirement.check(text_normalization_result, user)) @patch.object(ExternalClassifier, "find_best_answer", return_value=[{"answer": "нет", "score": 1.0, "other": False}]) - def test_classifier_requirement_true(self, mock_classifier_model): + async def test_classifier_requirement_true(self, mock_classifier_model): """Тест кейз проверяет что условие возвращает True, если результат классификации запроса относится к одной из указанных категорий, прошедших порог, но не равной классу other. """ @@ -392,33 +365,30 @@ def test_classifier_requirement_true(self, mock_classifier_model): classifier_requirement = ClassifierRequirement(test_items) mock_user = Mock() mock_user.descriptions = {"external_classifiers": ["read_book_or_not_classifier", "hello_scenario_classifier"]} - loop = asyncio.get_event_loop() - result = loop.run_until_complete(classifier_requirement.check(Mock(), mock_user)) + result = await classifier_requirement.check(Mock(), mock_user) self.assertTrue(result) @patch.object(ExternalClassifier, "find_best_answer", return_value=[]) - def test_classifier_requirement_false(self, mock_classifier_model): + async def test_classifier_requirement_false(self, mock_classifier_model): """Тест кейз проверяет что условие возвращает False, если модель классификации не вернула ответ.""" test_items = {"type": "classifier", "classifier": {"type": "external", "classifier": "hello_scenario_classifier"}} classifier_requirement = ClassifierRequirement(test_items) mock_user = Mock() mock_user.descriptions = {"external_classifiers": ["read_book_or_not_classifier", "hello_scenario_classifier"]} - loop = asyncio.get_event_loop() - result = loop.run_until_complete(classifier_requirement.check(Mock(), mock_user)) + result = await classifier_requirement.check(Mock(), mock_user) self.assertFalse(result) @patch.object(ExternalClassifier, "find_best_answer", return_value=[{"answer": "other", "score": 1.0, "other": True}]) - def test_classifier_requirement_false_if_class_other(self, mock_classifier_model): + async def test_classifier_requirement_false_if_class_other(self, mock_classifier_model): """Тест кейз проверяет что условие возвращает False, если наиболее вероятный вариант есть класс other.""" test_items = {"type": "classifier", "classifier": {"type": "external", "classifier": "hello_scenario_classifier"}} classifier_requirement = ClassifierRequirement(test_items) mock_user = Mock() mock_user.descriptions = {"external_classifiers": ["read_book_or_not_classifier", "hello_scenario_classifier"]} - loop = asyncio.get_event_loop() - result = loop.run_until_complete(classifier_requirement.check(Mock(), mock_user)) + result = await classifier_requirement.check(Mock(), mock_user) self.assertFalse(result) - def test_form_field_value_requirement_true(self): + async def test_form_field_value_requirement_true(self): """Тест кейз проверяет что условие возвращает True, т.к в форме form_name в поле form_field значение совпадает с переданным field_value. """ @@ -434,11 +404,10 @@ def test_form_field_value_requirement_true(self): user.forms[form_name].fields = {form_field: Mock(), "value": field_value} user.forms[form_name].fields[form_field].value = field_value - loop = asyncio.get_event_loop() - result = loop.run_until_complete(req_form_field_value.check(Mock(), user)) + result = await req_form_field_value.check(Mock(), user) self.assertTrue(result) - def test_form_field_value_requirement_false(self): + async def test_form_field_value_requirement_false(self): """Тест кейз проверяет что условие возвращает False, т.к в форме form_name в поле form_field значение НЕ совпадает с переданным field_value. """ @@ -454,82 +423,72 @@ def test_form_field_value_requirement_false(self): user.forms[form_name].fields = {form_field: Mock(), "value": "OTHER_TEST_VAL"} user.forms[form_name].fields[form_field].value = "OTHER_TEST_VAL" - loop = asyncio.get_event_loop() - result = loop.run_until_complete(req_form_field_value.check(Mock(), user)) + result = await req_form_field_value.check(Mock(), user) self.assertFalse(result) @patch("smart_kit.configs.get_app_config") - def test_environment_requirement_true(self, mock_get_app_config): + async def test_environment_requirement_true(self, mock_get_app_config): """Тест кейз проверяет что условие возвращает True, т.к среда исполнения из числа values.""" patch_get_app_config(mock_get_app_config) environment_req = EnvironmentRequirement({"values": ["ift", "uat"]}) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(environment_req.check(Mock(), Mock()))) + self.assertTrue(await environment_req.check(Mock(), Mock())) @patch("smart_kit.configs.get_app_config") - def test_environment_requirement_false(self, mock_get_app_config): + async def test_environment_requirement_false(self, mock_get_app_config): """Тест кейз проверяет что условие возвращает False, т.к среда исполнения НЕ из числа values.""" patch_get_app_config(mock_get_app_config) environment_req = EnvironmentRequirement({"values": ["uat", "pt"]}) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(environment_req.check(Mock(), Mock()))) + self.assertFalse(await environment_req.check(Mock(), Mock())) - def test_any_substring_in_lowered_text_requirement_true(self): + async def test_any_substring_in_lowered_text_requirement_true(self): """Тест кейз проверяет что условие возвращает True, т.к нашлась подстрока из списка substrings, которая встречается в оригинальном тексте в нижнем регистре. """ req = AnySubstringInLoweredTextRequirement({"substrings": ["искомая подстрока", "другое знанчение"]}) text_preprocessing_result = Mock() text_preprocessing_result.raw = {"original_text": "КАКОЙ-ТО ТЕКСТ С ИСКОМАЯ ПОДСТРОКА"} - loop = asyncio.get_event_loop() - result = loop.run_until_complete(req.check(text_preprocessing_result, Mock())) + result = await req.check(text_preprocessing_result, Mock()) self.assertTrue(result) - def test_any_substring_in_lowered_text_requirement_false(self): + async def test_any_substring_in_lowered_text_requirement_false(self): """Тест кейз проверяет что условие возвращает False, т.к НЕ нашлась ни одна подстрока из списка substrings, которая бы встречалась в оригинальном тексте в нижнем регистре. """ req = AnySubstringInLoweredTextRequirement({"substrings": ["искомая подстрока", "другая подстрока"]}) text_preprocessing_result = Mock() text_preprocessing_result.raw = {"original_text": "КАКОЙ-ТО ТЕКСТ"} - loop = asyncio.get_event_loop() - result = loop.run_until_complete(req.check(text_preprocessing_result, Mock())) + result = await req.check(text_preprocessing_result, Mock()) self.assertFalse(result) - def test_num_in_range_requirement_true(self): + async def test_num_in_range_requirement_true(self): """Тест кейз проверяет что условие возвращает True, т.к число находится в заданном диапазоне.""" req = NumInRangeRequirement({"min_num": "5", "max_num": "10"}) text_preprocessing_result = Mock() - text_preprocessing_result.num_token_values = 7 - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) + self.assertTrue(await req.check(text_preprocessing_result, Mock())) - def test_num_in_range_requirement_false(self): + async def test_num_in_range_requirement_false(self): """Тест кейз проверяет что условие возвращает False, т.к число НЕ находится в заданном диапазоне.""" req = NumInRangeRequirement({"min_num": "5", "max_num": "10"}) text_preprocessing_result = Mock() text_preprocessing_result.num_token_values = 20 - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) + self.assertFalse(await req.check(text_preprocessing_result, Mock())) - def test_phone_number_number_requirement_true(self): + async def test_phone_number_number_requirement_true(self): """Тест кейз проверяет что условие возвращает True, т.к кол-во номеров телефонов больше заданного.""" req = PhoneNumberNumberRequirement({"operator": {"type": "more", "amount": 1}}) text_preprocessing_result = Mock() text_preprocessing_result.get_token_values_by_type.return_value = ["89030478799", "89092534523"] - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) + self.assertTrue(await req.check(text_preprocessing_result, Mock())) - def test_phone_number_number_requirement_false(self): + async def test_phone_number_number_requirement_false(self): """Тест кейз проверяет что условие возвращает False, т.к кол-во номеров телефонов НЕ больше заданного.""" req = PhoneNumberNumberRequirement({"operator": {"type": "more", "amount": 10}}) text_preprocessing_result = Mock() text_preprocessing_result.get_token_values_by_type.return_value = ["89030478799"] - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) + self.assertFalse(await req.check(text_preprocessing_result, Mock())) @patch("smart_kit.configs.get_app_config") - def test_intersection_with_tokens_requirement_true(self, mock_get_app_config): + async def test_intersection_with_tokens_requirement_true(self, mock_get_app_config): """Тест кейз проверяет что условие возвращает True, т.к хотя бы одно слово из нормализованного вида запроса входит в список слов input_words. """ @@ -548,11 +507,10 @@ def test_intersection_with_tokens_requirement_true(self, mock_get_app_config): "part_of_speech": "NOUN"}, "lemma": "погода"} ]} - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) + self.assertTrue(await req.check(text_preprocessing_result, Mock())) @patch("smart_kit.configs.get_app_config") - def test_intersection_with_tokens_requirement_false(self, mock_get_app_config): + async def test_intersection_with_tokens_requirement_false(self, mock_get_app_config): """Тест кейз проверяет что условие возвращает False, т.к ни одно слово из нормализованного вида запроса не входит в список слов input_words. """ @@ -571,11 +529,10 @@ def test_intersection_with_tokens_requirement_false(self, mock_get_app_config): "part_of_speech": "NOUN"}, "lemma": "погода"} ]} - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) + self.assertFalse(await req.check(text_preprocessing_result, Mock())) @patch("smart_kit.configs.get_app_config") - def test_normalized_text_in_set_requirement_true(self, mock_get_app_config): + async def test_normalized_text_in_set_requirement_true(self, mock_get_app_config): """Тест кейз проверяет что условие возвращает True, т.к в нормализованном представлении запрос полностью совпадает с одной из нормализованных строк из input_words. """ @@ -586,11 +543,10 @@ def test_normalized_text_in_set_requirement_true(self, mock_get_app_config): text_preprocessing_result = Mock() text_preprocessing_result.raw = {"normalized_text": "погода ."} - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) + self.assertTrue(await req.check(text_preprocessing_result, Mock())) @patch("smart_kit.configs.get_app_config") - def test_normalized_text_in_set_requirement_false(self, mock_get_app_config): + async def test_normalized_text_in_set_requirement_false(self, mock_get_app_config): """Тест кейз проверяет что условие возвращает False, т.к в нормализованном представлении запрос НЕ совпадает ни с одной из нормализованных строк из input_words. """ @@ -601,8 +557,7 @@ def test_normalized_text_in_set_requirement_false(self, mock_get_app_config): text_preprocessing_result = Mock() text_preprocessing_result.raw = {"normalized_text": "хотеть узнать ."} - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(req.check(text_preprocessing_result, Mock()))) + self.assertFalse(await req.check(text_preprocessing_result, Mock())) if __name__ == '__main__': diff --git a/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker b/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/scenarios_tests/actions_test/test_action.py b/tests/scenarios_tests/actions_test/test_action.py index afcbaac2..a0aa16f1 100644 --- a/tests/scenarios_tests/actions_test/test_action.py +++ b/tests/scenarios_tests/actions_test/test_action.py @@ -1,4 +1,3 @@ -import asyncio import unittest from typing import Dict, Any, Union, Optional from unittest.mock import MagicMock, Mock, ANY @@ -53,38 +52,35 @@ def collect(self, text_preprocessing_result=None, filter_params=None): return data -class ClearFormIdActionTest(unittest.TestCase): - def test_run(self): +class ClearFormIdActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): action = ClearFormAction({"form": "form"}) user = MagicMock() - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) user.forms.remove_item.assert_called_once_with("form") -class RemoveCompositeFormFieldActionTest(unittest.TestCase): - def test_run(self): +class RemoveCompositeFormFieldActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): action = ClearInnerFormAction({"form": "form", "inner_form": "inner_form"}) user, form = MagicMock(), MagicMock() user.forms.__getitem__.return_value = form - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) form.forms.remove_item.assert_called_once_with("inner_form") -class BreakScenarioTest(unittest.TestCase): - def test_run_1(self): +class BreakScenarioTest(unittest.IsolatedAsyncioTestCase): + async def test_run_1(self): scenario_id = "test_id" action = BreakScenarioAction({"scenario_id": scenario_id}) user = Mock() scenario_model = MagicMock() scenario_model.set_break = Mock(return_value=None) user.scenario_models = {scenario_id: scenario_model} - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) user.scenario_models[scenario_id].set_break.assert_called_once() - def test_run_2(self): + async def test_run_2(self): scenario_id = "test_id" action = BreakScenarioAction({}) user = Mock() @@ -92,13 +88,12 @@ def test_run_2(self): scenario_model = MagicMock() scenario_model.set_break = Mock(return_value=None) user.scenario_models = {scenario_id: scenario_model} - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) user.scenario_models[scenario_id].set_break.assert_called_once() -class AskAgainActionTest(unittest.TestCase): - def test_run(self): +class AskAgainActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): items = dict() user = Mock() last_scenario_name = "test_id" @@ -108,33 +103,30 @@ def test_run(self): scenarios = {last_scenario_name: scenario} user.descriptions = {"scenarios": scenarios} action = AskAgainAction(items) - loop = asyncio.get_event_loop() - tesult = loop.run_until_complete(action.run(user, None)) + tesult = await action.run(user, None) self.assertEqual(tesult, "test_result") -class RemoveFormFieldActionTest(unittest.TestCase): - def test_run(self): +class RemoveFormFieldActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): action = RemoveFormFieldAction({"form": "form", "field": "field"}) user, form = MagicMock(), MagicMock() user.forms.__getitem__.return_value = form - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) form.fields.remove_item.assert_called_once_with("field") -class RemoveCompositeFormFieldActionTest(unittest.TestCase): - def test_run(self): +class RemoveCompositeFormFieldActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): action = RemoveCompositeFormFieldAction({"form": "form", "inner_form": "form", "field": "field"}) user, inner_form, form = MagicMock(), MagicMock(), MagicMock() form.forms.__getitem__.return_value = inner_form user.forms.__getitem__.return_value = form - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) inner_form.fields.remove_item.assert_called_once_with("field") -class SaveBehaviorActionTest(unittest.TestCase): +class SaveBehaviorActionTest(unittest.IsolatedAsyncioTestCase): @classmethod def setUpClass(cls): user = Mock() @@ -145,7 +137,7 @@ def setUpClass(cls): user.message.incremental_id = test_incremental_id cls.user = user - def test_save_behavior_scenario_name(self): + async def test_save_behavior_scenario_name(self): data = {"behavior": "test"} behavior = Mock() behavior.add = Mock() @@ -153,13 +145,12 @@ def test_save_behavior_scenario_name(self): action = SaveBehaviorAction(data) tpr = Mock() tpr_raw = tpr.raw - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, tpr)) + await action.run(self.user, tpr) self.user.behaviors.add.assert_called_once_with(self.user.message.generate_new_callback_id(), "test", self.user.last_scenarios.last_scenario_name, tpr_raw, action_params=None) - def test_save_behavior_without_scenario_name(self): + async def test_save_behavior_without_scenario_name(self): data = {"behavior": "test", "check_scenario": False} behavior = Mock() behavior.add = Mock() @@ -167,18 +158,17 @@ def test_save_behavior_without_scenario_name(self): action = SaveBehaviorAction(data) text_preprocessing_result_raw = Mock() text_preprocessing_result = Mock(raw=text_preprocessing_result_raw) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, text_preprocessing_result, None)) + await action.run(self.user, text_preprocessing_result, None) self.user.behaviors.add.assert_called_once_with(self.user.message.generate_new_callback_id(), "test", None, text_preprocessing_result_raw, action_params=None) -class SelfServiceActionWithStateTest(unittest.TestCase): +class SelfServiceActionWithStateTest(unittest.IsolatedAsyncioTestCase): def setUp(self) -> None: self.user = Mock() self.user.settings = {"template_settings": {"self_service_with_state_save_messages": True}} - def test_action_1(self): + async def test_action_1(self): data = {"behavior": "test", "check_scenario": False, "command_action": {"command": "cmd_id", "nodes": {}, "request_data": {}}} registered_factories[Action] = action_factory @@ -196,14 +186,13 @@ def test_action_1(self): action = SelfServiceActionWithState(data) text_preprocessing_result_raw = Mock() text_preprocessing_result = Mock(raw=text_preprocessing_result_raw) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(self.user, text_preprocessing_result, None)) + result = await action.run(self.user, text_preprocessing_result, None) behavior.check_got_saved_id.assert_called_once() behavior.add.assert_called_once() self.assertEqual(result[0].name, "cmd_id") self.assertEqual(result[0].raw, {'messageName': 'cmd_id', 'payload': {}}) - def test_action_2(self): + async def test_action_2(self): data = {"behavior": "test", "check_scenario": False, "command_action": {"command": "cmd_id", "nodes": {}}} self.user.parametrizer = MockParametrizer(self.user, {}) self.user.message = Mock() @@ -213,12 +202,11 @@ def test_action_2(self): self.user.behaviors = behavior behavior.check_got_saved_id = Mock(return_value=True) action = SelfServiceActionWithState(data) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(self.user, None)) + result = await action.run(self.user, None) behavior.add.assert_not_called() self.assertIsNone(result) - def test_action_3(self): + async def test_action_3(self): data = {"behavior": "test", "command_action": {"command": "cmd_id", "nodes": {}, "request_data": {}}} registered_factories[Action] = action_factory actions["action_mock"] = MockAction @@ -243,8 +231,7 @@ def test_action_3(self): action = SelfServiceActionWithState(data) text_preprocessing_result_raw = Mock() text_preprocessing_result = Mock(raw=text_preprocessing_result_raw) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(self.user, text_preprocessing_result, None)) + result = await action.run(self.user, text_preprocessing_result, None) behavior.check_got_saved_id.assert_called_once() behavior.add.assert_called_once() self.assertEqual(result[0].name, "cmd_id") @@ -254,7 +241,7 @@ def test_action_3(self): ) -class SetVariableActionTest(unittest.TestCase): +class SetVariableActionTest(unittest.IsolatedAsyncioTestCase): def setUp(self): template = Mock() @@ -287,7 +274,7 @@ async def test_action_jinja_no_key(self): self.user.variables.set.assert_called_with("some_key", "", None) -class DeleteVariableActionTest(unittest.TestCase): +class DeleteVariableActionTest(unittest.IsolatedAsyncioTestCase): def setUp(self): user = Mock() @@ -299,14 +286,13 @@ def setUp(self): user.variables.delete = Mock() self.user = user - def test_action(self): + async def test_action(self): action = DeleteVariableAction({"key": "some_key_1"}) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, None)) + await action.run(self.user, None) self.user.variables.delete.assert_called_with("some_key_1") -class ClearVariablesActionTest(unittest.TestCase): +class ClearVariablesActionTest(unittest.IsolatedAsyncioTestCase): def setUp(self): self.var_value = { @@ -322,16 +308,15 @@ def setUp(self): user.variables.clear = Mock() self.user = user - def test_action(self): + async def test_action(self): action = ClearVariablesAction() - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, None)) + await action.run(self.user, None) self.user.variables.clear.assert_called_with() -class FillFieldActionTest(unittest.TestCase): +class FillFieldActionTest(unittest.IsolatedAsyncioTestCase): - def test_fill_field(self): + async def test_fill_field(self): params = {"test_field": "test_data"} data = {"form": "test_form", "field": "test_field", "data_path": "{{test_field}}"} action = FillFieldAction(data) @@ -341,14 +326,13 @@ def test_fill_field(self): field = Mock() field.fill = Mock() user.forms["test_form"].fields = {"test_field": field} - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) field.fill.assert_called_once_with(params["test_field"]) -class CompositeFillFieldActionTest(unittest.TestCase): +class CompositeFillFieldActionTest(unittest.IsolatedAsyncioTestCase): - def test_fill_field(self): + async def test_fill_field(self): params = {"test_field": "test_data"} data = {"form": "test_form", "field": "test_field", "internal_form": "test_internal_form", "data_path": "{{test_field}}", "parametrizer": {"data": params}} @@ -362,13 +346,12 @@ def test_fill_field(self): field = Mock() field.fill = Mock() user.forms["test_form"].forms["test_internal_form"].fields = {"test_field": field} - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) field.fill.assert_called_once_with(params["test_field"]) -class ScenarioActionTest(unittest.TestCase): - def test_scenario_action(self): +class ScenarioActionTest(unittest.IsolatedAsyncioTestCase): + async def test_scenario_action(self): action = RunScenarioAction({"scenario": "test"}) user = Mock() user.parametrizer = MockParametrizer(user, {}) @@ -376,11 +359,10 @@ def test_scenario_action(self): scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"test": scen}} - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, Mock())) + result = await action.run(user, Mock()) self.assertEqual(result, scen_result) - def test_scenario_action_with_jinja_good(self): + async def test_scenario_action_with_jinja_good(self): params = {'next_scenario': 'ANNA.pipeline.scenario'} items = {"scenario": "{{next_scenario}}"} @@ -391,11 +373,10 @@ def test_scenario_action_with_jinja_good(self): scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"ANNA.pipeline.scenario": scen}} - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, Mock())) + result = await action.run(user, Mock()) self.assertEqual(result, scen_result) - def test_scenario_action_no_scenario(self): + async def test_scenario_action_no_scenario(self): action = RunScenarioAction({"scenario": "{{next_scenario}}"}) user = Mock() user.parametrizer = MockParametrizer(user, {}) @@ -403,11 +384,10 @@ def test_scenario_action_no_scenario(self): scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"next_scenario": scen}} - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, Mock())) + result = await action.run(user, Mock()) self.assertEqual(result, None) - def test_scenario_action_without_jinja(self): + async def test_scenario_action_without_jinja(self): action = RunScenarioAction({"scenario": "next_scenario"}) user = Mock() user.parametrizer = MockParametrizer(user, {}) @@ -415,14 +395,13 @@ def test_scenario_action_without_jinja(self): scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"next_scenario": scen}} - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, Mock())) + result = await action.run(user, Mock()) self.assertEqual(result, scen_result) -class RunLastScenarioActionTest(unittest.TestCase): +class RunLastScenarioActionTest(unittest.IsolatedAsyncioTestCase): - def test_scenario_action(self): + async def test_scenario_action(self): action = RunLastScenarioAction({}) user = Mock() scen = Mock() @@ -433,15 +412,14 @@ def test_scenario_action(self): last_scenario_name = "test" user.last_scenarios.scenarios_names = [last_scenario_name] user.last_scenarios.last_scenario_name = last_scenario_name - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, Mock())) + result = await action.run(user, Mock()) self.assertEqual(result, scen_result) -class ChoiceScenarioActionTest(unittest.TestCase): +class ChoiceScenarioActionTest(unittest.IsolatedAsyncioTestCase): @staticmethod - def mock_and_perform_action(test_items: Dict[str, Any], expected_result: Optional[str] = None, + async def mock_and_perform_action(test_items: Dict[str, Any], expected_result: Optional[str] = None, expected_scen: Optional[str] = None) -> Union[str, None]: action = ChoiceScenarioAction(test_items) user = Mock() @@ -449,11 +427,9 @@ def mock_and_perform_action(test_items: Dict[str, Any], expected_result: Optiona scen = Mock() scen.run.return_value = expected_result if expected_scen: - user.descriptions = {"scenarios": {expected_scen: scen}} - loop = asyncio.get_event_loop() - return loop.run_until_complete(action.run(user, Mock())) + return await action.run(user, Mock()) - def test_choice_scenario_action(self): + async def test_choice_scenario_action(self): # Проверяем, что запустили нужный сценарий, в случае если выполнился его requirement test_items = { "scenarios": [ @@ -473,11 +449,11 @@ def test_choice_scenario_action(self): "else_action": {"type": "test", "result": "ELSE ACTION IS DONE"} } expected_scen_result = "test_N_done" - real_scen_result = self.mock_and_perform_action( + real_scen_result = await self.mock_and_perform_action( test_items, expected_result=expected_scen_result, expected_scen="test_N") self.assertEqual(real_scen_result, expected_scen_result) - def test_choice_scenario_action_no_else_action(self): + async def test_choice_scenario_action_no_else_action(self): # Проверяем, что вернули None в случае если ни один сценарий не запустился (requirement=False) и else_action нет test_items = { "scenarios": [ @@ -491,10 +467,10 @@ def test_choice_scenario_action_no_else_action(self): } ] } - real_scen_result = self.mock_and_perform_action(test_items) + real_scen_result = await self.mock_and_perform_action(test_items) self.assertIsNone(real_scen_result) - def test_choice_scenario_action_with_else_action(self): + async def test_choice_scenario_action_with_else_action(self): # Проверяем, что выполняется else_action в случае если ни один сценарий не запустился т.к их requirement=False test_items = { "scenarios": [ @@ -510,13 +486,13 @@ def test_choice_scenario_action_with_else_action(self): "else_action": {"type": "test", "result": "ELSE ACTION IS DONE"} } expected_scen_result = "ELSE ACTION IS DONE" - real_scen_result = self.mock_and_perform_action(test_items, expected_result=expected_scen_result) + real_scen_result = await self.mock_and_perform_action(test_items, expected_result=expected_scen_result) self.assertEqual(real_scen_result, expected_scen_result) class ClearCurrentScenarioActionTest(unittest.TestCase): - def test_action(self): + async def test_action(self): scenario_name = "test_scenario" user = Mock() user.forms.remove_item = Mock() @@ -528,13 +504,12 @@ def test_action(self): user.descriptions = {"scenarios": {scenario_name: scenario}} action = ClearCurrentScenarioAction({}) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, {}, {})) + result = await action.run(user, {}, {}) self.assertIsNone(result) user.last_scenarios.delete.assert_called_once() user.forms.remove_item.assert_called_once() - def test_action_with_empty_scenarios_names(self): + async def test_action_with_empty_scenarios_names(self): user = Mock() user.forms.remove_item = Mock() @@ -542,16 +517,15 @@ def test_action_with_empty_scenarios_names(self): user.last_scenarios.delete = Mock() action = ClearCurrentScenarioAction({}) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, {}, {})) + result = await action.run(user, {}, {}) self.assertIsNone(result) user.last_scenarios.delete.assert_not_called() user.forms.remove_item.assert_not_called() -class ClearScenarioByIdActionTest(unittest.TestCase): +class ClearScenarioByIdActionTest(unittest.IsolatedAsyncioTestCase): - def test_action(self): + async def test_action(self): scenario_name = "test_scenario" user = Mock() user.forms = Mock() @@ -562,28 +536,26 @@ def test_action(self): user.descriptions = {"scenarios": {scenario_name: scenario}} action = ClearScenarioByIdAction({"scenario_id": scenario_name}) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, {}, {})) + result = await action.run(user, {}, {}) self.assertIsNone(result) user.last_scenarios.delete.assert_called_once() user.forms.remove_item.assert_called_once() - def test_action_with_empty_scenarios_names(self): + async def test_action_with_empty_scenarios_names(self): user = Mock() user.forms = Mock() user.last_scenarios.last_scenario_name = "test_scenario" action = ClearScenarioByIdAction({}) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, {}, {})) + result = await action.run(user, {}, {}) self.assertIsNone(result) user.last_scenarios.delete.assert_not_called() user.forms.remove_item.assert_not_called() -class ClearCurrentScenarioFormActionTest(unittest.TestCase): - def test_action(self): +class ClearCurrentScenarioFormActionTest(unittest.IsolatedAsyncioTestCase): + async def test_action(self): scenario_name = "test_scenario" user = Mock() user.forms = Mock() @@ -596,12 +568,11 @@ def test_action(self): user.descriptions = {"scenarios": {scenario_name: scenario}} action = ClearCurrentScenarioFormAction({}) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, {}, {})) + result = await action.run(user, {}, {}) self.assertIsNone(result) user.forms.clear_form.assert_called_once() - def test_action_with_empty_last_scenario(self): + async def test_action_with_empty_last_scenario(self): scenario_name = "test_scenario" user = Mock() user.forms = Mock() @@ -614,14 +585,13 @@ def test_action_with_empty_last_scenario(self): user.descriptions = {"scenarios": {scenario_name: scenario}} action = ClearCurrentScenarioFormAction({}) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, {}, {})) + result = await action.run(user, {}, {}) self.assertIsNone(result) user.forms.remove_item.assert_not_called() -class ResetCurrentNodeActionTest(unittest.TestCase): - def test_action(self): +class ResetCurrentNodeActionTest(unittest.IsolatedAsyncioTestCase): + async def test_action(self): user = Mock() user.forms = Mock() user.last_scenarios.last_scenario_name = 'test_scenario' @@ -630,12 +600,11 @@ def test_action(self): user.scenario_models = {'test_scenario': scenario_model} action = ResetCurrentNodeAction({}) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, {}, {})) + result = await action.run(user, {}, {}) self.assertIsNone(result) self.assertIsNone(user.scenario_models['test_scenario'].current_node) - def test_action_with_empty_last_scenario(self): + async def test_action_with_empty_last_scenario(self): user = Mock() user.forms = Mock() user.last_scenarios.last_scenario_name = None @@ -644,12 +613,11 @@ def test_action_with_empty_last_scenario(self): user.scenario_models = {'test_scenario': scenario_model} action = ResetCurrentNodeAction({}) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, {}, {})) + result = await action.run(user, {}, {}) self.assertIsNone(result) self.assertEqual('some_node', user.scenario_models['test_scenario'].current_node) - def test_specific_target(self): + async def test_specific_target(self): user = Mock() user.forms = Mock() user.last_scenarios.last_scenario_name = 'test_scenario' @@ -661,13 +629,12 @@ def test_specific_target(self): 'node_id': 'another_node' } action = ResetCurrentNodeAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, {}, {})) + result = await action.run(user, {}, {}) self.assertIsNone(result) self.assertEqual('another_node', user.scenario_models['test_scenario'].current_node) -class AddHistoryEventActionTest(unittest.TestCase): +class AddHistoryEventActionTest(unittest.IsolatedAsyncioTestCase): def setUp(self): main_form = Mock() @@ -682,7 +649,7 @@ def setUp(self): self.user.history.add_event = Mock() self.user.last_scenarios.last_scenario_name = 'test_scenario' - def test_action_with_non_empty_scenario(self): + async def test_action_with_non_empty_scenario(self): scenario = Mock() scenario.id = 'name' scenario.version = '1.0' @@ -701,13 +668,12 @@ def test_action_with_non_empty_scenario(self): ) action = AddHistoryEventAction(items) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, None, None)) + await action.run(self.user, None, None) self.user.history.add_event.assert_called_once() self.user.history.add_event.assert_called_once_with(expected) - def test_action_with_empty_scenario(self): + async def test_action_with_empty_scenario(self): self.user.descriptions = {'scenarios': {}} items = { 'event_type': 'type', @@ -716,12 +682,11 @@ def test_action_with_empty_scenario(self): } action = AddHistoryEventAction(items) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, None, None)) + await action.run(self.user, None, None) self.user.history.add_event.assert_not_called() - def test_action_with_jinja(self): + async def test_action_with_jinja(self): scenario = Mock() scenario.id = 'name' scenario.version = '1.0' @@ -740,8 +705,7 @@ def test_action_with_jinja(self): ) action = AddHistoryEventAction(items) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(self.user, None, None)) + await action.run(self.user, None, None) self.user.history.add_event.assert_called_once() self.user.history.add_event.assert_called_once_with(expected) diff --git a/tests/scenarios_tests/behaviors_test/_trial_temp/_trial_marker b/tests/scenarios_tests/behaviors_test/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/scenarios_tests/behaviors_test/test_behavior_model.py b/tests/scenarios_tests/behaviors_test/test_behavior_model.py index c0b666b1..5f2ee5a9 100644 --- a/tests/scenarios_tests/behaviors_test/test_behavior_model.py +++ b/tests/scenarios_tests/behaviors_test/test_behavior_model.py @@ -2,11 +2,11 @@ import unittest from collections import OrderedDict from collections import namedtuple -from unittest.mock import Mock +from unittest.mock import Mock, AsyncMock import scenarios.behaviors.behaviors -class BehaviorsTest(unittest.TestCase): +class BehaviorsTest(unittest.IsolatedAsyncioTestCase): def setUp(self): self.user = Mock() self.user.settings = Mock() @@ -15,9 +15,9 @@ def setUp(self): self.description = Mock() self.description.timeout = Mock(return_value=10) self.success_action = Mock() - self.success_action.run = Mock() - self.fail_action = Mock() - self.timeout_action = Mock() + self.success_action.run = AsyncMock() + self.fail_action = AsyncMock() + self.timeout_action = AsyncMock() self.description.success_action = self.success_action self.description.fail_action = self.fail_action @@ -25,7 +25,7 @@ def setUp(self): self.descriptions = {"test": self.description} self._callback = namedtuple('Callback', 'behavior_id expire_time scenario_id') - def test_success(self): + async def test_success(self): callback_id = "123" behavior_id = "test" item = {"behavior_id": behavior_id, "expire_time": 2554416000, "scenario_id": None, @@ -33,38 +33,38 @@ def test_success(self): items = {str(callback_id): item} behaviors = scenarios.behaviors.behaviors.Behaviors(items, self.descriptions, self.user) behaviors.initialize() - behaviors.success(callback_id) + await behaviors.success(callback_id) # self.success_action.run.assert_called_once_with(self.user, TextPreprocessingResult({})) self.success_action.run.assert_called_once() self.assertDictEqual(behaviors.raw, {}) - def test_success_2(self): + async def test_success_2(self): callback_id = "123" items = {} behaviors = scenarios.behaviors.behaviors.Behaviors(items, self.descriptions, self.user) behaviors.initialize() - behaviors.success(callback_id) + await behaviors.success(callback_id) self.success_action.run.assert_not_called() - def test_fail(self): + async def test_fail(self): callback_id = "123" behavior_id = "test" item = {"behavior_id": behavior_id, "expire_time": 2554416000, "scenario_id": None} items = {str(callback_id): item} behaviors = scenarios.behaviors.behaviors.Behaviors(items, self.descriptions, self.user) behaviors.initialize() - behaviors.fail(callback_id) + await behaviors.fail(callback_id) self.fail_action.run.assert_called_once() self.assertDictEqual(behaviors.raw, {}) - def test_timeout(self): + async def test_timeout(self): callback_id = "123" behavior_id = "test" item = {"behavior_id": behavior_id, "expire_time": 2554416000, "scenario_id": None} items = {str(callback_id): item} behaviors = scenarios.behaviors.behaviors.Behaviors(items, self.descriptions, self.user) behaviors.initialize() - behaviors.timeout(callback_id) + await behaviors.timeout(callback_id) self.timeout_action.run.assert_called_once() self.assertDictEqual(behaviors.raw, {}) diff --git a/tests/scenarios_tests/fillers/test_external_filler.py b/tests/scenarios_tests/fillers/test_external_filler.py index 892a660f..a0894d4f 100644 --- a/tests/scenarios_tests/fillers/test_external_filler.py +++ b/tests/scenarios_tests/fillers/test_external_filler.py @@ -1,5 +1,5 @@ from unittest import TestCase -from unittest.mock import Mock +from unittest.mock import Mock, AsyncMock from scenarios.scenario_models.field.field_filler_description import ExternalFieldFillerDescription @@ -10,7 +10,7 @@ async def test_1(self): expected = 5 items = {"filler": "my_key"} mock_filler = Mock() - mock_filler.run = Mock(return_value=expected) + mock_filler.run = AsyncMock(return_value=expected) mock_user = Mock() mock_user.descriptions = {"external_field_fillers": {"my_key": mock_filler}} diff --git a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py index 8b48732a..7b107762 100644 --- a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py +++ b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py @@ -1,5 +1,5 @@ from unittest import TestCase -from unittest.mock import Mock, MagicMock +from unittest.mock import Mock, MagicMock, AsyncMock from core.basic_models.actions.command import Command from core.model.registered import registered_factories @@ -50,7 +50,7 @@ async def test_1(self): "scenario_nodes": {"node_1": node_mock}} field_descriptor = Mock(name="field_descriptor_mock") - field_descriptor.filler.extract = Mock(name="my_field_value_1", return_value=61) + field_descriptor.filler.extract = AsyncMock(name="my_field_value_1", return_value=61) field_descriptor.fill_other = False field_descriptor.field_validator.actions = [] @@ -107,7 +107,7 @@ async def test_breake(self): "scenario_nodes": {"node_1": node_mock}, "actions": [{"type": "success"}]} field_descriptor = Mock(name="field_descriptor_mock") - field_descriptor.filler.extract = Mock(name="my_field_value_1", return_value=61) + field_descriptor.filler.extract = AsyncMock(name="my_field_value_1", return_value=61) field_descriptor.fill_other = False field_descriptor.field_validator.actions = [] field_descriptor.on_filled_actions = [BreakAction(), MockAction(command_name="break action result")] diff --git a/tests/smart_kit_tests/models/_trial_temp/_trial_marker b/tests/smart_kit_tests/models/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/smart_kit_tests/models/test_dialogue_manager.py b/tests/smart_kit_tests/models/test_dialogue_manager.py index 80bacaa2..1618a162 100644 --- a/tests/smart_kit_tests/models/test_dialogue_manager.py +++ b/tests/smart_kit_tests/models/test_dialogue_manager.py @@ -1,6 +1,6 @@ # coding: utf-8 import unittest -from unittest.mock import Mock +from unittest.mock import Mock, AsyncMock from smart_kit.models import dialogue_manager @@ -9,7 +9,7 @@ def get_keys(self): return self.keys() -class ModelsTest1(unittest.TestCase): +class ModelsTest1(unittest.IsolatedAsyncioTestCase): def setUp(self): self.test_user1 = Mock() self.test_user1.name = "TestName" From 8363b12f8214182fde80b3feb4a4e1d17f72f80f Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 9 Nov 2021 21:07:15 +0300 Subject: [PATCH 039/116] remove useless files --- tests/core_tests/requirements_test/_trial_temp/_trial_marker | 0 tests/scenarios_tests/actions_test/_trial_temp/_trial_marker | 0 tests/scenarios_tests/behaviors_test/_trial_temp/_trial_marker | 0 tests/smart_kit_tests/models/_trial_temp/_trial_marker | 0 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100755 tests/core_tests/requirements_test/_trial_temp/_trial_marker delete mode 100755 tests/scenarios_tests/actions_test/_trial_temp/_trial_marker delete mode 100755 tests/scenarios_tests/behaviors_test/_trial_temp/_trial_marker delete mode 100755 tests/smart_kit_tests/models/_trial_temp/_trial_marker diff --git a/tests/core_tests/requirements_test/_trial_temp/_trial_marker b/tests/core_tests/requirements_test/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker b/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/scenarios_tests/behaviors_test/_trial_temp/_trial_marker b/tests/scenarios_tests/behaviors_test/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/smart_kit_tests/models/_trial_temp/_trial_marker b/tests/smart_kit_tests/models/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 From 9757553710bc6a47348c6799cf9de80f1a2e3833 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 11:40:31 +0300 Subject: [PATCH 040/116] async tests for dialogue_manager --- .../models/test_dialogue_manager.py | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/tests/smart_kit_tests/models/test_dialogue_manager.py b/tests/smart_kit_tests/models/test_dialogue_manager.py index 1618a162..ce5a1036 100644 --- a/tests/smart_kit_tests/models/test_dialogue_manager.py +++ b/tests/smart_kit_tests/models/test_dialogue_manager.py @@ -9,6 +9,22 @@ def get_keys(self): return self.keys() +async def mock_scenario1_text_fits(): + return False + + +async def mock_scenario2_text_fits(): + return True + + +async def mock_scenario1_run(x, y): + return x.name + y.name + + +async def mock_scenario2_run(x, y): + return y.name + x.name + + class ModelsTest1(unittest.IsolatedAsyncioTestCase): def setUp(self): self.test_user1 = Mock() @@ -30,12 +46,12 @@ def setUp(self): self.test_text_preprocessing_result.name = "Result" self.test_scenario1 = Mock() self.test_scenario1.scenario_description = "This is test scenario 1 desc" - self.test_scenario1.text_fits = lambda x, y: False - self.test_scenario1.run = lambda x, y: x.name + y.name + self.test_scenario1.text_fits = mock_scenario1_text_fits + self.test_scenario1.run = mock_scenario1_run self.test_scenario2 = Mock() self.test_scenario2.scenario_description = "This is test scenario 2 desc" - self.test_scenario2.text_fits = lambda x, y: True - self.test_scenario2.run = lambda x, y: y.name + x.name + self.test_scenario2.text_fits = mock_scenario2_text_fits + self.test_scenario2.run = mock_scenario2_run self.test_scenarios = TestScenarioDesc({1: self.test_scenario1, 2: self.test_scenario2}) self.TestAction = Mock() # должно быть async? self.TestAction.description = "test_function" @@ -77,17 +93,30 @@ async def test_dialogue_manager_run(self): 'external_actions': {}}, self.app_name) # путь по умолчанию без выполнения условий - self.assertTrue(await obj1.run(self.test_text_preprocessing_result, self.test_user1) == ("TestNameResult", True)) - self.assertTrue(await obj2.run(self.test_text_preprocessing_result, self.test_user1) == ("TestNameResult", True)) + self.assertTrue( + await obj1.run(self.test_text_preprocessing_result, self.test_user1) == ("TestNameResult", True) + ) + self.assertTrue( + await obj2.run(self.test_text_preprocessing_result, self.test_user1) == ("TestNameResult", True) + ) # случай когда срабатоли оба условия - self.assertTrue(obj1.run(self.test_text_preprocessing_result, self.test_user2) == ("TestNameResult", True)) + self.assertTrue( + await obj1.run(self.test_text_preprocessing_result, self.test_user2) == ("TestNameResult", True) + ) # случай, когда 2-е условие не выполнено - self.assertTrue(obj2.run(self.test_text_preprocessing_result, self.test_user3) == ('TestNameResult', True)) + self.assertTrue( + await obj2.run(self.test_text_preprocessing_result, self.test_user3) == ('TestNameResult', True) + ) async def test_dialogue_manager_run_scenario(self): obj = dialogue_manager.DialogueManager({'scenarios': self.test_scenarios, 'external_actions': {'nothing_found_action': self.TestAction}}, self.app_name) - self.assertTrue(await obj.run_scenario(1, self.test_text_preprocessing_result, self.test_user1) == "ResultTestName") - self.assertTrue(await obj.run_scenario(2, self.test_text_preprocessing_result, self.test_user1) == "TestNameResult") + print(await obj.run_scenario(1, self.test_text_preprocessing_result, self.test_user1)) + self.assertTrue( + await obj.run_scenario(1, self.test_text_preprocessing_result, self.test_user1) == "ResultTestName" + ) + self.assertTrue( + await obj.run_scenario(2, self.test_text_preprocessing_result, self.test_user1) == "TestNameResult" + ) From 2475255a61666f9f13ea3d6f7329d29df020d044 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 11:46:10 +0300 Subject: [PATCH 041/116] async tests for device_requirements --- .../requirement/test_device_requirements.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/smart_kit_tests/requirement/test_device_requirements.py b/tests/smart_kit_tests/requirement/test_device_requirements.py index f92bd2e7..5b2b7db9 100644 --- a/tests/smart_kit_tests/requirement/test_device_requirements.py +++ b/tests/smart_kit_tests/requirement/test_device_requirements.py @@ -5,7 +5,7 @@ from core.basic_models.requirement import device_requirements -class RequirementTest1(unittest.TestCase): +class RequirementTest1(unittest.IsolatedAsyncioTestCase): def setUp(self): self.test_items1 = {"platfrom_type": "any platform"} # PLATFROM - так задумано? self.test_items2 = {"platform_type": "any platform 2"} @@ -30,11 +30,10 @@ def test_platform_type_requirement_init(self): with self.assertRaises(KeyError): obj3 = device_requirements.PlatformTypeRequirement(self.test_items2, self.test_id) - def test_platform_type_requirement_check(self): + async def test_platform_type_requirement_check(self): obj1 = device_requirements.PlatformTypeRequirement(self.test_items1, self.test_id) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(obj1.check(self.test_text_processing_result, self.test_user1))) - self.assertTrue(not loop.run_until_complete(obj1.check(self.test_text_processing_result, self.test_user2))) + self.assertTrue(await obj1.check(self.test_text_processing_result, self.test_user1)) + self.assertTrue(not await obj1.check(self.test_text_processing_result, self.test_user2)) class RequirementTest2(unittest.TestCase): From cc52688d7438fea75dbca984b0e41dc673bacfff Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 11:48:33 +0300 Subject: [PATCH 042/116] remove useless imports and comments --- .../scenarios_tests/actions_test/_trial_temp/_trial_marker | 0 tests/smart_kit_tests/models/_trial_temp/_trial_marker | 0 tests/smart_kit_tests/models/test_dialogue_manager.py | 6 +++--- tests/smart_kit_tests/requirement/_trial_temp/_trial_marker | 0 4 files changed, 3 insertions(+), 3 deletions(-) create mode 100755 tests/scenarios_tests/actions_test/_trial_temp/_trial_marker create mode 100755 tests/smart_kit_tests/models/_trial_temp/_trial_marker create mode 100755 tests/smart_kit_tests/requirement/_trial_temp/_trial_marker diff --git a/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker b/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/smart_kit_tests/models/_trial_temp/_trial_marker b/tests/smart_kit_tests/models/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/smart_kit_tests/models/test_dialogue_manager.py b/tests/smart_kit_tests/models/test_dialogue_manager.py index ce5a1036..a57e6a95 100644 --- a/tests/smart_kit_tests/models/test_dialogue_manager.py +++ b/tests/smart_kit_tests/models/test_dialogue_manager.py @@ -1,6 +1,6 @@ # coding: utf-8 import unittest -from unittest.mock import Mock, AsyncMock +from unittest.mock import Mock from smart_kit.models import dialogue_manager @@ -53,9 +53,9 @@ def setUp(self): self.test_scenario2.text_fits = mock_scenario2_text_fits self.test_scenario2.run = mock_scenario2_run self.test_scenarios = TestScenarioDesc({1: self.test_scenario1, 2: self.test_scenario2}) - self.TestAction = Mock() # должно быть async? + self.TestAction = Mock() self.TestAction.description = "test_function" - self.TestAction.run = lambda x, y: x.name + y.name # должно быть async? + self.TestAction.run = lambda x, y: x.name + y.name self.app_name = "test" def test_log_const(self): diff --git a/tests/smart_kit_tests/requirement/_trial_temp/_trial_marker b/tests/smart_kit_tests/requirement/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b From eae3bfcda5c0f0f84d2227bd60c87e4db4345ce7 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 11:49:39 +0300 Subject: [PATCH 043/116] remove useless files --- tests/scenarios_tests/actions_test/_trial_temp/_trial_marker | 0 tests/smart_kit_tests/models/_trial_temp/_trial_marker | 0 tests/smart_kit_tests/requirement/_trial_temp/_trial_marker | 0 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100755 tests/scenarios_tests/actions_test/_trial_temp/_trial_marker delete mode 100755 tests/smart_kit_tests/models/_trial_temp/_trial_marker delete mode 100755 tests/smart_kit_tests/requirement/_trial_temp/_trial_marker diff --git a/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker b/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/smart_kit_tests/models/_trial_temp/_trial_marker b/tests/smart_kit_tests/models/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/smart_kit_tests/requirement/_trial_temp/_trial_marker b/tests/smart_kit_tests/requirement/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 From d99b4265d81f69b2b97119cc91f797ef95dee813 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 12:35:34 +0300 Subject: [PATCH 044/116] async tests for requirements --- tests/core_tests/requirements_test/test_requirements.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/core_tests/requirements_test/test_requirements.py b/tests/core_tests/requirements_test/test_requirements.py index 93aadac4..21965228 100644 --- a/tests/core_tests/requirements_test/test_requirements.py +++ b/tests/core_tests/requirements_test/test_requirements.py @@ -1,3 +1,4 @@ +import asyncio import os import unittest from time import time @@ -19,6 +20,10 @@ from smart_kit.text_preprocessing.local_text_normalizer import LocalTextNormalizer +def _run(coro): + return asyncio.get_event_loop().run_until_complete(coro) + + def patch_get_app_config(mock_get_app_config): result = Mock() sk_path = os.path.dirname(smart_kit.__file__) @@ -217,7 +222,7 @@ async def test_template_req_raise(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - self.assertRaises(TypeError, requirement.check, None, user) + self.assertRaises(TypeError, _run, requirement.check, None, user) async def test_rolling_requirement_true(self): user = Mock() @@ -464,6 +469,7 @@ async def test_num_in_range_requirement_true(self): """Тест кейз проверяет что условие возвращает True, т.к число находится в заданном диапазоне.""" req = NumInRangeRequirement({"min_num": "5", "max_num": "10"}) text_preprocessing_result = Mock() + text_preprocessing_result.num_token_values = 7 self.assertTrue(await req.check(text_preprocessing_result, Mock())) async def test_num_in_range_requirement_false(self): From 15f1fa6cd5de9c3365d7da540d8c223df064fbe9 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 14:24:11 +0300 Subject: [PATCH 045/116] async tests fot token_part_in_set_requirements --- .../test_token_part_in_set_requirement.py | 326 +++++++++--------- 1 file changed, 167 insertions(+), 159 deletions(-) diff --git a/tests/scenarios_tests/user_models/test_token_part_in_set_requirement.py b/tests/scenarios_tests/user_models/test_token_part_in_set_requirement.py index d2bd095a..e5ef7e96 100644 --- a/tests/scenarios_tests/user_models/test_token_part_in_set_requirement.py +++ b/tests/scenarios_tests/user_models/test_token_part_in_set_requirement.py @@ -1,182 +1,190 @@ # coding: utf-8 -import asyncio import unittest from scenarios.scenario_models.field_requirements.field_requirements import TokenPartInSet -class RequirementTest(unittest.TestCase): +class RequirementTest(unittest.IsolatedAsyncioTestCase): - def test_token_part_in_set_requirement_equal_false(self): - requirement_items ={ - "type": "token_part_in_set", - "part": "locality_type", - "values": ["DISTRICT", "REGION"] - } - token_val = {'value': 'Амадора', - 'locality_type': 'CITY', - 'latitude': 38.75382, - 'longitude': -9.23083, - 'capital': None, - 'locative_value': None, - 'timezone': [[None, 1.0]], - 'currency': ['EUR', 'евро'], - 'country': 'Португалия', - 'country_hidden': False} + async def test_token_part_in_set_requirement_equal_false(self): + requirement_items = { + "type": "token_part_in_set", + "part": "locality_type", + "values": ["DISTRICT", "REGION"] + } + token_val = { + 'value': 'Амадора', + 'locality_type': 'CITY', + 'latitude': 38.75382, + 'longitude': -9.23083, + 'capital': None, + 'locative_value': None, + 'timezone': [[None, 1.0]], + 'currency': ['EUR', 'евро'], + 'country': 'Португалия', + 'country_hidden': False + } requirement = TokenPartInSet(requirement_items) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(token_val))) + self.assertFalse(await requirement.check(token_val)) - def test_token_part_in_set_requirement_equal_true(self): - requirement_items ={ - "type": "token_part_in_set", - "part": "locality_type", - "values": ["DISTRICT", "CITY"] - } - token_val = {'value': 'Амадора', - 'locality_type': 'CITY', - 'latitude': 38.75382, - 'longitude': -9.23083, - 'capital': None, - 'locative_value': None, - 'timezone': [[None, 1.0]], - 'currency': ['EUR', 'евро'], - 'country': 'Португалия', - 'country_hidden': False} + async def test_token_part_in_set_requirement_equal_true(self): + requirement_items = { + "type": "token_part_in_set", + "part": "locality_type", + "values": ["DISTRICT", "CITY"] + } + token_val = { + 'value': 'Амадора', + 'locality_type': 'CITY', + 'latitude': 38.75382, + 'longitude': -9.23083, + 'capital': None, + 'locative_value': None, + 'timezone': [[None, 1.0]], + 'currency': ['EUR', 'евро'], + 'country': 'Португалия', + 'country_hidden': False + } requirement = TokenPartInSet(requirement_items) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(token_val))) + self.assertTrue(await requirement.check(token_val)) - def test_token_part_in_set_requirement_equal_False_double_empty(self): - requirement_items ={ - "type": "token_part_in_set", - "part": "locality_type", - "values": [] - } - token_val = {'value': 'Амадора', - 'latitude': 38.75382, - 'longitude': -9.23083, - 'capital': None, - 'locative_value': None, - 'timezone': [[None, 1.0]], - 'currency': ['EUR', 'евро'], - 'country': 'Португалия', - 'country_hidden': False, - 'locality_type': []} + async def test_token_part_in_set_requirement_equal_False_double_empty(self): + requirement_items = { + "type": "token_part_in_set", + "part": "locality_type", + "values": [] + } + token_val = { + 'value': 'Амадора', + 'latitude': 38.75382, + 'longitude': -9.23083, + 'capital': None, + 'locative_value': None, + 'timezone': [[None, 1.0]], + 'currency': ['EUR', 'евро'], + 'country': 'Португалия', + 'country_hidden': False, + 'locality_type': [] + } requirement = TokenPartInSet(requirement_items) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(token_val))) + self.assertFalse(await requirement.check(token_val)) - def test_token_part_in_set_requirement_equal_False_empty_val_none(self): - requirement_items ={ - "type": "token_part_in_set", - "part": "locality_type", - "values": [] - } - token_val = {'value': 'Амадора', - 'latitude': 38.75382, - 'longitude': -9.23083, - 'capital': None, - 'locative_value': None, - 'timezone': [[None, 1.0]], - 'currency': ['EUR', 'евро'], - 'country': 'Португалия', - 'country_hidden': False, - 'locality_type': None} + async def test_token_part_in_set_requirement_equal_False_empty_val_none(self): + requirement_items = { + "type": "token_part_in_set", + "part": "locality_type", + "values": [] + } + token_val = { + 'value': 'Амадора', + 'latitude': 38.75382, + 'longitude': -9.23083, + 'capital': None, + 'locative_value': None, + 'timezone': [[None, 1.0]], + 'currency': ['EUR', 'евро'], + 'country': 'Португалия', + 'country_hidden': False, + 'locality_type': None + } requirement = TokenPartInSet(requirement_items) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(token_val))) + self.assertFalse(await requirement.check(token_val)) - def test_token_part_in_set_requirement_equal_False_string(self): - requirement_items ={ - "type": "token_part_in_set", - "part": "value", - "values": 'cba' - } - token_val = {'value': 'abc', - 'latitude': 38.75382, - 'longitude': -9.23083, - 'capital': None, - 'locative_value': None, - 'timezone': [[None, 1.0]], - 'currency': ['EUR', 'евро'], - 'country': 'Португалия', - 'country_hidden': False} + async def test_token_part_in_set_requirement_equal_False_string(self): + requirement_items = { + "type": "token_part_in_set", + "part": "value", + "values": 'cba' + } + token_val = { + 'value': 'abc', + 'latitude': 38.75382, + 'longitude': -9.23083, + 'capital': None, + 'locative_value': None, + 'timezone': [[None, 1.0]], + 'currency': ['EUR', 'евро'], + 'country': 'Португалия', + 'country_hidden': False + } requirement = TokenPartInSet(requirement_items) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(token_val))) + self.assertFalse(await requirement.check(token_val)) - def test_token_part_in_set_requirement_equal_False(self): - requirement_items ={ - "type": "token_part_in_set", - "part": "country_hidden", - "values": [1, 2, 3] - } - token_val = {'value': 'Амадора', - 'latitude': 38.75382, - 'longitude': -9.23083, - 'capital': None, - 'locative_value': None, - 'timezone': [[None, 1.0]], - 'currency': ['EUR', 'евро'], - 'country': 'Португалия', - 'country_hidden': False} + async def test_token_part_in_set_requirement_equal_False(self): + requirement_items = { + "type": "token_part_in_set", + "part": "country_hidden", + "values": [1, 2, 3] + } + token_val = { + 'value': 'Амадора', + 'latitude': 38.75382, + 'longitude': -9.23083, + 'capital': None, + 'locative_value': None, + 'timezone': [[None, 1.0]], + 'currency': ['EUR', 'евро'], + 'country': 'Португалия', + 'country_hidden': False + } requirement = TokenPartInSet(requirement_items) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(token_val))) + self.assertFalse(await requirement.check(token_val)) - def test_token_part_in_set_requirement_equal_False_val_int(self): - requirement_items ={ - "type": "token_part_in_set", - "part": 'capital', - "values": [-9.23083] - } - token_val = {'value': 'Амадора', - 'latitude': 38.75382, - 'longitude': -9.23083, - 'capital': None, - 'locative_value': None, - 'timezone': [[None, 1.0]], - 'currency': ['EUR', 'евро'], - 'country': 'Португалия', - 'country_hidden': False} + async def test_token_part_in_set_requirement_equal_False_val_int(self): + requirement_items = { + "type": "token_part_in_set", + "part": 'capital', + "values": [-9.23083] + } + token_val = { + 'value': 'Амадора', + 'latitude': 38.75382, + 'longitude': -9.23083, + 'capital': None, + 'locative_value': None, + 'timezone': [[None, 1.0]], + 'currency': ['EUR', 'евро'], + 'country': 'Португалия', + 'country_hidden': False + } requirement = TokenPartInSet(requirement_items) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(token_val))) + self.assertFalse(await requirement.check(token_val)) - def test_token_part_in_set_requirement_equal_True_arr(self): - requirement_items ={ - "type": "token_part_in_set", - "part": 'timezone', - "values": [[[None, 1.0]]] - } - token_val = {'value': 'Амадора', - 'latitude': 38.75382, - 'longitude': -9.23083, - 'capital': None, - 'locative_value': None, - 'timezone': [[None, 1.0]], - 'currency': ['EUR', 'евро'], - 'country': 'Португалия', - 'country_hidden': False} + async def test_token_part_in_set_requirement_equal_True_arr(self): + requirement_items = { + "type": "token_part_in_set", + "part": 'timezone', + "values": [[[None, 1.0]]] + } + token_val = { + 'value': 'Амадора', + 'latitude': 38.75382, + 'longitude': -9.23083, + 'capital': None, + 'locative_value': None, + 'timezone': [[None, 1.0]], + 'currency': ['EUR', 'евро'], + 'country': 'Португалия', + 'country_hidden': False + } requirement = TokenPartInSet(requirement_items) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(token_val))) + self.assertTrue(await requirement.check(token_val)) - def test_token_part_in_set_requirement_equal_False_arr(self): - requirement_items ={ - "type": "token_part_in_set", - "part": 'timezone', - "values": [[[1.0, None]]] - } - token_val = {'value': 'Амадора', - 'latitude': 38.75382, - 'longitude': -9.23083, - 'capital': None, - 'locative_value': None, - 'timezone': [[None, 1.0]], - 'currency': ['EUR', 'евро'], - 'country': 'Португалия', - 'country_hidden': False} + async def test_token_part_in_set_requirement_equal_False_arr(self): + requirement_items = { + "type": "token_part_in_set", + "part": 'timezone', + "values": [[[1.0, None]]] + } + token_val = { + 'value': 'Амадора', + 'latitude': 38.75382, + 'longitude': -9.23083, + 'capital': None, + 'locative_value': None, + 'timezone': [[None, 1.0]], + 'currency': ['EUR', 'евро'], + 'country': 'Португалия', + 'country_hidden': False + } requirement = TokenPartInSet(requirement_items) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(token_val))) + self.assertFalse(await requirement.check(token_val)) From e70970fc4551060f077f6b1d78aa475d97b37c4a Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 14:40:21 +0300 Subject: [PATCH 046/116] async test_is_int_value --- .../user_models/test_is_int_value.py | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/tests/scenarios_tests/user_models/test_is_int_value.py b/tests/scenarios_tests/user_models/test_is_int_value.py index 19bb99c0..a719762d 100644 --- a/tests/scenarios_tests/user_models/test_is_int_value.py +++ b/tests/scenarios_tests/user_models/test_is_int_value.py @@ -1,33 +1,28 @@ # coding: utf-8 -import asyncio -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from scenarios.scenario_models.field_requirements.field_requirements import IsIntFieldRequirement -class IsIntFieldRequirementTest(TestCase): +class IsIntFieldRequirementTest(IsolatedAsyncioTestCase): @classmethod - def setUpClass(cls): + def setUp(cls): items = {} cls.requirement = IsIntFieldRequirement(items) - def test_is_int_number_string(self): + async def test_is_int_number_string(self): text = "123" - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(self.requirement.check(text))) + self.assertTrue(await self.requirement.check(text)) - def test_is_int_float_string(self): + async def test_is_int_float_string(self): text = "1.23" - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(self.requirement.check(text))) + self.assertFalse(await self.requirement.check(text)) - def test_is_int_text_string(self): + async def test_is_int_text_string(self): text = "test" - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(self.requirement.check(text))) + self.assertFalse(await self.requirement.check(text)) - def test_is_int_empty_string(self): + async def test_is_int_empty_string(self): text = "" - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(self.requirement.check(text))) + self.assertFalse(await self.requirement.check(text)) From 91d5798f9e65691e3827de26721365f324669cc1 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 14:45:07 +0300 Subject: [PATCH 047/116] async tests for requirements --- .../requirements_test/test_requirements.py | 43 ++++++++----------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/tests/scenarios_tests/requirements_test/test_requirements.py b/tests/scenarios_tests/requirements_test/test_requirements.py index a8c4c5c6..814a4ed3 100644 --- a/tests/scenarios_tests/requirements_test/test_requirements.py +++ b/tests/scenarios_tests/requirements_test/test_requirements.py @@ -1,5 +1,4 @@ # coding: utf-8 -import asyncio import unittest from unittest.mock import Mock @@ -55,9 +54,9 @@ def compare(self, value): return value == self.amount -class RequirementTest(unittest.TestCase): +class RequirementTest(unittest.IsolatedAsyncioTestCase): - def test_template_in_array_req_true(self): + async def test_template_in_array_req_true(self): items = { "template": "{{ payload.userInfo.tbcode }}", "items": ["32", "33"] @@ -72,10 +71,9 @@ def test_template_in_array_req_true(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, user))) + self.assertTrue(await requirement.check(None, user)) - def test_template_in_array_req_true2(self): + async def test_template_in_array_req_true2(self): items = { "template": "{{ payload.message.strip() }}", "items": ["AAA", "BBB", "CCC"] @@ -90,10 +88,9 @@ def test_template_in_array_req_true2(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, user))) + self.assertTrue(await requirement.check(None, user)) - def test_template_in_array_req_false(self): + async def test_template_in_array_req_false(self): items = { "template": "{{ payload.message.strip() }}", "items": ["AAA", "CCC"] @@ -108,10 +105,9 @@ def test_template_in_array_req_false(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(None, user))) + self.assertFalse(await requirement.check(None, user)) - def test_array_in_template_req_true(self): + async def test_array_in_template_req_true(self): items = { "template": { "type": "unified_template", @@ -131,10 +127,9 @@ def test_array_in_template_req_true(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, user))) + self.assertTrue(await requirement.check(None, user)) - def test_array_in_template_req_true2(self): + async def test_array_in_template_req_true2(self): items = { "template": "{{ payload.message.strip() }}", "items": ["AAA", "BBB"] @@ -150,10 +145,9 @@ def test_array_in_template_req_true2(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, user))) + self.assertTrue(await requirement.check(None, user)) - def test_array_in_template_req_false(self): + async def test_array_in_template_req_false(self): items = { "template": { "type": "unified_template", @@ -173,10 +167,9 @@ def test_array_in_template_req_false(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(None, user))) + self.assertFalse(await requirement.check(None, user)) - def test_regexp_in_template_req_true(self): + async def test_regexp_in_template_req_true(self): items = { "template": "{{ payload.message.strip() }}", "regexp": "(^|\s)[Фф](\.|-)?1(\-)?(у|У)?($|\s)" @@ -191,10 +184,9 @@ def test_regexp_in_template_req_true(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertTrue(loop.run_until_complete(requirement.check(None, user))) + self.assertTrue(await requirement.check(None, user)) - def test_regexp_in_template_req_false(self): + async def test_regexp_in_template_req_false(self): items = { "template": "{{ payload.message.strip() }}", "regexp": "(^|\s)[Фф](\.|-)?1(\-)?(у|У)?($|\s)" @@ -209,8 +201,7 @@ def test_regexp_in_template_req_false(self): user = Mock() user.parametrizer = Mock() user.parametrizer.collect = Mock(return_value=params) - loop = asyncio.get_event_loop() - self.assertFalse(loop.run_until_complete(requirement.check(None, user))) + self.assertFalse(await requirement.check(None, user)) if __name__ == '__main__': From 2f42418c1aafea2a3b2a051e35a94fe1dc68c735 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 14:55:11 +0300 Subject: [PATCH 048/116] async test_handler_timeout --- tests/smart_kit_tests/handlers/test_handler_timeout.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/smart_kit_tests/handlers/test_handler_timeout.py b/tests/smart_kit_tests/handlers/test_handler_timeout.py index 6ac6ceee..0af6fcf1 100644 --- a/tests/smart_kit_tests/handlers/test_handler_timeout.py +++ b/tests/smart_kit_tests/handlers/test_handler_timeout.py @@ -4,7 +4,11 @@ from smart_kit.handlers import handler_timeout -class HandlerTest2(unittest.TestCase): +async def mock_behaviors_timeout(x): + return 120 + + +class HandlerTest2(unittest.IsolatedAsyncioTestCase): def setUp(self): self.app_name = "TastAppName" self.test_user = Mock('user') @@ -20,7 +24,7 @@ def setUp(self): self.test_user.message.device.surface = "surface" self.test_user.behaviors = Mock('behaviors') - self.test_user.behaviors.timeout = lambda x: 120 + self.test_user.behaviors.timeout = mock_behaviors_timeout self.test_user.behaviors.has_callback = lambda *x, **y: MagicMock() self.test_user.behaviors.get_callback_action_params = lambda *x, **y: {} self.test_payload = Mock('payload') From 1a621feea2ed47562822be295516dd4bb2460c36 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 14:59:20 +0300 Subject: [PATCH 049/116] async test_handler_text --- .../smart_kit_tests/handlers/test_handler_text.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/smart_kit_tests/handlers/test_handler_text.py b/tests/smart_kit_tests/handlers/test_handler_text.py index 2f7ea5eb..6fe2f074 100644 --- a/tests/smart_kit_tests/handlers/test_handler_text.py +++ b/tests/smart_kit_tests/handlers/test_handler_text.py @@ -4,13 +4,21 @@ from smart_kit.handlers import handler_text -class HandlerTest5(unittest.TestCase): +async def mock_dialogue_manager1_run(x, y): + return "TestAnswer", True + + +async def mock_dialogue_manager2_run(x, y): + return "", False + + +class HandlerTest5(unittest.IsolatedAsyncioTestCase): def setUp(self): self.app_name = "TestAppName" self.test_dialog_manager1 = Mock('dialog_manager') - self.test_dialog_manager1.run = lambda x, y: ("TestAnswer", True) + self.test_dialog_manager1.run = mock_dialogue_manager1_run self.test_dialog_manager2 = Mock('dialog_manager') - self.test_dialog_manager2.run = lambda x, y: ("", False) + self.test_dialog_manager2.run = mock_dialogue_manager2_run self.test_text_preprocessing_result = Mock('text_preprocessing_result') self.test_text_preprocessing_result.raw = 'any raw' self.test_user = Mock('User') From ba95af5e9bddf17321f9f17c79c94fb59e8f13b8 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 15:01:17 +0300 Subject: [PATCH 050/116] async test_nothing_found_action --- .../smart_kit_tests/system_answers/test_nothing_found_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/smart_kit_tests/system_answers/test_nothing_found_action.py b/tests/smart_kit_tests/system_answers/test_nothing_found_action.py index 28fd67d2..1936cdaa 100644 --- a/tests/smart_kit_tests/system_answers/test_nothing_found_action.py +++ b/tests/smart_kit_tests/system_answers/test_nothing_found_action.py @@ -7,7 +7,7 @@ from core.basic_models.actions.command import Command -class SystemAnswersTest1(unittest.TestCase): +class SystemAnswersTest1(unittest.IsolatedAsyncioTestCase): def setUp(self): self.test_command_1 = Mock('Command') self.test_id = '123-345-678' # пусть чему-то равняется From ca997ed85ae7f22d26176a92699473ac77f3ce8f Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 15:06:29 +0300 Subject: [PATCH 051/116] async test_handle_respond --- tests/smart_kit_tests/handlers/test_handle_respond.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/smart_kit_tests/handlers/test_handle_respond.py b/tests/smart_kit_tests/handlers/test_handle_respond.py index 531d612c..d0fc1358 100644 --- a/tests/smart_kit_tests/handlers/test_handle_respond.py +++ b/tests/smart_kit_tests/handlers/test_handle_respond.py @@ -4,7 +4,11 @@ from smart_kit.handlers import handle_respond -class HandlerTest4(unittest.TestCase): +async def mock_test_action_run(x, y, z): + return 10 + + +class HandlerTest4(unittest.IsolatedAsyncioTestCase): def setUp(self): self.app_name = "TestAppName" self.test_user1 = Mock('user') @@ -21,8 +25,8 @@ def setUp(self): self.test_user1.message.app_info = {} self.test_user1.behaviors = MagicMock() - self.test_action = Mock('action') # должно быть async? - self.test_action.run = lambda x, y, z: 10 # пусть что то возвращает. Должно быть async? + self.test_action = Mock('action') + self.test_action.run = mock_test_action_run # пусть что то возвращает. self.test_user2 = MagicMock('user') self.test_user2.id = '123-345-678' # пусть чему-то равняется self.test_user2.descriptions = {'external_actions': {'any action name': self.test_action}} From a4d14c8bb2fcb950b59344edf2bbaa063e1a15eb Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 15:09:23 +0300 Subject: [PATCH 052/116] async test_handle_close_app --- tests/smart_kit_tests/handlers/test_handle_close_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/smart_kit_tests/handlers/test_handle_close_app.py b/tests/smart_kit_tests/handlers/test_handle_close_app.py index a2ad1245..f15160be 100644 --- a/tests/smart_kit_tests/handlers/test_handle_close_app.py +++ b/tests/smart_kit_tests/handlers/test_handle_close_app.py @@ -9,7 +9,7 @@ def form_type(self): return 'type' -class HandlerTest6(unittest.TestCase): +class HandlerTest6(unittest.IsolatedAsyncioTestCase): def setUp(self): self.test_text_preprocessing_result = Mock('text_preprocessing_result') self.test_user = Mock() From 0667020403ab19374477a14216b71c8432f9bb35 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 15:11:23 +0300 Subject: [PATCH 053/116] async test_run_scenario_by_project_name --- .../smart_kit_tests/action/test_run_scenario_by_project_name.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py b/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py index be29a6c1..0c3288fe 100644 --- a/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py +++ b/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py @@ -11,7 +11,7 @@ def run(self, argv1, argv2, params): return 'result to run scenario' -class RunScenarioByProjectNameActionTest1(unittest.TestCase): +class RunScenarioByProjectNameActionTest1(unittest.IsolatedAsyncioTestCase): def setUp(self): self.test_text_preprocessing_result = Mock('text_preprocessing_result') self.test_user1 = Mock('User') From 98e6863ce930d15af6985d984b9a33d8434faea2 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 15:20:51 +0300 Subject: [PATCH 054/116] async test_composite_filler --- .../scenarios_tests/fillers/test_composite_filler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/scenarios_tests/fillers/test_composite_filler.py b/tests/scenarios_tests/fillers/test_composite_filler.py index fe9e2b49..cd1fa5fe 100644 --- a/tests/scenarios_tests/fillers/test_composite_filler.py +++ b/tests/scenarios_tests/fillers/test_composite_filler.py @@ -1,4 +1,4 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from scenarios.scenario_models.field.field_filler_description import FieldFillerDescription, CompositeFiller @@ -11,17 +11,17 @@ def __init__(self, items=None): items = items or {} self.result = items.get("result") - def extract(self, text_preprocessing_result, user, params): + async def extract(self, text_preprocessing_result, user, params): return self.result -class TestCompositeFiller(TestCase): +class TestCompositeFiller(IsolatedAsyncioTestCase): @classmethod - def setUpClass(cls): + def setUp(cls): registered_factories[FieldFillerDescription] = field_filler_factory field_filler_description["mock_filler"] = MockFiller - TestCompositeFiller.user = Mock() + cls.user = Mock() async def test_first_filler(self): expected = "first" @@ -56,4 +56,4 @@ async def test_not_fit(self): } filler = CompositeFiller(items) result = await filler.extract(None, self.user) - self.assertIsNone(result) \ No newline at end of file + self.assertIsNone(result) From 560d1c510688d1f3cf9693981bb94a23d4ea017a Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 10 Nov 2021 15:21:46 +0300 Subject: [PATCH 055/116] small refactoring --- scenarios/scenario_models/field/field_filler_description.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/scenario_models/field/field_filler_description.py b/scenarios/scenario_models/field/field_filler_description.py index 3dc4ffc7..4f5da0e1 100644 --- a/scenarios/scenario_models/field/field_filler_description.py +++ b/scenarios/scenario_models/field/field_filler_description.py @@ -44,7 +44,7 @@ def _log_params(self): } async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> None: + params: Dict[str, Any] = None) -> None: return None def on_extract_error(self, text_preprocessing_result, user, params=None): From 2dc233cef34416157b3c6de6a7ef448bbeb12dee Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Thu, 11 Nov 2021 09:27:16 +0300 Subject: [PATCH 056/116] fix tests --- scenarios/actions/action.py | 6 ++++-- tests/scenarios_tests/actions_test/test_action.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scenarios/actions/action.py b/scenarios/actions/action.py index 72ab959f..c9cb1ddf 100644 --- a/scenarios/actions/action.py +++ b/scenarios/actions/action.py @@ -222,7 +222,7 @@ def __init__(self, items: Dict[str, Any], id: Optional[str] = None): super(SetVariableAction, self).__init__(items, id) self.ttl: int = items.get("ttl") - async def _set(self, user, value): + def _set(self, user, value): user.variables.set(self.key, value, self.ttl) @@ -309,7 +309,9 @@ async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessing else: params.update(user.parametrizer.collect(text_preprocessing_result)) scenario_id = self.scenario.render(params) - scenario = user.descriptions["scenarios"].get(scenario_id) + scenario = user.descriptions + scenario = scenario["scenarios"] + scenario = scenario.get(scenario_id) if scenario: return scenario.run(text_preprocessing_result, user, params) diff --git a/tests/scenarios_tests/actions_test/test_action.py b/tests/scenarios_tests/actions_test/test_action.py index a0aa16f1..b8d21757 100644 --- a/tests/scenarios_tests/actions_test/test_action.py +++ b/tests/scenarios_tests/actions_test/test_action.py @@ -427,7 +427,8 @@ async def mock_and_perform_action(test_items: Dict[str, Any], expected_result: O scen = Mock() scen.run.return_value = expected_result if expected_scen: - return await action.run(user, Mock()) + user.descriptions = {"scenarios": {expected_scen: scen}} + return await action.run(user, Mock()) async def test_choice_scenario_action(self): # Проверяем, что запустили нужный сценарий, в случае если выполнился его requirement From de458f8074315ed268ae852ff511ae44436cc250 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Thu, 11 Nov 2021 09:50:20 +0300 Subject: [PATCH 057/116] update smartapp template --- smart_kit/template/app/basic_entities/fillers.py-tpl | 2 +- smart_kit/template/app/handlers/handlers.py-tpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/smart_kit/template/app/basic_entities/fillers.py-tpl b/smart_kit/template/app/basic_entities/fillers.py-tpl index 05234d32..2f3152e2 100644 --- a/smart_kit/template/app/basic_entities/fillers.py-tpl +++ b/smart_kit/template/app/basic_entities/fillers.py-tpl @@ -20,5 +20,5 @@ class CustomFieldFiller(FieldFillerDescription): self.test_item = items.get("test_item") @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, params) -> Optional[str]: + async def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, params) -> Optional[str]: return None diff --git a/smart_kit/template/app/handlers/handlers.py-tpl b/smart_kit/template/app/handlers/handlers.py-tpl index e5d4a6f3..faaca554 100644 --- a/smart_kit/template/app/handlers/handlers.py-tpl +++ b/smart_kit/template/app/handlers/handlers.py-tpl @@ -8,5 +8,5 @@ class CustomHandler(HandlerBase): Тут создаются Handlers, которые используются для запуска логики в зависимости от типа входяшего сообщения """ - def run(self, payload, user): + async def run(self, payload, user): return [] From cf0fb4cb2b698275f9779c28d5a48c5fc9d7af3c Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Thu, 11 Nov 2021 10:28:23 +0300 Subject: [PATCH 058/116] fix tests --- scenarios/actions/action.py | 4 +--- .../template/app/adapters/db_adapters.py-tpl | 22 +++++++++---------- .../app/basic_entities/actions.py-tpl | 8 +++---- .../app/basic_entities/fillers.py-tpl | 8 +++---- .../app/basic_entities/requirements.py-tpl | 8 +++---- .../template/app/handlers/handlers.py-tpl | 2 +- .../local_testing/custom_local_testing.py-tpl | 12 +++++----- .../app/models/dialogue_manager.py-tpl | 4 ++-- smart_kit/template/app/models/model.py-tpl | 6 ++--- .../app/resources/custom_app_resourses.py-tpl | 22 +++++++++---------- .../template/app/user/parametrizer.py-tpl | 8 +++---- smart_kit/template/app/user/user.py-tpl | 16 +++++++------- .../actions_test/_trial_temp/_trial_marker | 0 .../actions_test/test_action.py | 7 +++--- .../fillers/test_classifier_filler.py | 6 ++--- .../fillers/test_first_meeting.py | 6 ++--- .../fillers/test_geo_token_filler.py | 4 ++-- .../fillers/test_intersection.py | 6 ++--- .../fillers/test_org_token_filler.py | 4 ++-- .../fillers/test_regexps_filler.py | 4 ++-- .../_trial_temp/_trial_marker | 0 .../scenarios_test/_trial_temp/_trial_marker | 0 .../scenarios_test/test_tree_scenario.py | 6 ++--- .../user_models/_trial_temp/_trial_marker | 0 .../action/_trial_temp/_trial_marker | 0 .../handlers/_trial_temp/_trial_marker | 0 .../system_answers/_trial_temp/_trial_marker | 0 27 files changed, 81 insertions(+), 82 deletions(-) create mode 100755 tests/scenarios_tests/actions_test/_trial_temp/_trial_marker create mode 100755 tests/scenarios_tests/requirements_test/_trial_temp/_trial_marker create mode 100755 tests/scenarios_tests/scenarios_test/_trial_temp/_trial_marker create mode 100755 tests/scenarios_tests/user_models/_trial_temp/_trial_marker create mode 100755 tests/smart_kit_tests/action/_trial_temp/_trial_marker create mode 100755 tests/smart_kit_tests/handlers/_trial_temp/_trial_marker create mode 100755 tests/smart_kit_tests/system_answers/_trial_temp/_trial_marker diff --git a/scenarios/actions/action.py b/scenarios/actions/action.py index c9cb1ddf..f1d65303 100644 --- a/scenarios/actions/action.py +++ b/scenarios/actions/action.py @@ -309,9 +309,7 @@ async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessing else: params.update(user.parametrizer.collect(text_preprocessing_result)) scenario_id = self.scenario.render(params) - scenario = user.descriptions - scenario = scenario["scenarios"] - scenario = scenario.get(scenario_id) + scenario = user.descriptions["scenarios"].get(scenario_id) if scenario: return scenario.run(text_preprocessing_result, user, params) diff --git a/smart_kit/template/app/adapters/db_adapters.py-tpl b/smart_kit/template/app/adapters/db_adapters.py-tpl index d7828974..92b18a44 100644 --- a/smart_kit/template/app/adapters/db_adapters.py-tpl +++ b/smart_kit/template/app/adapters/db_adapters.py-tpl @@ -9,32 +9,32 @@ class CustomDBAdapter(DBAdapter): пометьте их raise error.NotSupportedOperation """ - def __init__(self, config): - super(CustomDBAdapter, self).__init__(config) + def __init__(cls, config): + super(CustomDBAdapter, cls).__init__(config) - def connect(self): + def connect(cls): pass - def _open(self, filename, *args, **kwargs): + def _open(cls, filename, *args, **kwargs): pass - def _save(self, id, data): + def _save(cls, id, data): pass - def _replace_if_equals(self, id, sample, data): + def _replace_if_equals(cls, id, sample, data): pass - def _get(self, id): + def _get(cls, id): pass - def _list_dir(self, path): + def _list_dir(cls, path): raise error.NotSupportedOperation - def _glob(self, path, pattern): + def _glob(cls, path, pattern): raise error.NotSupportedOperation - def _path_exists(self, path): + def _path_exists(cls, path): pass - def _on_prepare(self): + def _on_prepare(cls): pass diff --git a/smart_kit/template/app/basic_entities/actions.py-tpl b/smart_kit/template/app/basic_entities/actions.py-tpl index 9a67afad..a0307342 100644 --- a/smart_kit/template/app/basic_entities/actions.py-tpl +++ b/smart_kit/template/app/basic_entities/actions.py-tpl @@ -13,12 +13,12 @@ class CustomAction(Action): Тут можно создать собственные Actions для использования их в сценариях """ - def __init__(self, items: Dict[str, Any], id: Optional[str] = None): - super(CustomAction, self).__init__(items, id) + def __init__(cls, items: Dict[str, Any], id: Optional[str] = None): + super(CustomAction, cls).__init__(items, id) items = items or {} - self.test_param = items.get("test_param") + cls.test_param = items.get("test_param") - async def run(self, user: User, text_preprocessing_result: TextPreprocessingResult, + async def run(cls, user: User, text_preprocessing_result: TextPreprocessingResult, params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: print("Test Action") return None diff --git a/smart_kit/template/app/basic_entities/fillers.py-tpl b/smart_kit/template/app/basic_entities/fillers.py-tpl index 05234d32..26bbd3fa 100644 --- a/smart_kit/template/app/basic_entities/fillers.py-tpl +++ b/smart_kit/template/app/basic_entities/fillers.py-tpl @@ -14,11 +14,11 @@ class CustomFieldFiller(FieldFillerDescription): Тут можно создать собственные Fillers для использования их в заполнении полей форм и """ - def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> None: - super(SampleFieldFiller, self).__init__(items, id) + def __init__(cls, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> None: + super(SampleFieldFiller, cls).__init__(items, id) items = items or {} - self.test_item = items.get("test_item") + cls.test_item = items.get("test_item") @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, params) -> Optional[str]: + def extract(cls, text_preprocessing_result: TextPreprocessingResult, user: User, params) -> Optional[str]: return None diff --git a/smart_kit/template/app/basic_entities/requirements.py-tpl b/smart_kit/template/app/basic_entities/requirements.py-tpl index 153743f7..2acec3c0 100644 --- a/smart_kit/template/app/basic_entities/requirements.py-tpl +++ b/smart_kit/template/app/basic_entities/requirements.py-tpl @@ -13,11 +13,11 @@ class CustomRequirement(Requirement): Тут можно создать собственные Requirements для использования их в сценариях """ - def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: - super(CustomRequirement, self).__init__(items, id) + def __init__(cls, items: Dict[str, Any], id: Optional[str] = None) -> None: + super(CustomRequirement, cls).__init__(items, id) items = items or {} - self.test_param = items.get("test_param") + cls.test_param = items.get("test_param") - async def check(self, text_preprocessing_result: TextPreprocessingResult, + async def check(cls, text_preprocessing_result: TextPreprocessingResult, user: User, params: Dict[str, Any] = None) -> bool: return False diff --git a/smart_kit/template/app/handlers/handlers.py-tpl b/smart_kit/template/app/handlers/handlers.py-tpl index e5d4a6f3..3821939e 100644 --- a/smart_kit/template/app/handlers/handlers.py-tpl +++ b/smart_kit/template/app/handlers/handlers.py-tpl @@ -8,5 +8,5 @@ class CustomHandler(HandlerBase): Тут создаются Handlers, которые используются для запуска логики в зависимости от типа входяшего сообщения """ - def run(self, payload, user): + def run(cls, payload, user): return [] diff --git a/smart_kit/template/app/local_testing/custom_local_testing.py-tpl b/smart_kit/template/app/local_testing/custom_local_testing.py-tpl index ca4f2993..f498af88 100644 --- a/smart_kit/template/app/local_testing/custom_local_testing.py-tpl +++ b/smart_kit/template/app/local_testing/custom_local_testing.py-tpl @@ -4,18 +4,18 @@ from smart_kit.testing.local import CLInterface class CustomLocalTesting(CLInterface): """ Тут модифицируется local_testing для эмулирования ответов от внешних систем - формат функции - on_(self, message): + формат функции - on_(cls, message): где - это тип сообщения,написанный в нижнем регистре например: - def on_back_get_token_request(self, message): + def on_back_get_token_request(cls, message): # BACK_GET_TOKEN_REQUEST - тип сообщения return json.dumps({ - "messageId": self.environment.message_id, + "messageId": cls.environment.message_id, "messageName": "BACK_GET_TOKEN_RESPONSE", "uuid": { - "userChannel": self.environment.user_channel, - "userId": self.environment.user_id, - "chatId": self.environment.chat_id + "userChannel": cls.environment.user_channel, + "userId": cls.environment.user_id, + "chatId": cls.environment.chat_id }, "payload": { "token": "test", diff --git a/smart_kit/template/app/models/dialogue_manager.py-tpl b/smart_kit/template/app/models/dialogue_manager.py-tpl index 73864d6d..692bbb56 100644 --- a/smart_kit/template/app/models/dialogue_manager.py-tpl +++ b/smart_kit/template/app/models/dialogue_manager.py-tpl @@ -7,5 +7,5 @@ class CustomDialogueManager(DialogueManager): В собственном DialogManager можно переопределить логику, связанную с вызовом сценариев по входяшему интенту """ - def __init__(self, scenario_descriptions, app_name, **kwargs): - super(CustomDialogueManager, self).__init__(scenario_descriptions, app_name, **kwargs) \ No newline at end of file + def __init__(cls, scenario_descriptions, app_name, **kwargs): + super(CustomDialogueManager, cls).__init__(scenario_descriptions, app_name, **kwargs) \ No newline at end of file diff --git a/smart_kit/template/app/models/model.py-tpl b/smart_kit/template/app/models/model.py-tpl index 3d7d03c2..78e22f52 100644 --- a/smart_kit/template/app/models/model.py-tpl +++ b/smart_kit/template/app/models/model.py-tpl @@ -7,7 +7,7 @@ class CustomModel(SmartAppModel): В собственной Model можно добавить новые хендлеры для обработки входящих сообщений """ - def __init__(self, resources, dialogue_manager_cls, custom_settings, **kwargs): - super(CustomModel, self).__init__(resources, dialogue_manager_cls, custom_settings, **kwargs) - self._handlers.update({}) + def __init__(cls, resources, dialogue_manager_cls, custom_settings, **kwargs): + super(CustomModel, cls).__init__(resources, dialogue_manager_cls, custom_settings, **kwargs) + cls._handlers.update({}) diff --git a/smart_kit/template/app/resources/custom_app_resourses.py-tpl b/smart_kit/template/app/resources/custom_app_resourses.py-tpl index e1791c8e..f9d3d67e 100644 --- a/smart_kit/template/app/resources/custom_app_resourses.py-tpl +++ b/smart_kit/template/app/resources/custom_app_resourses.py-tpl @@ -13,10 +13,10 @@ from app.basic_entities.requirements import CustomRequirement class CustomAppResourses(SmartAppResources): - def __init__(self, source, references_path, settings): - super(CustomAppResourses, self).__init__(source, references_path, settings) + def __init__(cls, source, references_path, settings): + super(CustomAppResourses, cls).__init__(source, references_path, settings) - def override_repositories(self, repositories: list): + def override_repositories(cls, repositories: list): """ Метод предназначен для переопределения репозиториев в дочерних классах. :param repositories: Список репозиториев родителя @@ -24,19 +24,19 @@ class CustomAppResourses(SmartAppResources): """ return repositories - def init_field_filler_description(self): - super(CustomAppResourses, self).init_field_filler_description() + def init_field_filler_description(cls): + super(CustomAppResourses, cls).init_field_filler_description() ffd.field_filler_description["simple_filler"] = CustomFieldFiller - def init_actions(self): - super(CustomAppResourses, self).init_actions() + def init_actions(cls): + super(CustomAppResourses, cls).init_actions() actions["custom_action"] = CustomAction - def init_requirements(self): - super(CustomAppResourses, self).init_requirements() + def init_requirements(cls): + super(CustomAppResourses, cls).init_requirements() requirements["sample_requirement"] = CustomRequirement - def init_db_adapters(self): - super(CustomAppResourses, self).init_db_adapters() + def init_db_adapters(cls): + super(CustomAppResourses, cls).init_db_adapters() db_adapters["custom_db_adapter"] = CustomDBAdapter diff --git a/smart_kit/template/app/user/parametrizer.py-tpl b/smart_kit/template/app/user/parametrizer.py-tpl index 9fe54a6f..ca774e25 100644 --- a/smart_kit/template/app/user/parametrizer.py-tpl +++ b/smart_kit/template/app/user/parametrizer.py-tpl @@ -7,10 +7,10 @@ class CustomParametrizer(Parametrizer): Тут можно добавить новые данные, которые будут доступны для использования в ASL при использовании jinja """ - def __init__(self, user, items): - super(CustomParametrizer, self).__init__(user, items) + def __init__(cls, user, items): + super(CustomParametrizer, cls).__init__(user, items) - def _get_user_data(self, text_preprocessing_result=None): - data = super(CustomParametrizer, self)._get_user_data(text_preprocessing_result) + def _get_user_data(cls, text_preprocessing_result=None): + data = super(CustomParametrizer, cls)._get_user_data(text_preprocessing_result) data.update({}) return data diff --git a/smart_kit/template/app/user/user.py-tpl b/smart_kit/template/app/user/user.py-tpl index e0520de5..d3f81630 100644 --- a/smart_kit/template/app/user/user.py-tpl +++ b/smart_kit/template/app/user/user.py-tpl @@ -12,17 +12,17 @@ class CustomeUser(User): в метод fields можно добавляются собственные поля, для использования внутри базовых сущностей """ - def __init__(self, id, message, db_data, settings, descriptions, parametrizer_cls, load_error=False): - super(CustomeUser, self).__init__(id, message, db_data, settings, + def __init__(cls, id, message, db_data, settings, descriptions, parametrizer_cls, load_error=False): + super(CustomeUser, cls).__init__(id, message, db_data, settings, descriptions, parametrizer_cls, load_error) @property - def fields(self): - return super(CustomeUser, self).fields + [] + def fields(cls): + return super(CustomeUser, cls).fields + [] @lazy - def parametrizer(self): - return CustomParametrizer(self, {}) + def parametrizer(cls): + return CustomParametrizer(cls, {}) - def expire(self): - super(CustomeUser, self).expire() + def expire(cls): + super(CustomeUser, cls).expire() diff --git a/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker b/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/scenarios_tests/actions_test/test_action.py b/tests/scenarios_tests/actions_test/test_action.py index b8d21757..7d948aec 100644 --- a/tests/scenarios_tests/actions_test/test_action.py +++ b/tests/scenarios_tests/actions_test/test_action.py @@ -1,6 +1,6 @@ import unittest from typing import Dict, Any, Union, Optional -from unittest.mock import MagicMock, Mock, ANY +from unittest.mock import MagicMock, Mock, ANY, AsyncMock from core.basic_models.actions.basic_actions import Action, action_factory, actions from core.model.registered import registered_factories @@ -420,7 +420,7 @@ class ChoiceScenarioActionTest(unittest.IsolatedAsyncioTestCase): @staticmethod async def mock_and_perform_action(test_items: Dict[str, Any], expected_result: Optional[str] = None, - expected_scen: Optional[str] = None) -> Union[str, None]: + expected_scen: Optional[str] = None) -> Union[str, None]: action = ChoiceScenarioAction(test_items) user = Mock() user.parametrizer = MockParametrizer(user, {}) @@ -451,7 +451,8 @@ async def test_choice_scenario_action(self): } expected_scen_result = "test_N_done" real_scen_result = await self.mock_and_perform_action( - test_items, expected_result=expected_scen_result, expected_scen="test_N") + test_items, expected_result=expected_scen_result, expected_scen="test_N" + ) self.assertEqual(real_scen_result, expected_scen_result) async def test_choice_scenario_action_no_else_action(self): diff --git a/tests/scenarios_tests/fillers/test_classifier_filler.py b/tests/scenarios_tests/fillers/test_classifier_filler.py index f1066596..521e91f0 100644 --- a/tests/scenarios_tests/fillers/test_classifier_filler.py +++ b/tests/scenarios_tests/fillers/test_classifier_filler.py @@ -1,11 +1,11 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock, patch from core.basic_models.classifiers.basic_classifiers import ExternalClassifier from scenarios.scenario_models.field.field_filler_description import ClassifierFiller, ClassifierFillerMeta -class TestClassifierFiller(TestCase): +class TestClassifierFiller(IsolatedAsyncioTestCase): def setUp(self): test_items = { @@ -38,7 +38,7 @@ async def test_filler_extract_if_no_model_answer(self, mock_classifier_model): self.assertIsNone(actual_res) -class TestClassifierFillerMeta(TestCase): +class TestClassifierFillerMeta(IsolatedAsyncioTestCase): def setUp(self): test_items = { diff --git a/tests/scenarios_tests/fillers/test_first_meeting.py b/tests/scenarios_tests/fillers/test_first_meeting.py index 98e89c52..09766c03 100644 --- a/tests/scenarios_tests/fillers/test_first_meeting.py +++ b/tests/scenarios_tests/fillers/test_first_meeting.py @@ -1,10 +1,10 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from scenarios.scenario_models.field.field_filler_description import FirstNumberFiller, \ FirstCurrencyFiller -class TestFirstNumberFiller(TestCase): +class TestFirstNumberFiller(IsolatedAsyncioTestCase): async def test_1(self): expected = "5" items = {} @@ -27,7 +27,7 @@ async def test_2(self): self.assertIsNone(result) -class TestFirstCurrencyFiller(TestCase): +class TestFirstCurrencyFiller(IsolatedAsyncioTestCase): async def test_1(self): expected = "ru" items = {} diff --git a/tests/scenarios_tests/fillers/test_geo_token_filler.py b/tests/scenarios_tests/fillers/test_geo_token_filler.py index 4f81e378..f5964838 100644 --- a/tests/scenarios_tests/fillers/test_geo_token_filler.py +++ b/tests/scenarios_tests/fillers/test_geo_token_filler.py @@ -1,10 +1,10 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from scenarios.scenario_models.field.field_filler_description import FirstGeoFiller -class TestFirstGeoFiller(TestCase): +class TestFirstGeoFiller(IsolatedAsyncioTestCase): def setUp(self): items = {} self.filler = FirstGeoFiller(items) diff --git a/tests/scenarios_tests/fillers/test_intersection.py b/tests/scenarios_tests/fillers/test_intersection.py index 1ff84222..c6647542 100644 --- a/tests/scenarios_tests/fillers/test_intersection.py +++ b/tests/scenarios_tests/fillers/test_intersection.py @@ -1,5 +1,5 @@ import os -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock, patch import smart_kit @@ -19,7 +19,7 @@ def patch_get_app_config(mock_get_app_config): mock_get_app_config.return_value = result -class TestIntersectionFieldFiller(TestCase): +class TestIntersectionFieldFiller(IsolatedAsyncioTestCase): @patch('smart_kit.configs.get_app_config') async def test_1(self, mock_get_app_config): @@ -147,7 +147,7 @@ async def test_5(self, mock_get_app_config): self.assertEqual(expected, result) -class TestIntersectionOriginalTextFiller(TestCase): +class TestIntersectionOriginalTextFiller(IsolatedAsyncioTestCase): @patch('smart_kit.configs.get_app_config') async def test_1(self, mock_get_app_config): patch_get_app_config(mock_get_app_config) diff --git a/tests/scenarios_tests/fillers/test_org_token_filler.py b/tests/scenarios_tests/fillers/test_org_token_filler.py index 27690030..2bc3aa08 100644 --- a/tests/scenarios_tests/fillers/test_org_token_filler.py +++ b/tests/scenarios_tests/fillers/test_org_token_filler.py @@ -1,10 +1,10 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from scenarios.scenario_models.field.field_filler_description import FirstOrgFiller -class TestFirstOrgFiller(TestCase): +class TestFirstOrgFiller(IsolatedAsyncioTestCase): def setUp(self): items = {} self.filler = FirstOrgFiller(items) diff --git a/tests/scenarios_tests/fillers/test_regexps_filler.py b/tests/scenarios_tests/fillers/test_regexps_filler.py index 40b75b64..98f59fd9 100644 --- a/tests/scenarios_tests/fillers/test_regexps_filler.py +++ b/tests/scenarios_tests/fillers/test_regexps_filler.py @@ -1,10 +1,10 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from scenarios.scenario_models.field.field_filler_description import AllRegexpsFieldFiller -class Test_regexps_filler(TestCase): +class Test_regexps_filler(IsolatedAsyncioTestCase): @classmethod def setUpClass(cls): cls.items = {} diff --git a/tests/scenarios_tests/requirements_test/_trial_temp/_trial_marker b/tests/scenarios_tests/requirements_test/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/scenarios_tests/scenarios_test/_trial_temp/_trial_marker b/tests/scenarios_tests/scenarios_test/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py index 7b107762..9b713aeb 100644 --- a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py +++ b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py @@ -1,4 +1,4 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock, MagicMock, AsyncMock from core.basic_models.actions.command import Command @@ -27,7 +27,7 @@ async def run(self, user, text_preprocessing_result, params): return [] -class TestTreeScenario(TestCase): +class TestTreeScenario(IsolatedAsyncioTestCase): async def test_1(self): """ @@ -88,7 +88,7 @@ async def test_1(self): self.assertIsNone(current_node_mock.current_node) context_forms.new.assert_called_once_with(form_type) - async def test_breake(self): + async def test_break(self): """ Тест проверяет выход из сценария если сработает флаг break_scenario """ diff --git a/tests/scenarios_tests/user_models/_trial_temp/_trial_marker b/tests/scenarios_tests/user_models/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/smart_kit_tests/action/_trial_temp/_trial_marker b/tests/smart_kit_tests/action/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/smart_kit_tests/handlers/_trial_temp/_trial_marker b/tests/smart_kit_tests/handlers/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/smart_kit_tests/system_answers/_trial_temp/_trial_marker b/tests/smart_kit_tests/system_answers/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b From 16ef0023e77b90fcfbbfa314abb117b3e8554a2b Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Thu, 11 Nov 2021 11:00:15 +0300 Subject: [PATCH 059/116] fix --- .../template/app/adapters/db_adapters.py-tpl | 22 +++++++++--------- .../app/basic_entities/actions.py-tpl | 8 +++---- .../app/basic_entities/fillers.py-tpl | 8 +++---- .../app/basic_entities/requirements.py-tpl | 8 +++---- .../template/app/handlers/handlers.py-tpl | 2 +- .../local_testing/custom_local_testing.py-tpl | 12 +++++----- .../app/models/dialogue_manager.py-tpl | 4 ++-- smart_kit/template/app/models/model.py-tpl | 7 +++--- .../app/resources/custom_app_resourses.py-tpl | 23 +++++++++---------- .../template/app/user/parametrizer.py-tpl | 8 +++---- smart_kit/template/app/user/user.py-tpl | 16 ++++++------- 11 files changed, 58 insertions(+), 60 deletions(-) diff --git a/smart_kit/template/app/adapters/db_adapters.py-tpl b/smart_kit/template/app/adapters/db_adapters.py-tpl index 92b18a44..d7828974 100644 --- a/smart_kit/template/app/adapters/db_adapters.py-tpl +++ b/smart_kit/template/app/adapters/db_adapters.py-tpl @@ -9,32 +9,32 @@ class CustomDBAdapter(DBAdapter): пометьте их raise error.NotSupportedOperation """ - def __init__(cls, config): - super(CustomDBAdapter, cls).__init__(config) + def __init__(self, config): + super(CustomDBAdapter, self).__init__(config) - def connect(cls): + def connect(self): pass - def _open(cls, filename, *args, **kwargs): + def _open(self, filename, *args, **kwargs): pass - def _save(cls, id, data): + def _save(self, id, data): pass - def _replace_if_equals(cls, id, sample, data): + def _replace_if_equals(self, id, sample, data): pass - def _get(cls, id): + def _get(self, id): pass - def _list_dir(cls, path): + def _list_dir(self, path): raise error.NotSupportedOperation - def _glob(cls, path, pattern): + def _glob(self, path, pattern): raise error.NotSupportedOperation - def _path_exists(cls, path): + def _path_exists(self, path): pass - def _on_prepare(cls): + def _on_prepare(self): pass diff --git a/smart_kit/template/app/basic_entities/actions.py-tpl b/smart_kit/template/app/basic_entities/actions.py-tpl index a0307342..9a67afad 100644 --- a/smart_kit/template/app/basic_entities/actions.py-tpl +++ b/smart_kit/template/app/basic_entities/actions.py-tpl @@ -13,12 +13,12 @@ class CustomAction(Action): Тут можно создать собственные Actions для использования их в сценариях """ - def __init__(cls, items: Dict[str, Any], id: Optional[str] = None): - super(CustomAction, cls).__init__(items, id) + def __init__(self, items: Dict[str, Any], id: Optional[str] = None): + super(CustomAction, self).__init__(items, id) items = items or {} - cls.test_param = items.get("test_param") + self.test_param = items.get("test_param") - async def run(cls, user: User, text_preprocessing_result: TextPreprocessingResult, + async def run(self, user: User, text_preprocessing_result: TextPreprocessingResult, params: Optional[Dict[str, Union[str, float, int]]] = None) -> None: print("Test Action") return None diff --git a/smart_kit/template/app/basic_entities/fillers.py-tpl b/smart_kit/template/app/basic_entities/fillers.py-tpl index 26bbd3fa..05234d32 100644 --- a/smart_kit/template/app/basic_entities/fillers.py-tpl +++ b/smart_kit/template/app/basic_entities/fillers.py-tpl @@ -14,11 +14,11 @@ class CustomFieldFiller(FieldFillerDescription): Тут можно создать собственные Fillers для использования их в заполнении полей форм и """ - def __init__(cls, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> None: - super(SampleFieldFiller, cls).__init__(items, id) + def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> None: + super(SampleFieldFiller, self).__init__(items, id) items = items or {} - cls.test_item = items.get("test_item") + self.test_item = items.get("test_item") @exc_handler(on_error_obj_method_name="on_extract_error") - def extract(cls, text_preprocessing_result: TextPreprocessingResult, user: User, params) -> Optional[str]: + def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, params) -> Optional[str]: return None diff --git a/smart_kit/template/app/basic_entities/requirements.py-tpl b/smart_kit/template/app/basic_entities/requirements.py-tpl index 2acec3c0..153743f7 100644 --- a/smart_kit/template/app/basic_entities/requirements.py-tpl +++ b/smart_kit/template/app/basic_entities/requirements.py-tpl @@ -13,11 +13,11 @@ class CustomRequirement(Requirement): Тут можно создать собственные Requirements для использования их в сценариях """ - def __init__(cls, items: Dict[str, Any], id: Optional[str] = None) -> None: - super(CustomRequirement, cls).__init__(items, id) + def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None: + super(CustomRequirement, self).__init__(items, id) items = items or {} - cls.test_param = items.get("test_param") + self.test_param = items.get("test_param") - async def check(cls, text_preprocessing_result: TextPreprocessingResult, + async def check(self, text_preprocessing_result: TextPreprocessingResult, user: User, params: Dict[str, Any] = None) -> bool: return False diff --git a/smart_kit/template/app/handlers/handlers.py-tpl b/smart_kit/template/app/handlers/handlers.py-tpl index 3821939e..e5d4a6f3 100644 --- a/smart_kit/template/app/handlers/handlers.py-tpl +++ b/smart_kit/template/app/handlers/handlers.py-tpl @@ -8,5 +8,5 @@ class CustomHandler(HandlerBase): Тут создаются Handlers, которые используются для запуска логики в зависимости от типа входяшего сообщения """ - def run(cls, payload, user): + def run(self, payload, user): return [] diff --git a/smart_kit/template/app/local_testing/custom_local_testing.py-tpl b/smart_kit/template/app/local_testing/custom_local_testing.py-tpl index f498af88..ca4f2993 100644 --- a/smart_kit/template/app/local_testing/custom_local_testing.py-tpl +++ b/smart_kit/template/app/local_testing/custom_local_testing.py-tpl @@ -4,18 +4,18 @@ from smart_kit.testing.local import CLInterface class CustomLocalTesting(CLInterface): """ Тут модифицируется local_testing для эмулирования ответов от внешних систем - формат функции - on_(cls, message): + формат функции - on_(self, message): где - это тип сообщения,написанный в нижнем регистре например: - def on_back_get_token_request(cls, message): + def on_back_get_token_request(self, message): # BACK_GET_TOKEN_REQUEST - тип сообщения return json.dumps({ - "messageId": cls.environment.message_id, + "messageId": self.environment.message_id, "messageName": "BACK_GET_TOKEN_RESPONSE", "uuid": { - "userChannel": cls.environment.user_channel, - "userId": cls.environment.user_id, - "chatId": cls.environment.chat_id + "userChannel": self.environment.user_channel, + "userId": self.environment.user_id, + "chatId": self.environment.chat_id }, "payload": { "token": "test", diff --git a/smart_kit/template/app/models/dialogue_manager.py-tpl b/smart_kit/template/app/models/dialogue_manager.py-tpl index 692bbb56..cc68f113 100644 --- a/smart_kit/template/app/models/dialogue_manager.py-tpl +++ b/smart_kit/template/app/models/dialogue_manager.py-tpl @@ -7,5 +7,5 @@ class CustomDialogueManager(DialogueManager): В собственном DialogManager можно переопределить логику, связанную с вызовом сценариев по входяшему интенту """ - def __init__(cls, scenario_descriptions, app_name, **kwargs): - super(CustomDialogueManager, cls).__init__(scenario_descriptions, app_name, **kwargs) \ No newline at end of file + def __init__(self, scenario_descriptions, app_name, **kwargs): + super(CustomDialogueManager, self).__init__(scenario_descriptions, app_name, **kwargs) diff --git a/smart_kit/template/app/models/model.py-tpl b/smart_kit/template/app/models/model.py-tpl index 78e22f52..3b9fdea8 100644 --- a/smart_kit/template/app/models/model.py-tpl +++ b/smart_kit/template/app/models/model.py-tpl @@ -7,7 +7,6 @@ class CustomModel(SmartAppModel): В собственной Model можно добавить новые хендлеры для обработки входящих сообщений """ - def __init__(cls, resources, dialogue_manager_cls, custom_settings, **kwargs): - super(CustomModel, cls).__init__(resources, dialogue_manager_cls, custom_settings, **kwargs) - cls._handlers.update({}) - + def __init__(self, resources, dialogue_manager_cls, custom_settings, **kwargs): + super(CustomModel, self).__init__(resources, dialogue_manager_cls, custom_settings, **kwargs) + self._handlers.update({}) diff --git a/smart_kit/template/app/resources/custom_app_resourses.py-tpl b/smart_kit/template/app/resources/custom_app_resourses.py-tpl index f9d3d67e..90258c67 100644 --- a/smart_kit/template/app/resources/custom_app_resourses.py-tpl +++ b/smart_kit/template/app/resources/custom_app_resourses.py-tpl @@ -13,10 +13,10 @@ from app.basic_entities.requirements import CustomRequirement class CustomAppResourses(SmartAppResources): - def __init__(cls, source, references_path, settings): - super(CustomAppResourses, cls).__init__(source, references_path, settings) + def __init__(self, source, references_path, settings): + super(CustomAppResourses, self).__init__(source, references_path, settings) - def override_repositories(cls, repositories: list): + def override_repositories(self, repositories: list): """ Метод предназначен для переопределения репозиториев в дочерних классах. :param repositories: Список репозиториев родителя @@ -24,19 +24,18 @@ class CustomAppResourses(SmartAppResources): """ return repositories - def init_field_filler_description(cls): - super(CustomAppResourses, cls).init_field_filler_description() + def init_field_filler_description(self): + super(CustomAppResourses, self).init_field_filler_description() ffd.field_filler_description["simple_filler"] = CustomFieldFiller - def init_actions(cls): - super(CustomAppResourses, cls).init_actions() + def init_actions(self): + super(CustomAppResourses, self).init_actions() actions["custom_action"] = CustomAction - def init_requirements(cls): - super(CustomAppResourses, cls).init_requirements() + def init_requirements(self): + super(CustomAppResourses, self).init_requirements() requirements["sample_requirement"] = CustomRequirement - def init_db_adapters(cls): - super(CustomAppResourses, cls).init_db_adapters() + def init_db_adapters(self): + super(CustomAppResourses, self).init_db_adapters() db_adapters["custom_db_adapter"] = CustomDBAdapter - diff --git a/smart_kit/template/app/user/parametrizer.py-tpl b/smart_kit/template/app/user/parametrizer.py-tpl index ca774e25..9fe54a6f 100644 --- a/smart_kit/template/app/user/parametrizer.py-tpl +++ b/smart_kit/template/app/user/parametrizer.py-tpl @@ -7,10 +7,10 @@ class CustomParametrizer(Parametrizer): Тут можно добавить новые данные, которые будут доступны для использования в ASL при использовании jinja """ - def __init__(cls, user, items): - super(CustomParametrizer, cls).__init__(user, items) + def __init__(self, user, items): + super(CustomParametrizer, self).__init__(user, items) - def _get_user_data(cls, text_preprocessing_result=None): - data = super(CustomParametrizer, cls)._get_user_data(text_preprocessing_result) + def _get_user_data(self, text_preprocessing_result=None): + data = super(CustomParametrizer, self)._get_user_data(text_preprocessing_result) data.update({}) return data diff --git a/smart_kit/template/app/user/user.py-tpl b/smart_kit/template/app/user/user.py-tpl index d3f81630..e0520de5 100644 --- a/smart_kit/template/app/user/user.py-tpl +++ b/smart_kit/template/app/user/user.py-tpl @@ -12,17 +12,17 @@ class CustomeUser(User): в метод fields можно добавляются собственные поля, для использования внутри базовых сущностей """ - def __init__(cls, id, message, db_data, settings, descriptions, parametrizer_cls, load_error=False): - super(CustomeUser, cls).__init__(id, message, db_data, settings, + def __init__(self, id, message, db_data, settings, descriptions, parametrizer_cls, load_error=False): + super(CustomeUser, self).__init__(id, message, db_data, settings, descriptions, parametrizer_cls, load_error) @property - def fields(cls): - return super(CustomeUser, cls).fields + [] + def fields(self): + return super(CustomeUser, self).fields + [] @lazy - def parametrizer(cls): - return CustomParametrizer(cls, {}) + def parametrizer(self): + return CustomParametrizer(self, {}) - def expire(cls): - super(CustomeUser, cls).expire() + def expire(self): + super(CustomeUser, self).expire() From 8824b82fd0b758f2435e002e440c15e93d200556 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Thu, 11 Nov 2021 11:10:48 +0300 Subject: [PATCH 060/116] change python-version in github/workflows/python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e37ff5e0..92bbb25f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.9] + python-version: [3.8, 3.9] steps: - uses: actions/checkout@v2 From 569bd17ce9b1abcbc89842d437ec5cc785d71d2c Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Fri, 12 Nov 2021 03:52:17 +0300 Subject: [PATCH 061/116] fix tests --- .../actions_test/_trial_temp-1/_trial_marker | 0 tests/scenarios_tests/actions_test/_trial_temp.lock | 1 + tests/scenarios_tests/actions_test/test_action.py | 6 +++--- tests/scenarios_tests/fillers/test_approve.py | 6 +++--- tests/scenarios_tests/fillers/test_available_info_filler.py | 4 ++-- tests/scenarios_tests/fillers/test_external_filler.py | 4 ++-- tests/scenarios_tests/fillers/test_person_filler.py | 4 ++-- .../fillers/test_previous_messages_filler.py | 4 ++-- .../fillers/test_regexp_and_string_operations_filler.py | 4 ++-- tests/scenarios_tests/fillers/test_regexp_filler.py | 4 ++-- 10 files changed, 19 insertions(+), 18 deletions(-) create mode 100755 tests/scenarios_tests/actions_test/_trial_temp-1/_trial_marker create mode 120000 tests/scenarios_tests/actions_test/_trial_temp.lock diff --git a/tests/scenarios_tests/actions_test/_trial_temp-1/_trial_marker b/tests/scenarios_tests/actions_test/_trial_temp-1/_trial_marker new file mode 100755 index 00000000..e69de29b diff --git a/tests/scenarios_tests/actions_test/_trial_temp.lock b/tests/scenarios_tests/actions_test/_trial_temp.lock new file mode 120000 index 00000000..8b337943 --- /dev/null +++ b/tests/scenarios_tests/actions_test/_trial_temp.lock @@ -0,0 +1 @@ +59039 \ No newline at end of file diff --git a/tests/scenarios_tests/actions_test/test_action.py b/tests/scenarios_tests/actions_test/test_action.py index 7d948aec..89481a63 100644 --- a/tests/scenarios_tests/actions_test/test_action.py +++ b/tests/scenarios_tests/actions_test/test_action.py @@ -1,6 +1,6 @@ import unittest from typing import Dict, Any, Union, Optional -from unittest.mock import MagicMock, Mock, ANY, AsyncMock +from unittest.mock import MagicMock, Mock, ANY from core.basic_models.actions.basic_actions import Action, action_factory, actions from core.model.registered import registered_factories @@ -128,7 +128,7 @@ async def test_run(self): class SaveBehaviorActionTest(unittest.IsolatedAsyncioTestCase): @classmethod - def setUpClass(cls): + def setUp(cls): user = Mock() user.message = Mock() user.parametrizer = MockParametrizer(user, {}) @@ -492,7 +492,7 @@ async def test_choice_scenario_action_with_else_action(self): self.assertEqual(real_scen_result, expected_scen_result) -class ClearCurrentScenarioActionTest(unittest.TestCase): +class ClearCurrentScenarioActionTest(unittest.IsolatedAsyncioTestCase): async def test_action(self): scenario_name = "test_scenario" diff --git a/tests/scenarios_tests/fillers/test_approve.py b/tests/scenarios_tests/fillers/test_approve.py index e0ddb412..5e4461d4 100644 --- a/tests/scenarios_tests/fillers/test_approve.py +++ b/tests/scenarios_tests/fillers/test_approve.py @@ -1,5 +1,5 @@ import os -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock, patch from core.text_preprocessing.preprocessing_result import TextPreprocessingResult @@ -20,7 +20,7 @@ def patch_get_app_config(mock_get_app_config): mock_get_app_config.return_value = result -class TestApproveFiller(TestCase): +class TestApproveFiller(IsolatedAsyncioTestCase): @patch('smart_kit.configs.get_app_config') async def test_1(self, mock_get_app_config): @@ -63,7 +63,7 @@ async def test_1(self, mock_get_app_config): self.assertIsNone(result) -class TestApproveRawTextFiller(TestCase): +class TestApproveRawTextFiller(IsolatedAsyncioTestCase): @patch('smart_kit.configs.get_app_config') async def test_1(self, mock_get_app_config): diff --git a/tests/scenarios_tests/fillers/test_available_info_filler.py b/tests/scenarios_tests/fillers/test_available_info_filler.py index b1fe8621..add4e265 100644 --- a/tests/scenarios_tests/fillers/test_available_info_filler.py +++ b/tests/scenarios_tests/fillers/test_available_info_filler.py @@ -1,4 +1,4 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from scenarios.scenario_models.field.field_filler_description import AvailableInfoFiller @@ -22,7 +22,7 @@ def collect(self, text_preprocessing_result=None, filter_params=None): return data -class TestAvailableInfoFiller(TestCase): +class TestAvailableInfoFiller(IsolatedAsyncioTestCase): @classmethod def setUpClass(cls): cls.address = "Address!" diff --git a/tests/scenarios_tests/fillers/test_external_filler.py b/tests/scenarios_tests/fillers/test_external_filler.py index a0894d4f..632154b5 100644 --- a/tests/scenarios_tests/fillers/test_external_filler.py +++ b/tests/scenarios_tests/fillers/test_external_filler.py @@ -1,10 +1,10 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock, AsyncMock from scenarios.scenario_models.field.field_filler_description import ExternalFieldFillerDescription -class TestExternalFieldFillerDescription(TestCase): +class TestExternalFieldFillerDescription(IsolatedAsyncioTestCase): async def test_1(self): expected = 5 diff --git a/tests/scenarios_tests/fillers/test_person_filler.py b/tests/scenarios_tests/fillers/test_person_filler.py index a35a91b7..48712643 100644 --- a/tests/scenarios_tests/fillers/test_person_filler.py +++ b/tests/scenarios_tests/fillers/test_person_filler.py @@ -1,10 +1,10 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from scenarios.scenario_models.field.field_filler_description import FirstPersonFiller -class TestFirstPersonFiller(TestCase): +class TestFirstPersonFiller(IsolatedAsyncioTestCase): def setUp(self): items = {} self.filler = FirstPersonFiller(items) diff --git a/tests/scenarios_tests/fillers/test_previous_messages_filler.py b/tests/scenarios_tests/fillers/test_previous_messages_filler.py index ba4dabd3..ac72cbeb 100644 --- a/tests/scenarios_tests/fillers/test_previous_messages_filler.py +++ b/tests/scenarios_tests/fillers/test_previous_messages_filler.py @@ -10,11 +10,11 @@ class MockFiller: def __init__(self, items=None): self.count = 0 - def extract(self, text_preprocessing_result, user, params): + async def extract(self, text_preprocessing_result, user, params): self.count += 1 -class PreviousMessagesFillerTest(unittest.TestCase): +class PreviousMessagesFillerTest(unittest.IsolatedAsyncioTestCase): async def test_fill_1(self): registered_factories[FieldFillerDescription] = field_filler_factory field_filler_description["mock_filler"] = MockFiller diff --git a/tests/scenarios_tests/fillers/test_regexp_and_string_operations_filler.py b/tests/scenarios_tests/fillers/test_regexp_and_string_operations_filler.py index 40e95fb8..43c16bb0 100644 --- a/tests/scenarios_tests/fillers/test_regexp_and_string_operations_filler.py +++ b/tests/scenarios_tests/fillers/test_regexp_and_string_operations_filler.py @@ -1,4 +1,4 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from scenarios.scenario_models.field.field_filler_description import RegexpAndStringOperationsFieldFiller @@ -8,7 +8,7 @@ def __reduce__(self): return (Mock, ()) -class TestRegexpStringOperationsFiller(TestCase): +class TestRegexpStringOperationsFiller(IsolatedAsyncioTestCase): def setUp(self): self.items = {"exp": "1-[0-9A-Z]{7}"} diff --git a/tests/scenarios_tests/fillers/test_regexp_filler.py b/tests/scenarios_tests/fillers/test_regexp_filler.py index e7c0e188..7996f9dc 100644 --- a/tests/scenarios_tests/fillers/test_regexp_filler.py +++ b/tests/scenarios_tests/fillers/test_regexp_filler.py @@ -1,9 +1,9 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from scenarios.scenario_models.field.field_filler_description import RegexpFieldFiller -class TestRegexpFiller(TestCase): +class TestRegexpFiller(IsolatedAsyncioTestCase): def setUp(self): self.items = {"exp": "1-[0-9A-Z]{7}"} self.user = Mock() From cbf76f3d63727e010e069533dacf0a6ee1b75976 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Fri, 12 Nov 2021 08:39:00 +0300 Subject: [PATCH 062/116] delete temp files --- tests/scenarios_tests/actions_test/_trial_temp-1/_trial_marker | 0 tests/scenarios_tests/actions_test/_trial_temp.lock | 1 - tests/scenarios_tests/actions_test/_trial_temp/_trial_marker | 0 tests/scenarios_tests/fillers/_trial_temp/_trial_marker | 0 .../scenarios_tests/requirements_test/_trial_temp/_trial_marker | 0 tests/scenarios_tests/scenarios_test/_trial_temp/_trial_marker | 0 tests/scenarios_tests/user_models/_trial_temp/_trial_marker | 0 tests/smart_kit_tests/action/_trial_temp/_trial_marker | 0 tests/smart_kit_tests/handlers/_trial_temp/_trial_marker | 0 tests/smart_kit_tests/system_answers/_trial_temp/_trial_marker | 0 10 files changed, 1 deletion(-) delete mode 100755 tests/scenarios_tests/actions_test/_trial_temp-1/_trial_marker delete mode 120000 tests/scenarios_tests/actions_test/_trial_temp.lock delete mode 100755 tests/scenarios_tests/actions_test/_trial_temp/_trial_marker delete mode 100755 tests/scenarios_tests/fillers/_trial_temp/_trial_marker delete mode 100755 tests/scenarios_tests/requirements_test/_trial_temp/_trial_marker delete mode 100755 tests/scenarios_tests/scenarios_test/_trial_temp/_trial_marker delete mode 100755 tests/scenarios_tests/user_models/_trial_temp/_trial_marker delete mode 100755 tests/smart_kit_tests/action/_trial_temp/_trial_marker delete mode 100755 tests/smart_kit_tests/handlers/_trial_temp/_trial_marker delete mode 100755 tests/smart_kit_tests/system_answers/_trial_temp/_trial_marker diff --git a/tests/scenarios_tests/actions_test/_trial_temp-1/_trial_marker b/tests/scenarios_tests/actions_test/_trial_temp-1/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/scenarios_tests/actions_test/_trial_temp.lock b/tests/scenarios_tests/actions_test/_trial_temp.lock deleted file mode 120000 index 8b337943..00000000 --- a/tests/scenarios_tests/actions_test/_trial_temp.lock +++ /dev/null @@ -1 +0,0 @@ -59039 \ No newline at end of file diff --git a/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker b/tests/scenarios_tests/actions_test/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/scenarios_tests/fillers/_trial_temp/_trial_marker b/tests/scenarios_tests/fillers/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/scenarios_tests/requirements_test/_trial_temp/_trial_marker b/tests/scenarios_tests/requirements_test/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/scenarios_tests/scenarios_test/_trial_temp/_trial_marker b/tests/scenarios_tests/scenarios_test/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/scenarios_tests/user_models/_trial_temp/_trial_marker b/tests/scenarios_tests/user_models/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/smart_kit_tests/action/_trial_temp/_trial_marker b/tests/smart_kit_tests/action/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/smart_kit_tests/handlers/_trial_temp/_trial_marker b/tests/smart_kit_tests/handlers/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/smart_kit_tests/system_answers/_trial_temp/_trial_marker b/tests/smart_kit_tests/system_answers/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 From 19d671a8a579b988b8383312d6268c977f69b003 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Sun, 14 Nov 2021 04:17:03 +0300 Subject: [PATCH 063/116] remove run_until_complete() where it is not necessary --- core/basic_models/scenarios/base_scenario.py | 6 +- scenarios/behaviors/behaviors.py | 7 +- .../form_filling_scenario.py | 31 ++- .../tree_scenario/tree_scenario.py | 12 +- .../last_scenarios_description.py | 9 +- smart_kit/models/dialogue_manager.py | 4 +- .../action_test/test_action.py | 192 +++++++----------- .../action_test/test_random_action.py | 15 +- .../_trial_temp/_trial_marker | 0 9 files changed, 114 insertions(+), 162 deletions(-) create mode 100755 tests/core_tests/requirements_test/_trial_temp/_trial_marker diff --git a/core/basic_models/scenarios/base_scenario.py b/core/basic_models/scenarios/base_scenario.py index bc1138d5..5e0965ce 100644 --- a/core/basic_models/scenarios/base_scenario.py +++ b/core/basic_models/scenarios/base_scenario.py @@ -1,5 +1,4 @@ # coding: utf-8 -import asyncio from typing import Dict, Any, List import core.logging.logger_constants as log_const @@ -47,10 +46,9 @@ def build_actions(self): def build_available_requirement(self): return self._available_requirement - def check_available(self, text_preprocessing_result, user): + async def check_available(self, text_preprocessing_result, user): if not self.switched_off: - loop = asyncio.get_event_loop() - return loop.run_until_complete(self.available_requirement.check(text_preprocessing_result, user)) + return await self.available_requirement.check(text_preprocessing_result, user) return False def _log_params(self): diff --git a/scenarios/behaviors/behaviors.py b/scenarios/behaviors/behaviors.py index 473ca2f9..d5b239c8 100644 --- a/scenarios/behaviors/behaviors.py +++ b/scenarios/behaviors/behaviors.py @@ -1,5 +1,4 @@ # coding: utf-8 -import asyncio from time import time from collections import namedtuple from typing import Dict @@ -176,7 +175,7 @@ async def timeout(self, callback_id: str): self._delete(callback_id) return result - def misstate(self, callback_id: str): + async def misstate(self, callback_id: str): log(f"behavior.misstate started: got callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s.", self._user, params={log_const.KEY_NAME: log_const.BEHAVIOR_MISSTATE_VALUE, @@ -191,9 +190,7 @@ def misstate(self, callback_id: str): smart_kit_metrics.counter_behavior_misstate, "misstate", callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - loop = asyncio.get_running_loop() - result = loop.run_until_complete( - behavior.misstate_action.run(self._user, text_preprocessing_result, callback_action_params)) + result = await behavior.misstate_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result diff --git a/scenarios/scenario_descriptions/form_filling_scenario.py b/scenarios/scenario_descriptions/form_filling_scenario.py index a8330d17..00808403 100644 --- a/scenarios/scenario_descriptions/form_filling_scenario.py +++ b/scenarios/scenario_descriptions/form_filling_scenario.py @@ -1,5 +1,4 @@ # coding: utf-8 -import asyncio from typing import Dict, Any from core.basic_models.scenarios.base_scenario import BaseScenario @@ -30,14 +29,14 @@ def _get_form(self, user): async def text_fits(self, text_preprocessing_result, user): return await self._check_field(text_preprocessing_result, user, None) - def check_ask_again_requests(self, text_preprocessing_result, user, params): + async def check_ask_again_requests(self, text_preprocessing_result, user, params): form = user.forms[self.form_type] - question_field = self._field(form, text_preprocessing_result, user, params) + question_field = await self._field(form, text_preprocessing_result, user, params) return question_field.ask_again_counter < len(question_field.description.ask_again_requests) - def ask_again(self, text_preprocessing_result, user, params): + async def ask_again(self, text_preprocessing_result, user, params): form = user.forms[self.form_type] - question_field = self._field(form, text_preprocessing_result, user, params) + question_field = await self._field(form, text_preprocessing_result, user, params) question = question_field.description.ask_again_requests[question_field.ask_again_counter] question_field.ask_again_counter += 1 @@ -51,19 +50,17 @@ def ask_again(self, text_preprocessing_result, user, params): async def _check_field(self, text_preprocessing_result, user, params): form = user.forms[self.form_type] - field = self._field(form, text_preprocessing_result, user, params) + field = await self._field(form, text_preprocessing_result, user, params) return await field.check_can_be_filled(text_preprocessing_result, user) if field else False - def _field(self, form, text_preprocessing_result, user, params): - return self._find_field(form, text_preprocessing_result, user, params) + async def _field(self, form, text_preprocessing_result, user, params): + return await self._find_field(form, text_preprocessing_result, user, params) - def _find_field(self, form, text_preprocessing_result, user, params): + async def _find_field(self, form, text_preprocessing_result, user, params): for field_name in form.fields.descriptions: field = form.fields[field_name] - loop = asyncio.get_event_loop() if not field.valid and field.description.has_requests and \ - loop.run_until_complete( - field.description.requirement.check(text_preprocessing_result, user, params)): + await field.description.requirement.check(text_preprocessing_result, user, params): return field def get_fields_data(self, form, form_key): @@ -115,14 +112,13 @@ async def _extract_data(self, form, text_normalization_result, user, params): text_normalization_result, user, params)) return result - def _validate_extracted_data(self, user, text_preprocessing_result, form, data_extracted, params): + async def _validate_extracted_data(self, user, text_preprocessing_result, form, data_extracted, params): error_msgs = [] for field_key, field in form.description.fields.items(): value = data_extracted.get(field_key) - loop = asyncio.get_event_loop() # is not None is necessary, because 0 and False should be checked, None - shouldn't fill if value is not None and \ - not loop.run_until_complete(field.field_validator.requirement.check(value, params)): + not await field.field_validator.requirement.check(value, params): log_params = { log_const.KEY_NAME: log_const.SCENARIO_RESULT_VALUE, "field_key": field_key @@ -189,13 +185,14 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No logging_params.update(self._log_params()) log("Extracted data=%(data_extracted_str)s", user, logging_params) - validation_error_msg = self._validate_extracted_data(user, text_preprocessing_result, form, data_extracted, params) + validation_error_msg = await self._validate_extracted_data(user, text_preprocessing_result, + form, data_extracted, params) if validation_error_msg: reply_messages = validation_error_msg else: reply_messages, is_break = self._fill_form(user, text_preprocessing_result, form, data_extracted) if not is_break: - field = self._field(form, text_preprocessing_result, user, params) + field = await self._field(form, text_preprocessing_result, user, params) if field: user.history.add_event( Event(type=HistoryConstants.types.FIELD_EVENT, diff --git a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py index f1e6aca7..de573a97 100644 --- a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py +++ b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py @@ -24,10 +24,10 @@ def __init__(self, items, id): def build_scenario_nodes(self): return self._scenario_nodes - def _field(self, form, text_preprocessing_result, user, params): + async def _field(self, form, text_preprocessing_result, user, params): current_node = self.get_current_node(user) internal_form = self._get_internal_form(form.forms, current_node.form_key) - return self._find_field(internal_form, text_preprocessing_result, user, params) + return await self._find_field(internal_form, text_preprocessing_result, user, params) def _set_current_node_id(self, user, node_id): user.scenario_models[self.id].current_node = node_id @@ -124,8 +124,8 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No if extracted is not None and fill_other: fill_other = fill_other and field_descr.fill_other field_data = {field_key: extracted} - _validation_error_msg = self._validate_extracted_data(user, text_preprocessing_result, - internal_form, field_data, params) + _validation_error_msg = await self._validate_extracted_data(user, text_preprocessing_result, + internal_form, field_data, params) if _validation_error_msg: # return only first validation message in form validation_error_msg = validation_error_msg or _validation_error_msg @@ -159,8 +159,8 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No self._set_current_node_id(user, current_node.id) new_node = self.get_next_node(user, current_node, text_preprocessing_result, params) - field = self._find_field(form, text_preprocessing_result, - user, params) if form else None + field = await self._find_field(form, text_preprocessing_result, + user, params) if form else None reply_commands = on_filled_actions if field: diff --git a/scenarios/user/last_scenarios/last_scenarios_description.py b/scenarios/user/last_scenarios/last_scenarios_description.py index a4b21706..2c48095d 100644 --- a/scenarios/user/last_scenarios/last_scenarios_description.py +++ b/scenarios/user/last_scenarios/last_scenarios_description.py @@ -1,6 +1,4 @@ # coding: utf-8 -import asyncio - from lazy import lazy from core.basic_models.requirement.basic_requirements import Requirement @@ -19,8 +17,7 @@ def __init__(self, items, id): def requirement(self): return self._requirement - def check(self, text_preprocessing_result, user): - loop = asyncio.get_event_loop() + async def check(self, text_preprocessing_result, user): return user.message.channel in self._channels and \ - loop.run_until_complete(self.requirement.check(text_preprocessing_result, user)) if self._channels else \ - loop.run_until_complete(self.requirement.check(text_preprocessing_result, user)) + await self.requirement.check(text_preprocessing_result, user) if self._channels else \ + await self.requirement.check(text_preprocessing_result, user) diff --git a/smart_kit/models/dialogue_manager.py b/smart_kit/models/dialogue_manager.py index 26ae9358..4dd1b358 100644 --- a/smart_kit/models/dialogue_manager.py +++ b/smart_kit/models/dialogue_manager.py @@ -40,8 +40,8 @@ async def run(self, text_preprocessing_result, user): if not await scenario.text_fits(text_preprocessing_result, user): - if scenario.check_ask_again_requests(text_preprocessing_result, user, params): - reply = scenario.ask_again(text_preprocessing_result, user, params) + if await scenario.check_ask_again_requests(text_preprocessing_result, user, params): + reply = await scenario.ask_again(text_preprocessing_result, user, params) return reply, True diff --git a/tests/core_tests/basic_scenario_models_test/action_test/test_action.py b/tests/core_tests/basic_scenario_models_test/action_test/test_action.py index 2ae3a7fe..9f5bd69f 100644 --- a/tests/core_tests/basic_scenario_models_test/action_test/test_action.py +++ b/tests/core_tests/basic_scenario_models_test/action_test/test_action.py @@ -1,5 +1,4 @@ # coding: utf-8 -import asyncio import unittest from unittest.mock import Mock, MagicMock, patch @@ -71,7 +70,7 @@ def collect(self, text_preprocessing_result, filter_params=None): return self.data -class ActionTest(unittest.TestCase): +class ActionTest(unittest.IsolatedAsyncioTestCase): def test_nodes_1(self): items = {"nodes": {"answer": "test"}} action = NodeAction(items) @@ -86,37 +85,34 @@ def test_nodes_2(self): nodes = action.nodes self.assertEqual(nodes, {}) - def test_base(self): + async def test_base(self): items = {"nodes": "test"} action = Action(items) try: - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(None, None)) + await action.run(None, None) result = False except NotImplementedError: result = True self.assertEqual(result, True) - def test_external(self): + async def test_external(self): items = {"action": "test_action_key"} action = ExternalAction(items) user = Mock() user.descriptions = {"external_actions": {"test_action_key": MockAction()}} - loop = asyncio.get_event_loop() - self.assertEqual(loop.run_until_complete(action.run(user, None)), ["test action run"]) + self.assertEqual(await action.run(user, None), ["test action run"]) - def test_doing_nothing_action(self): + async def test_doing_nothing_action(self): items = {"nodes": {"answer": "test"}, "command": "test_name"} action = DoingNothingAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(None, None)) + result = await action.run(None, None) self.assertIsInstance(result, list) command = result[0] self.assertIsInstance(command, Command) self.assertEqual(command.name, "test_name") self.assertEqual(command.payload, {"answer": "test"}) - def test_requirement_action(self): + async def test_requirement_action(self): registered_factories[Requirement] = requirement_factory requirements["test"] = MockRequirement registered_factories[Action] = action_factory @@ -125,14 +121,13 @@ def test_requirement_action(self): action = RequirementAction(items) self.assertIsInstance(action.requirement, MockRequirement) self.assertIsInstance(action.internal_item, MockAction) - loop = asyncio.get_event_loop() - self.assertEqual(loop.run_until_complete(action.run(None, None)), ["test action run"]) + self.assertEqual(await action.run(None, None), ["test action run"]) items = {"requirement": {"type": "test", "result": False}, "action": {"type": "test"}} action = RequirementAction(items) - result = loop.run_until_complete(action.run(None, None)) + result = await action.run(None, None) self.assertIsNone(result) - def test_requirement_choice(self): + async def test_requirement_choice(self): items = {"requirement_actions": [ {"requirement": {"type": "test", "result": False}, "action": {"type": "test", "result": "action1"}}, {"requirement": {"type": "test", "result": True}, "action": {"type": "test", "result": "action2"}} @@ -140,11 +135,10 @@ def test_requirement_choice(self): choice_action = ChoiceAction(items) self.assertIsInstance(choice_action.items, list) self.assertIsInstance(choice_action.items[0], RequirementAction) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(choice_action.run(None, None)) + result = await choice_action.run(None, None) self.assertEqual(result, "action2") - def test_requirement_choice_else(self): + async def test_requirement_choice_else(self): items = { "requirement_actions": [ {"requirement": {"type": "test", "result": False}, "action": {"type": "test", "result": "action1"}}, @@ -155,11 +149,10 @@ def test_requirement_choice_else(self): choice_action = ChoiceAction(items) self.assertIsInstance(choice_action.items, list) self.assertIsInstance(choice_action.items[0], RequirementAction) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(choice_action.run(None, None)) + result = await choice_action.run(None, None) self.assertEqual(result, "action3") - def test_string_action(self): + async def test_string_action(self): expected = [Command("cmd_id", {"item": "template", "params": "params"})] user = MagicMock() template = Mock() @@ -171,12 +164,11 @@ def test_string_action(self): "nodes": {"item": "template", "params": "{{params}}"}} action = StringAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual(expected[0].name, result[0].name) self.assertEqual(expected[0].payload, result[0].payload) - def test_else_action_if(self): + async def test_else_action_if(self): registered_factories[Requirement] = requirement_factory requirements["test"] = MockRequirement registered_factories[Action] = action_factory @@ -188,10 +180,9 @@ def test_else_action_if(self): "else_action": {"type": "test", "result": "else_action"} } action = ElseAction(items) - loop = asyncio.get_event_loop() - self.assertEqual(loop.run_until_complete(action.run(user, None)), "main_action") + self.assertEqual(await action.run(user, None), "main_action") - def test_else_action_else(self): + async def test_else_action_else(self): registered_factories[Requirement] = requirement_factory requirements["test"] = MockRequirement registered_factories[Action] = action_factory @@ -203,10 +194,9 @@ def test_else_action_else(self): "else_action": {"type": "test", "result": "else_action"} } action = ElseAction(items) - loop = asyncio.get_event_loop() - self.assertEqual(loop.run_until_complete(action.run(user, None)), "else_action") + self.assertEqual(await action.run(user, None), "else_action") - def test_else_action_no_else_if(self): + async def test_else_action_no_else_if(self): registered_factories[Requirement] = requirement_factory requirements["test"] = MockRequirement registered_factories[Action] = action_factory @@ -217,10 +207,9 @@ def test_else_action_no_else_if(self): "action": {"type": "test", "result": "main_action"}, } action = ElseAction(items) - loop = asyncio.get_event_loop() - self.assertEqual(loop.run_until_complete(action.run(user, None)), "main_action") + self.assertEqual(await action.run(user, None), "main_action") - def test_else_action_no_else_else(self): + async def test_else_action_no_else_else(self): registered_factories[Requirement] = requirement_factory requirements["test"] = MockRequirement registered_factories[Action] = action_factory @@ -231,11 +220,10 @@ def test_else_action_no_else_else(self): "action": {"type": "test", "result": "main_action"}, } action = ElseAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertIsNone(result) - def test_composite_action(self): + async def test_composite_action(self): registered_factories[Action] = action_factory actions["action_mock"] = MockAction user = Mock() @@ -246,11 +234,10 @@ def test_composite_action(self): ] } action = CompositeAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual(['test action run', 'test action run'], result) - def test_node_action_support_templates(self): + async def test_node_action_support_templates(self): params = { "markup": "italic", "email": "heyho@sberbank.ru", @@ -274,11 +261,10 @@ def test_node_action_support_templates(self): self.assertIsInstance(template, UnifiedTemplate) user = MagicMock() user.parametrizer = MockSimpleParametrizer(user, {"data": params}) - loop = asyncio.get_event_loop() - output = loop.run_until_complete(action.run(user=user, text_preprocessing_result=None))[0].payload["answer"] + output = await action.run(user=user, text_preprocessing_result=None)[0].payload["answer"] self.assertEqual(output, expected) - def test_string_action_support_templates(self): + async def test_string_action_support_templates(self): params = { "answer_text": "some_text", "buttons_number": 3 @@ -300,12 +286,11 @@ def test_string_action_support_templates(self): action = StringAction(items) user = MagicMock() user.parametrizer = MockSimpleParametrizer(user, {"data": params}) - loop = asyncio.get_event_loop() - output = loop.run_until_complete(action.run(user=user, text_preprocessing_result=None))[0].payload + output = await action.run(user=user, text_preprocessing_result=None)[0].payload self.assertEqual(output, expected) -class NonRepeatingActionTest(unittest.TestCase): +class NonRepeatingActionTest(unittest.IsolatedAsyncioTestCase): def setUp(self): self.expected = Mock() self.expected1 = Mock() @@ -319,60 +304,55 @@ def setUp(self): registered_factories[Action] = action_factory actions["action_mock"] = MockAction - def test_run_available_indexes(self): + async def test_run_available_indexes(self): self.user.last_action_ids["last_action_ids_storage"].get_list.side_effect = [[0]] - loop = asyncio.get_event_loop() - result = loop.run_until_complete(self.action.run(self.user, None)) + result = await self.action.run(self.user, None) self.user.last_action_ids["last_action_ids_storage"].add.assert_called_once() self.assertEqual(result, self.expected1) - def test_run_no_available_indexes(self): + async def test_run_no_available_indexes(self): self.user.last_action_ids["last_action_ids_storage"].get_list.side_effect = [[0, 1]] - loop = asyncio.get_event_loop() - result = loop.run_until_complete(self.action.run(self.user, None)) + result = await self.action.run(self.user, None) self.assertEqual(result, self.expected) -class CounterIncrementActionTest(unittest.TestCase): - def test_run(self): +class CounterIncrementActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): user = Mock() counter = Mock() counter.inc = Mock() user.counters = {"test": counter} items = {"key": "test"} action = CounterIncrementAction(items) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) user.counters["test"].inc.assert_called_once() -class CounterDecrementActionTest(unittest.TestCase): - def test_run(self): +class CounterDecrementActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): user = Mock() counter = Mock() counter.dec = Mock() user.counters = {"test": counter} items = {"key": "test"} action = CounterDecrementAction(items) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) user.counters["test"].dec.assert_called_once() -class CounterClearActionTest(unittest.TestCase): - def test_run(self): +class CounterClearActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): user = Mock() user.counters = Mock() user.counters.inc = Mock() items = {"key": "test"} action = CounterClearAction(items) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) user.counters.clear.assert_called_once() -class CounterSetActionTest(unittest.TestCase): - def test_run(self): +class CounterSetActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): user = Mock() counter = Mock() counter.inc = Mock() @@ -380,13 +360,12 @@ def test_run(self): user.counters = counters items = {"key": "test"} action = CounterSetAction(items) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) user.counters["test"].set.assert_called_once() -class CounterCopyActionTest(unittest.TestCase): - def test_run(self): +class CounterCopyActionTest(unittest.IsolatedAsyncioTestCase): + async def test_run(self): user = Mock() counter_src = Mock() counter_src.value = 10 @@ -394,14 +373,13 @@ def test_run(self): user.counters = {"src": counter_src, "dst": counter_dst} items = {"source": "src", "destination": "dst"} action = CounterCopyAction(items) - loop = asyncio.get_event_loop() - loop.run_until_complete(action.run(user, None)) + await action.run(user, None) user.counters["dst"].set.assert_called_once_with(user.counters["src"].value, action.reset_time, action.time_shift) -class AfinaAnswerActionTest(unittest.TestCase): - def test_typical_answer(self): +class AfinaAnswerActionTest(unittest.IsolatedAsyncioTestCase): + async def test_typical_answer(self): user = Mock() user.parametrizer = MockParametrizer(user, {}) expected = [MagicMock(_name="ANSWER_TO_USER", raw={'messageName': 'ANSWER_TO_USER', @@ -412,12 +390,11 @@ def test_typical_answer(self): } } action = AfinaAnswerAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual(expected[0]._name, result[0].name) self.assertEqual(expected[0].raw, result[0].raw) - def test_typical_answer_with_other(self): + async def test_typical_answer_with_other(self): user = Mock() user.parametrizer = MockParametrizer(user, {}) expected = [MagicMock(_name="ANSWER_TO_USER", raw={'messageName': 'ANSWER_TO_USER', @@ -432,12 +409,11 @@ def test_typical_answer_with_other(self): } } action = AfinaAnswerAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual(expected[0]._name, result[0].name) self.assertEqual(expected[0].raw, result[0].raw) - def test_typical_answer_with_pers_info(self): + async def test_typical_answer_with_pers_info(self): expected = [MagicMock(_name="ANSWER_TO_USER", raw={'messageName': 'ANSWER_TO_USER', 'payload': {'answer': 'Ivan Ivanov'}})] user = Mock() @@ -446,12 +422,11 @@ def test_typical_answer_with_pers_info(self): user.message.payload = {"personInfo": {"name": "Ivan Ivanov"}} items = {"nodes": {"answer": ["{{payload.personInfo.name}}"]}} action = AfinaAnswerAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual(expected[0]._name, result[0].name) self.assertEqual(expected[0].raw, result[0].raw) - def test_items_empty(self): + async def test_items_empty(self): user = Mock() user.parametrizer = MockParametrizer(user, {}) template = Mock() @@ -459,11 +434,10 @@ def test_items_empty(self): user.descriptions = {"render_templates": template} items = None action = AfinaAnswerAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual(result, []) - def test__items_empty_dict(self): + async def test__items_empty_dict(self): user = Mock() user.parametrizer = MockParametrizer(user, {}) template = Mock() @@ -471,13 +445,12 @@ def test__items_empty_dict(self): user.descriptions = {"render_templates": template} items = {} action = AfinaAnswerAction(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual(result, []) -class CardAnswerActionTest(unittest.TestCase): - def test_typical_answer(self): +class CardAnswerActionTest(unittest.IsolatedAsyncioTestCase): + async def test_typical_answer(self): user = Mock() user.parametrizer = MockParametrizer(user, {}) user.message = Mock() @@ -526,15 +499,13 @@ def test_typical_answer(self): exp3 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText1', 'items': [{'bubble': {'text': 'Text1'}}, {'card': {'type': 'simple_list', 'header': '1 доллар США ', 'items': [{'title': 'Купить', 'body': '67.73 RUR'}, {'title': 'Продать', 'body': '64.56 RUR'}], 'footer': 'Ivan Ivanov Сбербанк Онлайн на сегодня 17:53 при обмене до 1000 USD'}}], 'suggestions': {'buttons': [{'title': 'Отделения', 'action': {'text': 'Где ближайщие отделения сбера?', 'type': 'text'}}]}}}" exp4 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText1', 'items': [{'bubble': {'text': 'Text2'}}, {'card': {'type': 'simple_list', 'header': '1 доллар США ', 'items': [{'title': 'Купить', 'body': '67.73 RUR'}, {'title': 'Продать', 'body': '64.56 RUR'}], 'footer': 'Ivan Ivanov Сбербанк Онлайн на сегодня 17:53 при обмене до 1000 USD'}}], 'suggestions': {'buttons': [{'title': 'Отделения', 'action': {'text': 'Где ближайщие отделения сбера?', 'type': 'text'}}]}}}" expect_arr = [exp1, exp2, exp3, exp4] - loop = asyncio.get_event_loop() for i in range(10): action = SDKAnswer(items) - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual("ANSWER_TO_USER", result[0].name) self.assertTrue(str(result[0].raw) in expect_arr) - - def test_typical_answer_without_items(self): + async def test_typical_answer_without_items(self): user = Mock() user.parametrizer = MockParametrizer(user, {}) user.message = Mock() @@ -550,14 +521,13 @@ def test_typical_answer_without_items(self): exp3 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText2'}}" exp4 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText2'}}" exp_list = [exp1, exp2, exp3, exp4] - loop = asyncio.get_event_loop() for i in range(10): action = SDKAnswer(items) - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual("ANSWER_TO_USER", result[0].name) self.assertTrue(str(result[0].raw) in exp_list) - def test_typical_answer_without_nodes(self): + async def test_typical_answer_without_nodes(self): user = Mock() user.parametrizer = MockParametrizer(user, {}) user.message = Mock() @@ -589,16 +559,15 @@ def test_typical_answer_without_nodes(self): exp3 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText1', 'suggestions': {'buttons': [{'title': 'отделения2', 'action': {'text': 'отделения', 'type': 'text'}}, {'title': 'кредит1', 'action': {'text': 'кредит', 'type': 'text'}}]}}}" exp4 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'pronounceText1', 'suggestions': {'buttons': [{'title': 'отделения2', 'action': {'text': 'отделения', 'type': 'text'}}, {'title': 'кредит2', 'action': {'text': 'кредит', 'type': 'text'}}]}}}" expect_arr = [exp1, exp2, exp3, exp4] - loop = asyncio.get_event_loop() for i in range(10): action = SDKAnswer(items) - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertEqual("ANSWER_TO_USER", result[0].name) self.assertTrue(str(result[0].raw) in expect_arr) -class SDKRandomAnswer(unittest.TestCase): - def test_SDKItemAnswer_full(self): +class SDKRandomAnswer(unittest.IsolatedAsyncioTestCase): + async def test_SDKItemAnswer_full(self): registered_factories[SdkAnswerItem] = items_factory answer_items["bubble_text"] = BubbleText @@ -683,12 +652,11 @@ def test_SDKItemAnswer_full(self): exp2 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'items': [{'bubble': {'text': 'Ivan Ivanov', 'markdown': False}}, {'card': {'cards_params': 'a lot of params'}}], 'suggestions': {'buttons': [{'title': 'p1', 'action': {'text': 'Ivan Ivanov', 'type': 'text'}}, {'title': 'p1', 'action': {'text': 'Ivan Ivanov', 'type': 'text'}}, {'title': 'p1', 'action': {'deep_link': 'www.ww.w', 'type': 'deep_link'}}]}, 'pronounceText': 'p1'}}" action = SDKAnswerToUser(items) - loop = asyncio.get_event_loop() for i in range(3): - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertTrue(str(result[0].raw) in [exp1, exp2]) - def test_SDKItemAnswer_root(self): + async def test_SDKItemAnswer_root(self): registered_factories[SdkAnswerItem] = items_factory answer_items["bubble_text"] = BubbleText @@ -733,12 +701,11 @@ def test_SDKItemAnswer_root(self): exp2 = "{'messageName': 'ANSWER_TO_USER', 'payload': {'pronounceText': 'p2'}}" action = SDKAnswerToUser(items) - loop = asyncio.get_event_loop() for i in range(3): - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertTrue(str(result[0].raw) in [exp1, exp2]) - def test_SDKItemAnswer_simple(self): + async def test_SDKItemAnswer_simple(self): registered_factories[SdkAnswerItem] = items_factory answer_items["bubble_text"] = BubbleText @@ -756,11 +723,10 @@ def test_SDKItemAnswer_simple(self): ] } action = SDKAnswerToUser(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertDictEqual(result[0].raw, {'messageName': 'ANSWER_TO_USER', 'payload': {'items': [{'bubble': {'text': '42', 'markdown': True}}]}}) - def test_SDKItemAnswer_suggestions_template(self): + async def test_SDKItemAnswer_suggestions_template(self): registered_factories[SdkAnswerItem] = items_factory answer_items["bubble_text"] = BubbleText @@ -779,8 +745,7 @@ def test_SDKItemAnswer_suggestions_template(self): } } action = SDKAnswerToUser(items) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(user, None)) + result = await action.run(user, None) self.assertDictEqual( result[0].raw, { @@ -792,4 +757,5 @@ def test_SDKItemAnswer_suggestions_template(self): ] } } - }) + } + ) diff --git a/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py b/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py index 4155a8ef..81eef59c 100644 --- a/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py +++ b/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py @@ -1,18 +1,17 @@ -import asyncio -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from core.basic_models.actions.basic_actions import Action, action_factory, actions, DoingNothingAction, RandomAction from core.model.registered import registered_factories -class TestRandomAction(TestCase): +class TestRandomAction(IsolatedAsyncioTestCase): @classmethod def setUpClass(cls) -> None: registered_factories[Action] = action_factory actions["do_nothing"] = DoingNothingAction - def test_1(self): + async def test_1(self): items = { "actions": [ @@ -33,11 +32,10 @@ def test_1(self): ] } action = RandomAction(items, 5) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(None, None)) + result = await action.run(None, None) self.assertIsNotNone(result) - def test_2(self): + async def test_2(self): items = { "actions": [ { @@ -50,6 +48,5 @@ def test_2(self): ] } action = RandomAction(items, 5) - loop = asyncio.get_event_loop() - result = loop.run_until_complete(action.run(None, None)) + result = await action.run(None, None) self.assertIsNotNone(result) diff --git a/tests/core_tests/requirements_test/_trial_temp/_trial_marker b/tests/core_tests/requirements_test/_trial_temp/_trial_marker new file mode 100755 index 00000000..e69de29b From 48dd518050bfca868b8189e2a8e6d4e5e15b94f1 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Sun, 14 Nov 2021 04:18:36 +0300 Subject: [PATCH 064/116] remove useless files --- tests/core_tests/requirements_test/_trial_temp/_trial_marker | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100755 tests/core_tests/requirements_test/_trial_temp/_trial_marker diff --git a/tests/core_tests/requirements_test/_trial_temp/_trial_marker b/tests/core_tests/requirements_test/_trial_temp/_trial_marker deleted file mode 100755 index e69de29b..00000000 From 13023560e07e7961467a9da3a01bdd55a6cf4a78 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 15 Nov 2021 00:14:01 +0300 Subject: [PATCH 065/116] fix tests --- .../action_test/test_action.py | 4 ++-- .../action_test/test_random_action.py | 3 +-- .../basic_scenario_models_test/test_parametrizer.py | 5 ++--- tests/scenarios_tests/actions_test/test_action.py | 5 ++--- .../fillers/test_available_info_filler.py | 9 +++------ .../scenarios_tests/fillers/test_composite_filler.py | 5 ++--- tests/scenarios_tests/fillers/test_regexps_filler.py | 11 +++++------ .../scenarios_test/test_tree_scenario.py | 2 +- .../scenarios_tests/user_models/test_is_int_value.py | 5 ++--- 9 files changed, 20 insertions(+), 29 deletions(-) diff --git a/tests/core_tests/basic_scenario_models_test/action_test/test_action.py b/tests/core_tests/basic_scenario_models_test/action_test/test_action.py index 9f5bd69f..d11abd82 100644 --- a/tests/core_tests/basic_scenario_models_test/action_test/test_action.py +++ b/tests/core_tests/basic_scenario_models_test/action_test/test_action.py @@ -261,7 +261,7 @@ async def test_node_action_support_templates(self): self.assertIsInstance(template, UnifiedTemplate) user = MagicMock() user.parametrizer = MockSimpleParametrizer(user, {"data": params}) - output = await action.run(user=user, text_preprocessing_result=None)[0].payload["answer"] + output = (await action.run(user=user, text_preprocessing_result=None))[0].payload["answer"] self.assertEqual(output, expected) async def test_string_action_support_templates(self): @@ -286,7 +286,7 @@ async def test_string_action_support_templates(self): action = StringAction(items) user = MagicMock() user.parametrizer = MockSimpleParametrizer(user, {"data": params}) - output = await action.run(user=user, text_preprocessing_result=None)[0].payload + output = (await action.run(user=user, text_preprocessing_result=None))[0].payload self.assertEqual(output, expected) diff --git a/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py b/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py index 81eef59c..2449c9a0 100644 --- a/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py +++ b/tests/core_tests/basic_scenario_models_test/action_test/test_random_action.py @@ -6,8 +6,7 @@ class TestRandomAction(IsolatedAsyncioTestCase): - @classmethod - def setUpClass(cls) -> None: + def setUp(self) -> None: registered_factories[Action] = action_factory actions["do_nothing"] = DoingNothingAction diff --git a/tests/core_tests/basic_scenario_models_test/test_parametrizer.py b/tests/core_tests/basic_scenario_models_test/test_parametrizer.py index b409d62c..ca2859f6 100644 --- a/tests/core_tests/basic_scenario_models_test/test_parametrizer.py +++ b/tests/core_tests/basic_scenario_models_test/test_parametrizer.py @@ -6,9 +6,8 @@ class ParametrizerTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.user = Mock(message=Mock()) + def setUp(self): + self.user = Mock(message=Mock()) def test_get_user_data(self): expected = ["message"] diff --git a/tests/scenarios_tests/actions_test/test_action.py b/tests/scenarios_tests/actions_test/test_action.py index 89481a63..3409d95c 100644 --- a/tests/scenarios_tests/actions_test/test_action.py +++ b/tests/scenarios_tests/actions_test/test_action.py @@ -127,15 +127,14 @@ async def test_run(self): class SaveBehaviorActionTest(unittest.IsolatedAsyncioTestCase): - @classmethod - def setUp(cls): + def setUp(self): user = Mock() user.message = Mock() user.parametrizer = MockParametrizer(user, {}) user.last_scenarios.last_scenario_name = "scenario_id" test_incremental_id = "test_incremental_id" user.message.incremental_id = test_incremental_id - cls.user = user + self.user = user async def test_save_behavior_scenario_name(self): data = {"behavior": "test"} diff --git a/tests/scenarios_tests/fillers/test_available_info_filler.py b/tests/scenarios_tests/fillers/test_available_info_filler.py index add4e265..b0919ac0 100644 --- a/tests/scenarios_tests/fillers/test_available_info_filler.py +++ b/tests/scenarios_tests/fillers/test_available_info_filler.py @@ -23,13 +23,10 @@ def collect(self, text_preprocessing_result=None, filter_params=None): class TestAvailableInfoFiller(IsolatedAsyncioTestCase): - @classmethod - def setUpClass(cls): - cls.address = "Address!" - payload_items = {'value': '{{payload.sf_answer.address}}'} - cls.payload_filler = AvailableInfoFiller(payload_items) - def setUp(self): + self.address = "Address!" + payload_items = {'value': '{{payload.sf_answer.address}}'} + self.payload_filler = AvailableInfoFiller(payload_items) template = Mock() template.get_template = Mock(return_value=[]) user = Mock() diff --git a/tests/scenarios_tests/fillers/test_composite_filler.py b/tests/scenarios_tests/fillers/test_composite_filler.py index cd1fa5fe..3e423a75 100644 --- a/tests/scenarios_tests/fillers/test_composite_filler.py +++ b/tests/scenarios_tests/fillers/test_composite_filler.py @@ -17,11 +17,10 @@ async def extract(self, text_preprocessing_result, user, params): class TestCompositeFiller(IsolatedAsyncioTestCase): - @classmethod - def setUp(cls): + def setUp(self): registered_factories[FieldFillerDescription] = field_filler_factory field_filler_description["mock_filler"] = MockFiller - cls.user = Mock() + self.user = Mock() async def test_first_filler(self): expected = "first" diff --git a/tests/scenarios_tests/fillers/test_regexps_filler.py b/tests/scenarios_tests/fillers/test_regexps_filler.py index 98f59fd9..5651540c 100644 --- a/tests/scenarios_tests/fillers/test_regexps_filler.py +++ b/tests/scenarios_tests/fillers/test_regexps_filler.py @@ -5,16 +5,15 @@ class Test_regexps_filler(IsolatedAsyncioTestCase): - @classmethod - def setUpClass(cls): - cls.items = {} - cls.items["exps"] = ["номер[а-я]*\.?\s?(\d+)", "n\.?\s?(\d+)", "nn\.?\s?(\d+)", "#\.?\s?(\d+)", + def setUp(self): + self.items = {} + self.items["exps"] = ["номер[а-я]*\.?\s?(\d+)", "n\.?\s?(\d+)", "nn\.?\s?(\d+)", "#\.?\s?(\d+)", "##\.?\s?(\d+)", "№\.?\s?(\d+)", "№№\.?\s?(\d+)", "платеж[а-я]+\.?\s?(\d+)", "поручен[а-я]+\.?\s?(\d+)", "п\\s?,\\s?п\.?\s?(\d+)", "п\\s?\\/\\s?п\.?\s?(\d+)"] - cls.items["delimiter"] = "|" + self.items["delimiter"] = "|" - cls.filler = AllRegexpsFieldFiller(cls.items) + self.filler = AllRegexpsFieldFiller(self.items) async def test_extract_1(self): field_value = "Просим отозвать платежное поручение 14 от 23.01.19 на сумму 3500 и вернуть деньги на расчетный счет." diff --git a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py index 9b713aeb..2a4d3fac 100644 --- a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py +++ b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py @@ -56,7 +56,7 @@ async def test_1(self): internal_form = Mock(name="internal_form_mock") internal_form.description.fields.items = Mock(return_value=[("age", field_descriptor)]) - internal_form.field.field_validator.requirement.check = Mock(return_value=True) + internal_form.field.field_validator.requirement.check = AsyncMock(return_value=True) internal_form.fields = MagicMock() internal_form.fields.values.items = Mock(return_value={"age": 61}) internal_form.is_valid = Mock(return_value=True) diff --git a/tests/scenarios_tests/user_models/test_is_int_value.py b/tests/scenarios_tests/user_models/test_is_int_value.py index a719762d..df08405c 100644 --- a/tests/scenarios_tests/user_models/test_is_int_value.py +++ b/tests/scenarios_tests/user_models/test_is_int_value.py @@ -6,10 +6,9 @@ class IsIntFieldRequirementTest(IsolatedAsyncioTestCase): - @classmethod - def setUp(cls): + def setUp(self): items = {} - cls.requirement = IsIntFieldRequirement(items) + self.requirement = IsIntFieldRequirement(items) async def test_is_int_number_string(self): text = "123" From 32604e7fccc49a5f4cd947c6d4b44349d67a1004 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Mon, 15 Nov 2021 10:35:00 +0300 Subject: [PATCH 066/116] draft async db_adapters and repositories --- core/configs/base_config.py | 9 ++-- core/db_adapter/db_adapter.py | 58 ++++++++++++------------ core/db_adapter/os_adapter.py | 22 ++++----- core/repositories/file_repository.py | 4 +- core/utils/rerunable.py | 10 ++-- smart_kit/start_points/base_main_loop.py | 4 -- 6 files changed, 52 insertions(+), 55 deletions(-) diff --git a/core/configs/base_config.py b/core/configs/base_config.py index 0f9a56dd..09a369ee 100644 --- a/core/configs/base_config.py +++ b/core/configs/base_config.py @@ -1,3 +1,4 @@ +import asyncio import os from typing import List @@ -28,9 +29,11 @@ def init(self): self.init_repositories() def init_repositories(self): - for rep in self.repositories: - rep.load() - self.registered_repositories[rep.key] = rep + await asyncio.gather([self._register_repo(rep) for rep in self.repositories]) + + async def _register_repo(self, rep): + await rep.load() + self.registered_repositories[rep.key] = rep def raw(self): items = self.registered_repositories diff --git a/core/db_adapter/db_adapter.py b/core/db_adapter/db_adapter.py index 93cc3227..0c151f8a 100644 --- a/core/db_adapter/db_adapter.py +++ b/core/db_adapter/db_adapter.py @@ -13,72 +13,70 @@ class DBAdapterException(Exception): class DBAdapter(Rerunable): - IS_ASYNC = False - def __init__(self, config=None): super(DBAdapter, self).__init__(config) self._client = None - def _on_prepare(self): + async def _on_prepare(self): raise NotImplementedError - def connect(self): + async def connect(self): raise NotImplementedError - def _open(self, filename, *args, **kwargs): + async def _open(self, filename, *args, **kwargs): raise NotImplementedError - def _save(self, id, data): + async def _save(self, id, data): raise NotImplementedError - def _replace_if_equals(self, id, sample, data): + async def _replace_if_equals(self, id, sample, data): raise NotImplementedError - def _get(self, id): + async def _get(self, id): raise NotImplementedError - def _list_dir(self, path): + async def _list_dir(self, path): raise NotImplementedError - def _glob(self, path, pattern): + async def _glob(self, path, pattern): raise NotImplementedError - def _path_exists(self, path): + async def _path_exists(self, path): raise NotImplementedError - def _mtime(self, path): + async def _mtime(self, path): raise NotImplementedError - def open(self, filename, *args, **kwargs): - return self._run(self._open, filename, *args, **kwargs) + async def open(self, filename, *args, **kwargs): + return await self._run(self._open, filename, *args, **kwargs) - def glob(self, path, pattern): - return self._run(self._glob, path, pattern) + async def glob(self, path, pattern): + return await self._run(self._glob, path, pattern) - def path_exists(self, path): - return self._run(self._path_exists, path) + async def path_exists(self, path): + return await self._run(self._path_exists, path) - def mtime(self, path): - return self._run(self._mtime, path) + async def mtime(self, path): + return await self._run(self._mtime, path) @monitoring.got_histogram("save_time") - def save(self, id, data): - return self._run(self._save, id, data) + async def save(self, id, data): + return await self._run(self._save, id, data) @monitoring.got_histogram("save_time") - def replace_if_equals(self, id, sample, data): - return self._run(self._replace_if_equals, id, sample, data) + async def replace_if_equals(self, id, sample, data): + return await self._run(self._replace_if_equals, id, sample, data) @monitoring.got_histogram("get_time") - def get(self, id): - return self._run(self._get, id) + async def get(self, id): + return await self._run(self._get, id) - def list_dir(self, path): - return self._run(self._list_dir, path) + async def list_dir(self, path): + return await self._run(self._list_dir, path) @property - def _handled_exception(self): + async def _handled_exception(self): return Exception - def _on_all_tries_fail(self): + async def _on_all_tries_fail(self): raise diff --git a/core/db_adapter/os_adapter.py b/core/db_adapter/os_adapter.py index 14347434..56faf9d3 100644 --- a/core/db_adapter/os_adapter.py +++ b/core/db_adapter/os_adapter.py @@ -7,44 +7,44 @@ class OSAdapter(DBAdapter): - def _save(self, id, data): + async def _save(self, id, data): raise error.NotSupportedOperation - def _replace_if_equals(self, id, sample, data): + async def _replace_if_equals(self, id, sample, data): raise error.NotSupportedOperation - def _get(self, id): + async def _get(self, id): raise error.NotSupportedOperation - def connect(self): + async def connect(self): pass - def _on_prepare(self): + async def _on_prepare(self): pass @property def source(self): return self - def _list_dir(self, path): + async def _list_dir(self, path): result = [] for path, subdirs, files in os.walk(path): result.extend([os.path.join(path, name) for name in files if not name.startswith(".")]) return result - def _open(self, filename, *args, **kwargs): + async def _open(self, filename, *args, **kwargs): return io.open(filename, *args, **kwargs) - def _get_counter_name(self): + async def _get_counter_name(self): return "os_adapter" - def _glob(self, path, pattern): + async def _glob(self, path, pattern): files_list = self._list_dir(path) filtered = fnmatch.filter(files_list, pattern) return filtered - def _path_exists(self, path): + async def _path_exists(self, path): return os.path.exists(path) - def _mtime(self, path): + async def _mtime(self, path): return os.path.getmtime(path) diff --git a/core/repositories/file_repository.py b/core/repositories/file_repository.py index 642ef07d..ae72107a 100644 --- a/core/repositories/file_repository.py +++ b/core/repositories/file_repository.py @@ -17,10 +17,10 @@ def __init__(self, filename, loader, source=None, save_target=None, saver=None, self.save_target = save_target self._file_exist = False - def load(self): + async def load(self): if self.source.path_exists(self.filename): self._file_exist = True - with self.source.open(self.filename, 'rb') as stream: + with await self.source.open(self.filename, 'rb') as stream: binary_data = stream.read() data = binary_data.decode() self.fill(self.loader(data)) diff --git a/core/utils/rerunable.py b/core/utils/rerunable.py index 07703910..093f6c03 100644 --- a/core/utils/rerunable.py +++ b/core/utils/rerunable.py @@ -20,14 +20,14 @@ def _on_prepare(self): def _on_all_tries_fail(self): raise NotImplementedError - def _run(self, action, *args, _try_count=None, **kwargs): + async def _run(self, action, *args, _try_count=None, **kwargs): if _try_count is None: _try_count = self.try_count if _try_count <= 0: self._on_all_tries_fail() _try_count = _try_count - 1 try: - result = action(*args, **kwargs) + result = await action(*args, **kwargs) except self._handled_exception as e: params = { "class_name": str(self.__class__), @@ -36,10 +36,10 @@ def _run(self, action, *args, _try_count=None, **kwargs): log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE } log("%(class_name)s run failed with %(exception)s.\n Got %(try_count)s tries left.", - params=params, - level="ERROR") + params=params, + level="ERROR") self._on_prepare() - result = self._run(action, *args, _try_count=_try_count, **kwargs) + result = await self._run(action, *args, _try_count=_try_count, **kwargs) counter_name = self._get_counter_name() if counter_name: monitoring.got_counter(f"{counter_name}_exception") diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index c8d0a74e..ca472f70 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -72,10 +72,6 @@ def __init__( def get_db(self): db_adapter = db_adapter_factory(self.settings["template_settings"].get("db_adapter", {})) - if not db_adapter.IS_ASYNC: - raise Exception( - f"Blocking adapter {db_adapter.__class__.__name__} is not good for {self.__class__.__name__}" - ) self.loop.run_until_complete(db_adapter.connect()) return db_adapter From 94deb90daaeb40175d791f7b022d3c3b676e6daa Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 16 Nov 2021 04:35:36 +0300 Subject: [PATCH 067/116] small fixes in main_loop_kafka --- smart_kit/start_points/main_loop_kafka.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 48da294e..86be1e51 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -83,7 +83,7 @@ def __init__(self, *args, **kwargs): log("%(class_name)s.__init__ completed.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) - except: + except Exception: log("%(class_name)s.__init__ exception.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}, level="ERROR", exc_info=True) @@ -109,7 +109,7 @@ async def main_coro(self): await asyncio.gather(*tasks) async def healthcheck_coro(self): - while True: + while self.is_work: if not self.health_check_server_future or self.health_check_server_future.done() or \ self.health_check_server_future.cancelled(): self.health_check_server_future = self.loop.run_in_executor(None, self.health_check_server.iterate) @@ -533,8 +533,6 @@ def stop(self, signum, frame): log("Kafka handler is stopped", level="WARNING") self.is_work = False - sys.exit() - async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): if not self.is_work: return From a0672e6e198bebdb6f216e6647e27e6c6ea69718 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Tue, 16 Nov 2021 11:06:57 +0300 Subject: [PATCH 068/116] semi-fix async repos --- core/configs/base_config.py | 13 ++++++------ core/repositories/base_repository.py | 12 +++++------ core/repositories/classifier_repository.py | 6 +++--- core/repositories/csv_repository.py | 6 +++--- core/repositories/dill_repository.py | 6 +++--- core/repositories/file_repository.py | 8 ++++---- core/repositories/folder_repository.py | 20 +++++++++---------- core/repositories/items_repository.py | 6 +++--- core/repositories/shard_repository.py | 8 ++++---- core/utils/rerunable.py | 19 +++++++++++------- .../core_tests/repositories_test/test_repo.py | 6 +++--- 11 files changed, 57 insertions(+), 53 deletions(-) diff --git a/core/configs/base_config.py b/core/configs/base_config.py index 09a369ee..82bb80c9 100644 --- a/core/configs/base_config.py +++ b/core/configs/base_config.py @@ -11,6 +11,7 @@ def __init__(self, **kwargs): self.registered_repositories = Registered() self.repositories = [] self.source = kwargs.get("source") + self.loop = asyncio.get_event_loop() def __getitem__(self, key): return self.registered_repositories[key].data @@ -26,14 +27,12 @@ def subfolder_path(self, filename): return os.path.join(self._subfolder, filename) def init(self): - self.init_repositories() + self.loop.run_until_complete(self.init_repositories()) - def init_repositories(self): - await asyncio.gather([self._register_repo(rep) for rep in self.repositories]) - - async def _register_repo(self, rep): - await rep.load() - self.registered_repositories[rep.key] = rep + async def init_repositories(self): + for rep in self.repositories: + await rep.load() + self.registered_repositories[rep.key] = rep def raw(self): items = self.registered_repositories diff --git a/core/repositories/base_repository.py b/core/repositories/base_repository.py index fc160efe..6badfae1 100644 --- a/core/repositories/base_repository.py +++ b/core/repositories/base_repository.py @@ -16,10 +16,10 @@ def data(self): return self._data @data.setter - def data(self, value): + async def data(self, value): self._data = value - def load(self): + async def load(self): params = { "repository_class_name": self.__class__.__name__, "repository_key": self.key, @@ -28,10 +28,10 @@ def load(self): log("%(repository_class_name)s.load %(repository_key)s repo loading completed.", params=params, level="WARNING") - def fill(self, data): + async def fill(self, data): self.data = data - def clear(self): + async def clear(self): self.data.clear() log("%(repository_class_name)s.clear %(repository_key)s cleared.", params={"repository_class_name": self.__class__.__name__, @@ -39,8 +39,8 @@ def clear(self): log_const.KEY_NAME: log_const.REPOSITORY_CLEAR_VALUE}, level="WARNING") - def save(self, save_parameters): + async def save(self, save_parameters): raise NotImplementedError - def check_load_in_parts(self): + async def check_load_in_parts(self): return False diff --git a/core/repositories/classifier_repository.py b/core/repositories/classifier_repository.py index 135ebc14..f5a15012 100644 --- a/core/repositories/classifier_repository.py +++ b/core/repositories/classifier_repository.py @@ -60,7 +60,7 @@ def _check_classifier_config(self, classifier_key: str, classifier_params: Dict[ except KeyError: raise Exception(f"Missing field: '{req_param}' for classifier {classifier_key} in classifiers.json") - def load(self) -> None: + async def load(self) -> None: if not self._folder_repository: return None @@ -136,8 +136,8 @@ def load(self) -> None: ) classifiers_dict[classifier_key] = SkipClassifier.get_nothing() - super(ClassifierRepository, self).fill(classifiers_dict) + await super(ClassifierRepository, self).fill(classifiers_dict) classifiers_initial_launch(classifiers_dict) - def save(self, save_parameters: Any) -> None: + async def save(self, save_parameters: Any) -> None: pass diff --git a/core/repositories/csv_repository.py b/core/repositories/csv_repository.py index 7af241ea..afec8142 100644 --- a/core/repositories/csv_repository.py +++ b/core/repositories/csv_repository.py @@ -7,9 +7,9 @@ def __init__(self, filename, source=None, *args, **kwargs): super(CSVRepository, self).__init__(source=source, *args, **kwargs) self.filename = filename - def load(self): + async def load(self): with self.source.open(self.filename, newline='') as stream: reader = csv.DictReader(stream) data = list(reader) - self.fill(data) - super(CSVRepository, self).load() + await self.fill(data) + await super(CSVRepository, self).load() diff --git a/core/repositories/dill_repository.py b/core/repositories/dill_repository.py index 36596d2d..9218139e 100644 --- a/core/repositories/dill_repository.py +++ b/core/repositories/dill_repository.py @@ -19,13 +19,13 @@ def __init__(self, filename, source=None, required=True, *args, **kwargs): self.filename = filename self.required = required - def load(self): + async def load(self): dill._dill._reverse_typemap['ClassType'] = type try: with self.source.open(self.filename, 'rb') as stream: data = dill.load(stream) - self.fill(data) + await self.fill(data) except FileNotFoundError as error: params = { 'error': str(error), @@ -34,4 +34,4 @@ def load(self): log('DillRepository.load loading failed. Error %(error)s', params=params, level='WARNING') if self.required: raise - super(DillRepository, self).load() + await super(DillRepository, self).load() diff --git a/core/repositories/file_repository.py b/core/repositories/file_repository.py index ae72107a..5e20fc7d 100644 --- a/core/repositories/file_repository.py +++ b/core/repositories/file_repository.py @@ -23,7 +23,7 @@ async def load(self): with await self.source.open(self.filename, 'rb') as stream: binary_data = stream.read() data = binary_data.decode() - self.fill(self.loader(data)) + await self.fill(self.loader(data)) else: self._file_exist = False params = { @@ -32,7 +32,7 @@ async def load(self): } log("FileRepository.load loading failed with file %(error_repository_path)s", params=params, level="WARNING") - super(FileRepository, self).load() + await super(FileRepository, self).load() def save(self, save_parameters): with self.source.open(self.save_target, 'wb') as stream: @@ -60,8 +60,8 @@ def data(self): self.load() return self._data - def load(self): - super().load() + async def load(self): + await super().load() if self._file_exist: self._last_mtime = self.source.mtime(self.filename) self._last_update_time = time.time() diff --git a/core/repositories/folder_repository.py b/core/repositories/folder_repository.py index 205f8530..9c203493 100644 --- a/core/repositories/folder_repository.py +++ b/core/repositories/folder_repository.py @@ -33,14 +33,14 @@ def _form_file_upload_map(self, shard_desc): filename_to_data.update({shard: shard_data}) return filename_to_data - def load(self): - shard_desc = self.get_shard_desc() - self.fill(self._form_file_upload_map(shard_desc)) - super(FolderRepository, self).load() + async def load(self): + shard_desc = await self.get_shard_desc() + await self.fill(self._form_file_upload_map(shard_desc)) + await super(FolderRepository, self).load() - def load_in_parts(self, count): - self.clear() - shard_desc = self.get_shard_desc() + async def load_in_parts(self, count): + await self.clear() + shard_desc = await self.get_shard_desc() for i in range(0, len(shard_desc), count): desc_slice = shard_desc[i: i + count] self.fill_on_top(self._form_file_upload_map(desc_slice)) @@ -48,10 +48,10 @@ def load_in_parts(self, count): params={"current_count": i + len(desc_slice), "all_count": len(shard_desc)}, level="WARNING") - super(FolderRepository, self).load() + await super(FolderRepository, self).load() - def get_shard_desc(self): - shard_desc = self.source.list_dir(self.path) + async def get_shard_desc(self): + shard_desc = await self.source.list_dir(self.path) if len(shard_desc) == 0: params = { "error_repository_path": self.path, diff --git a/core/repositories/items_repository.py b/core/repositories/items_repository.py index 7a7346aa..95f286a0 100644 --- a/core/repositories/items_repository.py +++ b/core/repositories/items_repository.py @@ -9,14 +9,14 @@ def __init__(self, *args, **kwargs): self.data = dict() @BaseRepository.data.setter - def data(self, value): + async def data(self, value): if value is None: self._data = dict() else: self._data = value - def load(self): - super(ItemsRepository, self).load() + async def load(self): + await super(ItemsRepository, self).load() def __iter__(self): return iter(self.data) diff --git a/core/repositories/shard_repository.py b/core/repositories/shard_repository.py index b6ad62bf..2ef8e8fa 100644 --- a/core/repositories/shard_repository.py +++ b/core/repositories/shard_repository.py @@ -30,10 +30,10 @@ def _get_data_type(data): f" data type {t}") return t - def load(self): - super(ShardRepository, self).load() + async def load(self): + await super(ShardRepository, self).load() - def fill(self, data): + async def fill(self, data): res = None t = self._get_data_type(data) if t == dict: @@ -54,5 +54,5 @@ def fill_on_top(self, data): elif t == list: self.data.extend(v) - def save(self, save_parameters): + async def save(self, save_parameters): raise NotImplementedError diff --git a/core/utils/rerunable.py b/core/utils/rerunable.py index 093f6c03..2f939130 100644 --- a/core/utils/rerunable.py +++ b/core/utils/rerunable.py @@ -1,3 +1,5 @@ +import asyncio + import core.logging.logger_constants as log_const from core.logging.logger_utils import log from core.monitoring.monitoring import monitoring @@ -11,24 +13,27 @@ def __init__(self, config=None): self.try_count = self.config.get("try_count") or self.DEFAULT_RERUNABLE_TRY_COUNT @property - def _handled_exception(self): + async def _handled_exception(self): raise NotImplementedError - def _on_prepare(self): + async def _on_prepare(self): raise NotImplementedError - def _on_all_tries_fail(self): + async def _on_all_tries_fail(self): raise NotImplementedError async def _run(self, action, *args, _try_count=None, **kwargs): if _try_count is None: _try_count = self.try_count if _try_count <= 0: - self._on_all_tries_fail() + await self._on_all_tries_fail() _try_count = _try_count - 1 try: - result = await action(*args, **kwargs) - except self._handled_exception as e: + if asyncio.iscoroutinefunction(action): + result = await action(*args, **kwargs) + else: + result = action(*args, **kwargs) + except Exception as e: params = { "class_name": str(self.__class__), "exception": str(e), @@ -45,5 +50,5 @@ async def _run(self, action, *args, _try_count=None, **kwargs): monitoring.got_counter(f"{counter_name}_exception") return result - def _get_counter_name(self): + async def _get_counter_name(self): return diff --git a/tests/core_tests/repositories_test/test_repo.py b/tests/core_tests/repositories_test/test_repo.py index 9d46eb23..3638b476 100644 --- a/tests/core_tests/repositories_test/test_repo.py +++ b/tests/core_tests/repositories_test/test_repo.py @@ -22,9 +22,9 @@ class MockBaseRepository(BaseRepository): def __init__(self): super(MockBaseRepository, self).__init__(MockDescriptionItem) - def load(self): - self.fill({"test": {"value": 1}}) - super(MockBaseRepository, self).load() + async def load(self): + await self.fill({"test": {"value": 1}}) + await super(MockBaseRepository, self).load() class MockShardRepository(ShardRepository): From e5094a03daab446a97ea3d463645013b797835c3 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Thu, 18 Nov 2021 08:34:30 +0300 Subject: [PATCH 069/116] fixes of repos --- core/db_adapter/memory_adapter.py | 18 +++++++++--------- core/mq/kafka/kafka_consumer.py | 3 ++- core/repositories/folder_repository.py | 10 +++++----- core/repositories/items_repository.py | 2 +- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/core/db_adapter/memory_adapter.py b/core/db_adapter/memory_adapter.py index 5491b7cc..1466babb 100644 --- a/core/db_adapter/memory_adapter.py +++ b/core/db_adapter/memory_adapter.py @@ -8,34 +8,34 @@ def __init__(self, config=None): super(DBAdapter, self).__init__(config) self.memory_storage = {} - def _glob(self, path, pattern): + async def _glob(self, path, pattern): raise error.NotSupportedOperation - def _path_exists(self, path): + async def _path_exists(self, path): raise error.NotSupportedOperation - def _on_prepare(self): + async def _on_prepare(self): pass - def connect(self): + async def connect(self): pass - def _open(self, filename, *args, **kwargs): + async def _open(self, filename, *args, **kwargs): pass - def _save(self, id, data): + async def _save(self, id, data): self.memory_storage[id] = data - def _replace_if_equals(self, id, sample, data): + async def _replace_if_equals(self, id, sample, data): stored_data = self.memory_storage.get(id) if stored_data == sample: self.memory_storage[id] = data return True return False - def _get(self, id): + async def _get(self, id): data = self.memory_storage.get(id) return data - def _list_dir(self, path): + async def _list_dir(self, path): pass diff --git a/core/mq/kafka/kafka_consumer.py b/core/mq/kafka/kafka_consumer.py index f9d5b4f1..02863af3 100644 --- a/core/mq/kafka/kafka_consumer.py +++ b/core/mq/kafka/kafka_consumer.py @@ -62,7 +62,8 @@ def subscribe(self, topics=None): topics = topics or list(self._config["topics"].values()) self._consumer.subscribe(topics, - on_assign=self.get_on_assign_callback() if self.assign_offset_end else KafkaConsumer.on_assign_log) + on_assign=self.get_on_assign_callback() if self.assign_offset_end else + KafkaConsumer.on_assign_log) def get_on_assign_callback(self): if "cooperative" in self._config["conf"].get("partition.assignment.strategy", ""): diff --git a/core/repositories/folder_repository.py b/core/repositories/folder_repository.py index 9c203493..c052defb 100644 --- a/core/repositories/folder_repository.py +++ b/core/repositories/folder_repository.py @@ -8,9 +8,9 @@ class FolderRepository(ShardRepository): def __init__(self, path, loader, source=None, *args, **kwargs): super(FolderRepository, self).__init__(path, loader, source, *args, **kwargs) - def _load_item(self, name): + async def _load_item(self, name): try: - with self.source.open(name, mode='rb') as shard_stream: + with await self.source.open(name, mode='rb') as shard_stream: shard_binary_data = shard_stream.read() shard_data = shard_binary_data.decode() loaded_data = self.loader(shard_data) @@ -25,17 +25,17 @@ def _load_item(self, name): raise return loaded_data - def _form_file_upload_map(self, shard_desc): + async def _form_file_upload_map(self, shard_desc): filename_to_data = {} for shard in shard_desc: - shard_data = self._load_item(shard) + shard_data = await self._load_item(shard) if shard_data: filename_to_data.update({shard: shard_data}) return filename_to_data async def load(self): shard_desc = await self.get_shard_desc() - await self.fill(self._form_file_upload_map(shard_desc)) + await self.fill(await self._form_file_upload_map(shard_desc)) await super(FolderRepository, self).load() async def load_in_parts(self, count): diff --git a/core/repositories/items_repository.py b/core/repositories/items_repository.py index 95f286a0..ac3c87ef 100644 --- a/core/repositories/items_repository.py +++ b/core/repositories/items_repository.py @@ -9,7 +9,7 @@ def __init__(self, *args, **kwargs): self.data = dict() @BaseRepository.data.setter - async def data(self, value): + def data(self, value): if value is None: self._data = dict() else: From d6552d0d192ac6cfe90eb861ce78bc35f6237008 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Fri, 19 Nov 2021 10:29:14 +0300 Subject: [PATCH 070/116] fix logging kafka error --- core/mq/kafka/kafka_consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/mq/kafka/kafka_consumer.py b/core/mq/kafka/kafka_consumer.py index 02863af3..3e19fbf7 100644 --- a/core/mq/kafka/kafka_consumer.py +++ b/core/mq/kafka/kafka_consumer.py @@ -52,7 +52,7 @@ def on_assign_log(consumer, partitions): if p.error: log_level = "ERROR" params = { - "partitions": partitions, + "partitions": str(partitions), log_const.KEY_NAME: log_const.KAFKA_ON_ASSIGN_VALUE, "log_level": log_level } From 7ef0a44ed265c2cb99d0552f83147e5a6d4e4173 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Mon, 22 Nov 2021 09:34:27 +0300 Subject: [PATCH 071/116] async fixes --- core/basic_models/scenarios/base_scenario.py | 2 +- core/repositories/base_repository.py | 2 +- core/repositories/file_repository.py | 2 +- core/utils/exception_handlers.py | 3 ++- .../form_filling_scenario.py | 18 +++++++++--------- .../tree_scenario/tree_scenario.py | 6 +++--- scenarios/scenario_models/field/field.py | 2 +- smart_kit/configs/settings.py | 2 +- 8 files changed, 19 insertions(+), 18 deletions(-) diff --git a/core/basic_models/scenarios/base_scenario.py b/core/basic_models/scenarios/base_scenario.py index 5e0965ce..33bfa5c3 100644 --- a/core/basic_models/scenarios/base_scenario.py +++ b/core/basic_models/scenarios/base_scenario.py @@ -97,4 +97,4 @@ def history(self): return {"scenario_path": [{"scenario": self.id, "node": None}]} async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = None): - return self.get_action_results(user, text_preprocessing_result, self.actions, params) + return await self.get_action_results(user, text_preprocessing_result, self.actions, params) diff --git a/core/repositories/base_repository.py b/core/repositories/base_repository.py index 6badfae1..54f15918 100644 --- a/core/repositories/base_repository.py +++ b/core/repositories/base_repository.py @@ -16,7 +16,7 @@ def data(self): return self._data @data.setter - async def data(self, value): + def data(self, value): self._data = value async def load(self): diff --git a/core/repositories/file_repository.py b/core/repositories/file_repository.py index 5e20fc7d..020d7553 100644 --- a/core/repositories/file_repository.py +++ b/core/repositories/file_repository.py @@ -18,7 +18,7 @@ def __init__(self, filename, loader, source=None, save_target=None, saver=None, self._file_exist = False async def load(self): - if self.source.path_exists(self.filename): + if await self.source.path_exists(self.filename): self._file_exist = True with await self.source.open(self.filename, 'rb') as stream: binary_data = stream.read() diff --git a/core/utils/exception_handlers.py b/core/utils/exception_handlers.py index 375877fc..4a0c58e5 100644 --- a/core/utils/exception_handlers.py +++ b/core/utils/exception_handlers.py @@ -17,7 +17,8 @@ async def _wrapper(obj, *args, **kwarg): try: on_error = getattr(obj, on_error_obj_method_name) if \ on_error_obj_method_name else (lambda *x: None) - result = on_error(*args, **kwarg) + result = on_error(*args, **kwarg) if not asyncio.iscoroutinefunction(on_error) else \ + await on_error(*args, **kwarg) except: print(sys.exc_info()) return result diff --git a/scenarios/scenario_descriptions/form_filling_scenario.py b/scenarios/scenario_descriptions/form_filling_scenario.py index 00808403..9975d6d3 100644 --- a/scenarios/scenario_descriptions/form_filling_scenario.py +++ b/scenarios/scenario_descriptions/form_filling_scenario.py @@ -126,11 +126,11 @@ async def _validate_extracted_data(self, user, text_preprocessing_result, form, message = "Field is not valid: %(field_key)s" log(message, user, log_params) actions = field.field_validator.actions - error_msgs = self.get_action_results(user, text_preprocessing_result, actions) + error_msgs = await self.get_action_results(user, text_preprocessing_result, actions) break return error_msgs - def _fill_form(self, user, text_preprocessing_result, form, data_extracted): + async def _fill_form(self, user, text_preprocessing_result, form, data_extracted): on_filled_actions = [] fields = form.fields scenario_model = user.scenario_models[self.id] @@ -140,15 +140,15 @@ def _fill_form(self, user, text_preprocessing_result, form, data_extracted): value = data_extracted.get(key) field = fields[key] if field.fill(value): - _action = self.get_action_results(user=user, text_preprocessing_result=text_preprocessing_result, - actions=field.description.on_filled_actions) + _action = await self.get_action_results(user=user, text_preprocessing_result=text_preprocessing_result, + actions=field.description.on_filled_actions) on_filled_actions.extend(_action) if scenario_model.break_scenario: is_break = True return _action, is_break return on_filled_actions, is_break - def get_reply(self, user, text_preprocessing_result, reply_actions, field, form): + async def get_reply(self, user, text_preprocessing_result, reply_actions, field, form): action_params = {} if field: field.set_available() @@ -160,7 +160,7 @@ def get_reply(self, user, text_preprocessing_result, reply_actions, field, form) message = "Ask question on field: %(field)s" log(message, user, params) action_params[REQUEST_FIELD] = {"type": field.description.type, "id": field.description.id} - action_messages = self.get_action_results(user, text_preprocessing_result, actions, action_params) + action_messages = await self.get_action_results(user, text_preprocessing_result, actions, action_params) else: actions = reply_actions params = { @@ -170,7 +170,7 @@ def get_reply(self, user, text_preprocessing_result, reply_actions, field, form) message = "Finished scenario: %(id)s" log(message, user, params) user.preprocessing_messages_for_scenarios.clear() - action_messages = self.get_action_results(user, text_preprocessing_result, actions, action_params) + action_messages = await self.get_action_results(user, text_preprocessing_result, actions, action_params) user.last_scenarios.delete(self.id) return action_messages @@ -190,7 +190,7 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No if validation_error_msg: reply_messages = validation_error_msg else: - reply_messages, is_break = self._fill_form(user, text_preprocessing_result, form, data_extracted) + reply_messages, is_break = await self._fill_form(user, text_preprocessing_result, form, data_extracted) if not is_break: field = await self._field(form, text_preprocessing_result, user, params) if field: @@ -199,7 +199,7 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No scenario=self.root_id, content={HistoryConstants.content_fields.FIELD: field.description.id}, results=HistoryConstants.event_results.ASK_QUESTION)) - reply = self.get_reply(user, text_preprocessing_result, self.actions, field, form) + reply = await self.get_reply(user, text_preprocessing_result, self.actions, field, form) reply_messages.extend(reply) if not reply_messages: diff --git a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py index de573a97..e0a37607 100644 --- a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py +++ b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py @@ -131,8 +131,8 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No validation_error_msg = validation_error_msg or _validation_error_msg else: data_extracted.update(field_data) - on_filled_node_actions, is_break = self._fill_form(user, text_preprocessing_result, - internal_form, data_extracted) + on_filled_node_actions, is_break = await self._fill_form(user, text_preprocessing_result, + internal_form, data_extracted) if is_break: return on_filled_node_actions on_filled_actions.extend(on_filled_node_actions) @@ -170,7 +170,7 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No content={HistoryConstants.content_fields.FIELD: field.description.id}, results=HistoryConstants.event_results.ASK_QUESTION) user.history.add_event(event) - _command = self.get_reply(user, text_preprocessing_result, current_node.actions, field, main_form) + _command = await self.get_reply(user, text_preprocessing_result, current_node.actions, field, main_form) reply_commands.extend(_command) if not reply_commands: diff --git a/scenarios/scenario_models/field/field.py b/scenarios/scenario_models/field/field.py index 2d2b048e..d9de4ddc 100644 --- a/scenarios/scenario_models/field/field.py +++ b/scenarios/scenario_models/field/field.py @@ -41,7 +41,7 @@ def can_be_updated(self): async def check_can_be_filled(self, text_preprocessing_result, user): check, run = await asyncio.gather( self.description.requirement.check(text_preprocessing_result, user), - await self.description.filler.run(user, text_preprocessing_result)) + self.description.filler.run(user, text_preprocessing_result)) return check and run is not None @property diff --git a/smart_kit/configs/settings.py b/smart_kit/configs/settings.py index ddc18cc1..bd96c68e 100644 --- a/smart_kit/configs/settings.py +++ b/smart_kit/configs/settings.py @@ -60,6 +60,6 @@ def get_source(self): adapter_settings = self.registered_repositories[ adapter_key].data if adapter_key != Settings.OSAdapterKey else None adapter = self.adapters[adapter_key](adapter_settings) - adapter.connect() + self.loop.run_until_complete(adapter.connect()) source = adapter.source return source From c054e91b8b60661afdc0f4e9e85bf98f5d7eb2ff Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Tue, 23 Nov 2021 11:13:44 +0300 Subject: [PATCH 072/116] update async main loop kafka --- smart_kit/start_points/main_loop_kafka.py | 205 +++++++++++++--------- 1 file changed, 121 insertions(+), 84 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 86be1e51..87d8bf30 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -1,6 +1,9 @@ # coding=utf-8 import asyncio +import cProfile import json +import pstats +import signal import time import sys from collections import namedtuple @@ -72,14 +75,11 @@ def __init__(self, *args, **kwargs): for key in self.consumers: self.consumers[key].subscribe() self.publishers = publishers + self.concurrent_messages = 0 - # is needed? start # self.behaviors_timeouts_value_cls = namedtuple('behaviors_timeouts_value', 'db_uid, callback_id, mq_message, kafka_key') self.behaviors_timeouts = HeapqKV(value_to_key_func=lambda val: val.callback_id) - # is needed? end # - - self.concurrent_messages = 0 log("%(class_name)s.__init__ completed.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) @@ -94,16 +94,20 @@ async def pre_handle(self): await self.iterate_behavior_timeouts() def run(self): + signal.signal(signal.SIGINT, self.stop) + signal.signal(signal.SIGTERM, self.stop) log("%(class_name)s.run started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) try: - self.loop.run_until_complete(self.main_coro()) + self.loop.run_until_complete(self.general_coro()) except (SystemExit,): + self.loop.stop() + log("MainLoop stopped") log("%(class_name)s.run stopped", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) - async def main_coro(self): - tasks = [self.main_work(kafka_key) for kafka_key in self.consumers] + async def general_coro(self): + tasks = [self.process_consumer(kafka_key) for kafka_key in self.consumers] if self.health_check_server is not None: tasks.append(self.healthcheck_coro()) await asyncio.gather(*tasks) @@ -115,65 +119,96 @@ async def healthcheck_coro(self): self.health_check_server_future = self.loop.run_in_executor(None, self.health_check_server.iterate) await asyncio.sleep(0.5) - async def main_work(self, kafka_key): + async def process_consumer(self, kafka_key): consumer = self.consumers[kafka_key] - message_value = None - + loop = asyncio.get_event_loop() max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 10) + total_messages = 0 + + # in DP is absent start max_concurrent_messages_delay = self.settings["template_settings"].get("max_concurrent_messages_delay", 0.1) - last_poll_begin_time = time.time() - while self.is_work: - from_last_poll_begin_ms = int((time.time() - last_poll_begin_time) * 1000) - stats = "From last poll time: {} msecs\n".format(from_last_poll_begin_ms) - log_params = { - log_const.KEY_NAME: "timings", - "from_last_poll_begin_ms": from_last_poll_begin_ms - } + # in DP is absent end + + async def msg_loop(iteration): + nonlocal total_messages + message_value = None last_poll_begin_time = time.time() - if self.concurrent_messages >= max_concurrent_messages: - log(f"main_loop.main_work: max {max_concurrent_messages} concurrent messages occured", - params={log_const.KEY_NAME: "max_concurrent_messages"}, level='WARNING') - await asyncio.sleep(max_concurrent_messages_delay) - total_delay = log_params.get("waiting_max_concurrent_messages", 0) - total_delay += max_concurrent_messages_delay - log_params["waiting_max_concurrent_messages"] = total_delay - continue - try: - self.concurrent_messages += 1 - mq_message = None - with StatsTimer() as poll_timer: - # Max delay between polls configured in consumer.poll_timeout param - mq_message = await self.loop.run_in_executor(None, consumer.poll) - if mq_message: - headers = mq_message.headers() - if headers is None: - raise Exception("No incoming message headers found.") - stats += "Polling time: {} msecs\n".format(poll_timer.msecs) - log_params["kafka_polling"] = poll_timer.msecs - message_value = json.loads(mq_message.value()) - await self.process_message(mq_message, consumer, kafka_key, stats, log_params) - - except KafkaException as kafka_exp: - self.concurrent_messages -= 1 - log("kafka error: %(kafka_exp)s.", - params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "kafka_exp": str(kafka_exp), - log_const.REQUEST_VALUE: str(message_value)}, - level="ERROR", exc_info=True) - except Exception: - self.concurrent_messages -= 1 - log("%(class_name)s iterate error. Kafka key %(kafka_key)s", - params={log_const.KEY_NAME: "worker_exception", - "kafka_key": kafka_key, - log_const.REQUEST_VALUE: str(message_value)}, - level="ERROR", exc_info=True) + print(f"\n-- Starting {iteration} iter\n") + + while self.is_work: + from_last_poll_begin_ms = int((time.time() - last_poll_begin_time) * 1000) + stats = "From last poll time: {} msecs\n".format(from_last_poll_begin_ms) + log_params = { + log_const.KEY_NAME: "timings", + "from_last_poll_begin_ms": from_last_poll_begin_ms + } + last_poll_begin_time = time.time() + + # in DP is absent start + if self.concurrent_messages >= max_concurrent_messages: + log(f"main_loop.main_work: max {max_concurrent_messages} concurrent messages occured", + params={log_const.KEY_NAME: "max_concurrent_messages"}, level='WARNING') + await asyncio.sleep(max_concurrent_messages_delay) + total_delay = log_params.get("waiting_max_concurrent_messages", 0) + total_delay += max_concurrent_messages_delay + log_params["waiting_max_concurrent_messages"] = total_delay + continue + # in DP is absent end + try: - consumer.commit_offset(mq_message) + self.concurrent_messages += 1 + mq_message = None + with StatsTimer() as poll_timer: + # Max delay between polls configured in consumer.poll_timeout param + mq_message = await loop.run_in_executor(None, consumer.poll) + if mq_message: + print(f"\n-- Processing {self.concurrent_messages} msgs at {iteration} iter\n") + total_messages += 1 + headers = mq_message.headers() + if headers is None: + raise Exception("No incoming message headers found.") + stats += "Polling time: {} msecs\n".format(poll_timer.msecs) + log_params["kafka_polling"] = poll_timer.msecs + message_value = json.loads(mq_message.value()) + await self.process_message(mq_message, consumer, kafka_key, stats, log_params) + + except KafkaException as kafka_exp: + self.concurrent_messages -= 1 + log("kafka error: %(kafka_exp)s.", + params={log_const.KEY_NAME: log_const.STARTUP_VALUE, + "kafka_exp": str(kafka_exp), + log_const.REQUEST_VALUE: str(message_value)}, + level="ERROR", exc_info=True) + except Exception: - log("Error handling worker fail exception.", level="ERROR", exc_info=True) - raise - else: - self.concurrent_messages -= 1 + self.concurrent_messages -= 1 + log("%(class_name)s iterate error. Kafka key %(kafka_key)s", + params={log_const.KEY_NAME: "worker_exception", + "kafka_key": kafka_key, + log_const.REQUEST_VALUE: str(message_value)}, + level="ERROR", exc_info=True) + try: + consumer.commit_offset(mq_message) + except Exception: + log("Error handling worker fail exception.", level="ERROR", exc_info=True) + raise + else: + self.concurrent_messages -= 1 + + print(f"-- Process Consumer enter with {max_concurrent_messages} loops") + start_time = time.time() + pr = cProfile.Profile() + pr.enable() + + await asyncio.gather(*(msg_loop(i) for i in range(max_concurrent_messages))) + + time_delta = time.time() - start_time + print(f"-- Process Consumer exit: {total_messages} msg in {int(time_delta)} sec") + pr.disable() + stats = pstats.Stats(pr) + stats.sort_stats(pstats.SortKey.TIME) + stats.print_stats(10) + stats.dump_stats(filename="dp.prof.log") def _generate_answers(self, user, commands, message, **kwargs): topic_key = kwargs["topic_key"] @@ -273,10 +308,10 @@ def _get_topic_key(self, mq_message, kafka_key): return self.default_topic_key(kafka_key) or topic_names_2_key[mq_message.topic()] async def process_message(self, mq_message, consumer, kafka_key, stats, log_params): + user = None topic_key = self._get_topic_key(mq_message, kafka_key) save_tries = 0 user_save_ok = False - user = None db_uid = None validation_failed = False while save_tries < self.user_save_collisions_tries and not user_save_ok: @@ -297,27 +332,13 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para stats += "Mid: {}\n".format(message.incremental_id) log_params[MESSAGE_ID_STR] = message.incremental_id + smart_kit_metrics.sampling_mq_waiting_time(self.app_name, waiting_message_time / 1000) if self._is_message_timeout_to_skip(message, waiting_message_time): skip_timeout = True break - self.check_message_key(message, mq_message.key(), user) - log("INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(incoming_data)s", - params={log_const.KEY_NAME: "incoming_message", - "topic": mq_message.topic(), - "message_partition": mq_message.partition(), - "message_key": mq_message.key(), - "kafka_key": kafka_key, - "incoming_data": str(message.masked_value), - "headers": str(mq_message.headers()), - "waiting_message": waiting_message_time, - "surface": message.device.surface, - MESSAGE_ID_STR: message.incremental_id}, - user=user - ) - db_uid = message.db_uid span = jaeger_utils.get_incoming_spam(self.tracer, message, mq_message) @@ -326,17 +347,35 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para with self.tracer.start_span('Loading time', child_of=scope.span) as span: with StatsTimer() as load_timer: user = await self.load_user(db_uid, message) + self.check_message_key(message, mq_message.key(), user) + stats += "Loading time from DB time: {} msecs\n".format(load_timer.msecs) + log_params["user_loading"] = load_timer.msecs + smart_kit_metrics.sampling_load_time(self.app_name, load_timer.secs) + + log( + "INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(" + "incoming_data)s", + params={log_const.KEY_NAME: "incoming_message", + "topic": mq_message.topic(), + "message_partition": mq_message.partition(), + "message_key": mq_message.key(), + "kafka_key": kafka_key, + "incoming_data": str(message.masked_value), + "headers": str(mq_message.headers()), + "waiting_message": waiting_message_time, + "surface": message.device.surface, + MESSAGE_ID_STR: message.incremental_id}, + user=user + ) - with self.tracer.start_span('Loading time', child_of=scope.span) as span: - smart_kit_metrics.sampling_load_time(self.app_name, load_timer.secs) - stats += "Loading time: {} msecs\n".format(load_timer.msecs) + with self.tracer.start_span('Script time', child_of=scope.span) as span: with StatsTimer() as script_timer: commands = await self.model.answer(message, user) - with self.tracer.start_span('Script time', child_of=scope.span) as span: answers = self._generate_answers(user=user, commands=commands, message=message, topic_key=topic_key, kafka_key=kafka_key) + stats += "Script time: {} msecs\n".format(script_timer.msecs) log_params["script_time"] = script_timer.msecs smart_kit_metrics.sampling_script_time(self.app_name, script_timer.secs) @@ -364,8 +403,6 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para if answers: self.save_behavior_timeouts(user, mq_message, kafka_key) - if mq_message.headers() is None: - mq_message.set_headers([]) self.tracer.inject(span_context=span.context, format=jaeger_kafka_codec.KAFKA_MAP, carrier=mq_message.headers()) @@ -374,7 +411,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para with StatsTimer() as publish_timer: self._send_request(user, answer, mq_message) smart_kit_metrics.counter_outgoing(self.app_name, answer.command.name, answer, user) - stats += "Publishing time: {} msecs".format(publish_timer.msecs) + stats += "Publishing to Kafka time: {} msecs".format(publish_timer.msecs) log_params["kafka_publishing"] = publish_timer.msecs log(stats) else: @@ -391,7 +428,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para if stats: log(stats, user=user, params=log_params) - if user and not user_save_ok and not validation_failed: + if user and not user_save_ok and not validation_failed and not skip_timeout: log("MainLoop.iterate: db_save collision all tries left on uid %(uid)s db_version %(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", From f0f7b89a74be25d843bedc45f83a2bbafa26784c Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 24 Nov 2021 05:23:31 +0300 Subject: [PATCH 073/116] add some awaits --- core/db_adapter/aioredis_sentinel_adapter.py | 1 - core/db_adapter/ceph/ceph_io.py | 4 +-- core/db_adapter/ignite_adapter.py | 2 +- core/repositories/file_repository.py | 15 ++++---- core/utils/rerunable.py | 4 +-- .../start_points/main_loop_async_http.py | 4 +-- tests/core_tests/test_utils/test_rerunable.py | 35 ++++++++++--------- 7 files changed, 33 insertions(+), 32 deletions(-) diff --git a/core/db_adapter/aioredis_sentinel_adapter.py b/core/db_adapter/aioredis_sentinel_adapter.py index 8379e888..27ab4589 100644 --- a/core/db_adapter/aioredis_sentinel_adapter.py +++ b/core/db_adapter/aioredis_sentinel_adapter.py @@ -28,7 +28,6 @@ def __init__(self, config=None): async def save(self, id, data): return await self._run(self._save, id, data) - @monitoring.got_histogram("save_time") async def replace_if_equals(self, id, sample, data): return await self._run(self._replace_if_equals, id, sample, data) diff --git a/core/db_adapter/ceph/ceph_io.py b/core/db_adapter/ceph/ceph_io.py index a42c8665..7d5fb79f 100644 --- a/core/db_adapter/ceph/ceph_io.py +++ b/core/db_adapter/ceph/ceph_io.py @@ -25,9 +25,9 @@ def _get_bucket_keys(self): return self.bucket.get_key(self.filename) def __enter__(self): - key = self._run(self._get_bucket_keys) + key = await self._run(self._get_bucket_keys) if key: - data = self._run(key.get_contents_as_string) + data = await self._run(key.get_contents_as_string) io_stream = None if self.mode == "r": io_stream = io.StringIO(data.decode("utf-8")) diff --git a/core/db_adapter/ignite_adapter.py b/core/db_adapter/ignite_adapter.py index 7619caee..987de927 100644 --- a/core/db_adapter/ignite_adapter.py +++ b/core/db_adapter/ignite_adapter.py @@ -106,7 +106,7 @@ async def _async_run(self, action, *args, _try_count=None, **kwargs): if _try_count is None: _try_count = self.try_count if _try_count <= 0: - self._on_all_tries_fail() + await self._on_all_tries_fail() _try_count = _try_count - 1 try: result = await action(*args, **kwargs) diff --git a/core/repositories/file_repository.py b/core/repositories/file_repository.py index 020d7553..7c70ad18 100644 --- a/core/repositories/file_repository.py +++ b/core/repositories/file_repository.py @@ -34,7 +34,7 @@ async def load(self): params=params, level="WARNING") await super(FileRepository, self).load() - def save(self, save_parameters): + async def save(self, save_parameters): with self.source.open(self.save_target, 'wb') as stream: stream.write(self.saver(self.data, **save_parameters).encode()) @@ -49,21 +49,22 @@ def __init__(self, *args, update_cooldown=5, **kwargs): raise Exception(f"{self.__class__.__name__} support only OSAdapter") @FileRepository.data.getter - def data(self): - if self._is_outdated: + # TODO: добавить await везде, где используется + async def data(self): + if await self._is_outdated: params = { "filename": self.filename } log("FileRepository.data %(filename)s is outdated. Data will be reloaded.", params=params, level="INFO") - self.load() + await self.load() return self._data async def load(self): await super().load() if self._file_exist: - self._last_mtime = self.source.mtime(self.filename) + self._last_mtime = await self.source.mtime(self.filename) self._last_update_time = time.time() @property @@ -71,7 +72,7 @@ def expired(self): return self._last_update_time + self.update_cooldown < time.time() @property - def _is_outdated(self): + async def _is_outdated(self): if self._file_exist and self.expired: - return self.source.mtime(self.filename) > self._last_mtime + return await self.source.mtime(self.filename) > self._last_mtime return False diff --git a/core/utils/rerunable.py b/core/utils/rerunable.py index 2f939130..6cef4c2f 100644 --- a/core/utils/rerunable.py +++ b/core/utils/rerunable.py @@ -43,9 +43,9 @@ async def _run(self, action, *args, _try_count=None, **kwargs): log("%(class_name)s run failed with %(exception)s.\n Got %(try_count)s tries left.", params=params, level="ERROR") - self._on_prepare() + await self._on_prepare() result = await self._run(action, *args, _try_count=_try_count, **kwargs) - counter_name = self._get_counter_name() + counter_name = await self._get_counter_name() if counter_name: monitoring.got_counter(f"{counter_name}_exception") return result diff --git a/smart_kit/start_points/main_loop_async_http.py b/smart_kit/start_points/main_loop_async_http.py index e893b9ea..2184d2c9 100644 --- a/smart_kit/start_points/main_loop_async_http.py +++ b/smart_kit/start_points/main_loop_async_http.py @@ -86,13 +86,13 @@ async def save_user(self, db_uid, user, message): await self.db_adapter.save(db_uid, str_data) else: if user.initial_db_data and self.user_save_check_for_collisions: - no_collisions = self.db_adapter.replace_if_equals( + no_collisions = await self.db_adapter.replace_if_equals( db_uid, sample=user.initial_db_data, data=str_data ) else: - self.db_adapter.save(db_uid, str_data) + await self.db_adapter.save(db_uid, str_data) except (DBAdapterException, ValueError): log("Failed to set user data", params={log_const.KEY_NAME: log_const.FAILED_DB_INTERACTION, log_const.REQUEST_VALUE: str(message.value)}, level="ERROR") diff --git a/tests/core_tests/test_utils/test_rerunable.py b/tests/core_tests/test_utils/test_rerunable.py index e3b64f0b..abddce2e 100644 --- a/tests/core_tests/test_utils/test_rerunable.py +++ b/tests/core_tests/test_utils/test_rerunable.py @@ -1,4 +1,4 @@ -from unittest import TestCase +from unittest import IsolatedAsyncioTestCase from unittest.mock import Mock from core.utils.rerunable import Rerunable @@ -22,20 +22,20 @@ def __init__(self, try_count): self._on_prepare_mock = Mock() @property - def _handled_exception(self): + async def _handled_exception(self): return HandledException - def _on_prepare(self): + async def _on_prepare(self): self._on_prepare_mock() - def _on_all_tries_fail(self): + async def _on_all_tries_fail(self): raise AllFailedException - def run(self, *args, **kwargs): - return self._run(*args, **kwargs) + async def run(self, *args, **kwargs): + return await self._run(*args, **kwargs) -class TestRerunable(TestCase): +class TestRerunable(IsolatedAsyncioTestCase): def setUp(self): self.try_count = 2 self.rerunable = RerunableOne(self.try_count) @@ -43,35 +43,36 @@ def setUp(self): self.param = Mock() self.param1 = {"param1_name": Mock()} - def test_pass(self): + async def test_pass(self): self.action = Mock() - self.rerunable.run(self.action, self.param, **self.param1) + await self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_once_with(self.param, **self.param1) - def test_wrong_exception(self): + # TODO: починить тест + async def test_wrong_exception(self): self.action = Mock(side_effect=CustomException()) with self.assertRaises(CustomException) as context: - self.rerunable.run(self.action, self.param, **self.param1) + await self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_once_with(self.param, **self.param1) - def test_all_retry_failed(self): + async def test_all_retry_failed(self): self.action = Mock(side_effect=HandledException()) with self.assertRaises(AllFailedException) as context: - self.rerunable.run(self.action, self.param, **self.param1) + await self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_with(self.param, **self.param1) self.assertEqual(self.action.call_count, self.try_count) - def test_first_try_failed(self): + async def test_first_try_failed(self): self.param = Mock() self.param1 = {"param1_name": Mock()} self.action = Mock(side_effect=[HandledException(), self.expected_value]) - result = self.rerunable.run(self.action, self.param, **self.param1) + result = await self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_with(self.param, **self.param1) self.assertEqual(self.action.call_count, self.try_count) self.assertEqual(result, self.expected_value) - def test_first_try_ok(self): + async def test_first_try_ok(self): self.action = Mock(return_value=self.expected_value) - result = self.rerunable.run(self.action, self.param, **self.param1) + result = await self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_once_with(self.param, **self.param1) self.assertEqual(result, self.expected_value) From a8958e3a21200016bcef5d8e5c78884a4569e0a2 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Fri, 26 Nov 2021 08:36:03 +0300 Subject: [PATCH 074/116] hotfix kafka bug main_loop_kafka --- smart_kit/start_points/main_loop_kafka.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 87d8bf30..633290fc 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -122,13 +122,9 @@ async def healthcheck_coro(self): async def process_consumer(self, kafka_key): consumer = self.consumers[kafka_key] loop = asyncio.get_event_loop() - max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 10) + max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 1) total_messages = 0 - # in DP is absent start - max_concurrent_messages_delay = self.settings["template_settings"].get("max_concurrent_messages_delay", 0.1) - # in DP is absent end - async def msg_loop(iteration): nonlocal total_messages message_value = None @@ -144,17 +140,6 @@ async def msg_loop(iteration): } last_poll_begin_time = time.time() - # in DP is absent start - if self.concurrent_messages >= max_concurrent_messages: - log(f"main_loop.main_work: max {max_concurrent_messages} concurrent messages occured", - params={log_const.KEY_NAME: "max_concurrent_messages"}, level='WARNING') - await asyncio.sleep(max_concurrent_messages_delay) - total_delay = log_params.get("waiting_max_concurrent_messages", 0) - total_delay += max_concurrent_messages_delay - log_params["waiting_max_concurrent_messages"] = total_delay - continue - # in DP is absent end - try: self.concurrent_messages += 1 mq_message = None From ecaeae0f3c8c3d78a779f9d37625eec32897d455 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Fri, 26 Nov 2021 12:57:48 +0300 Subject: [PATCH 075/116] add AsyncDBAdapter class --- core/configs/base_config.py | 8 +- core/db_adapter/aioredis_adapter.py | 20 +-- core/db_adapter/aioredis_sentinel_adapter.py | 21 +--- core/db_adapter/ceph/ceph_io.py | 4 +- core/db_adapter/db_adapter.py | 118 ++++++++++++++---- core/db_adapter/ignite_adapter.py | 42 +------ core/db_adapter/memory_adapter.py | 18 +-- core/db_adapter/os_adapter.py | 22 ++-- core/repositories/base_repository.py | 10 +- core/repositories/classifier_repository.py | 6 +- core/repositories/csv_repository.py | 6 +- core/repositories/dill_repository.py | 6 +- core/repositories/file_repository.py | 29 +++-- core/repositories/folder_repository.py | 28 ++--- core/repositories/items_repository.py | 4 +- core/repositories/shard_repository.py | 8 +- core/utils/rerunable.py | 23 ++-- smart_kit/start_points/base_main_loop.py | 4 + .../core_tests/repositories_test/test_repo.py | 6 +- tests/core_tests/test_utils/test_rerunable.py | 34 ++--- 20 files changed, 205 insertions(+), 212 deletions(-) diff --git a/core/configs/base_config.py b/core/configs/base_config.py index 82bb80c9..0f9a56dd 100644 --- a/core/configs/base_config.py +++ b/core/configs/base_config.py @@ -1,4 +1,3 @@ -import asyncio import os from typing import List @@ -11,7 +10,6 @@ def __init__(self, **kwargs): self.registered_repositories = Registered() self.repositories = [] self.source = kwargs.get("source") - self.loop = asyncio.get_event_loop() def __getitem__(self, key): return self.registered_repositories[key].data @@ -27,11 +25,11 @@ def subfolder_path(self, filename): return os.path.join(self._subfolder, filename) def init(self): - self.loop.run_until_complete(self.init_repositories()) + self.init_repositories() - async def init_repositories(self): + def init_repositories(self): for rep in self.repositories: - await rep.load() + rep.load() self.registered_repositories[rep.key] = rep def raw(self): diff --git a/core/db_adapter/aioredis_adapter.py b/core/db_adapter/aioredis_adapter.py index 1ab32779..b90451e0 100644 --- a/core/db_adapter/aioredis_adapter.py +++ b/core/db_adapter/aioredis_adapter.py @@ -3,15 +3,14 @@ import aioredis import typing -from core.db_adapter.db_adapter import DBAdapter +from core.db_adapter.db_adapter import AsyncDBAdapter from core.db_adapter import error from core.monitoring.monitoring import monitoring from core.logging.logger_utils import log -class AIORedisAdapter(DBAdapter): - IS_ASYNC = True +class AIORedisAdapter(AsyncDBAdapter): def __init__(self, config=None): super().__init__(config) @@ -22,21 +21,6 @@ def __init__(self, config=None): except KeyError: pass - @monitoring.got_histogram("save_time") - async def save(self, id, data): - return await self._run(self._save, id, data) - - @monitoring.got_histogram("save_time") - async def replace_if_equals(self, id, sample, data): - return await self._run(self._replace_if_equals, id, sample, data) - - @monitoring.got_histogram("get_time") - async def get(self, id): - return await self._run(self._get, id) - - async def path_exists(self, path): - return await self._run(self._path_exists, path) - async def connect(self): print("Here is the content of REDIS_CONFIG:", self.config) print("Connecting to a single redis server") diff --git a/core/db_adapter/aioredis_sentinel_adapter.py b/core/db_adapter/aioredis_sentinel_adapter.py index 27ab4589..16d8d140 100644 --- a/core/db_adapter/aioredis_sentinel_adapter.py +++ b/core/db_adapter/aioredis_sentinel_adapter.py @@ -1,17 +1,15 @@ import copy -import aioredis import typing from aioredis.sentinel import Sentinel -from core.db_adapter.db_adapter import DBAdapter +from core.db_adapter.db_adapter import AsyncDBAdapter from core.db_adapter import error from core.monitoring.monitoring import monitoring from core.logging.logger_utils import log -class AIORedisSentinelAdapter(DBAdapter): - IS_ASYNC = True +class AIORedisSentinelAdapter(AsyncDBAdapter): def __init__(self, config=None): super().__init__(config) @@ -24,21 +22,6 @@ def __init__(self, config=None): except KeyError: pass - @monitoring.got_histogram("save_time") - async def save(self, id, data): - return await self._run(self._save, id, data) - - @monitoring.got_histogram("save_time") - async def replace_if_equals(self, id, sample, data): - return await self._run(self._replace_if_equals, id, sample, data) - - @monitoring.got_histogram("get_time") - async def get(self, id): - return await self._run(self._get, id) - - async def path_exists(self, path): - return await self._run(self._path_exists, path) - async def connect(self): config = copy.deepcopy(self.config) diff --git a/core/db_adapter/ceph/ceph_io.py b/core/db_adapter/ceph/ceph_io.py index 7d5fb79f..a42c8665 100644 --- a/core/db_adapter/ceph/ceph_io.py +++ b/core/db_adapter/ceph/ceph_io.py @@ -25,9 +25,9 @@ def _get_bucket_keys(self): return self.bucket.get_key(self.filename) def __enter__(self): - key = await self._run(self._get_bucket_keys) + key = self._run(self._get_bucket_keys) if key: - data = await self._run(key.get_contents_as_string) + data = self._run(key.get_contents_as_string) io_stream = None if self.mode == "r": io_stream = io.StringIO(data.decode("utf-8")) diff --git a/core/db_adapter/db_adapter.py b/core/db_adapter/db_adapter.py index 0c151f8a..ab915bcb 100644 --- a/core/db_adapter/db_adapter.py +++ b/core/db_adapter/db_adapter.py @@ -1,4 +1,8 @@ # coding: utf-8 +import asyncio + +import core.logging.logger_constants as log_const +from core.logging.logger_utils import log from core.model.factory import build_factory from core.model.registered import Registered from core.monitoring.monitoring import monitoring @@ -13,52 +17,98 @@ class DBAdapterException(Exception): class DBAdapter(Rerunable): + IS_ASYNC = False + def __init__(self, config=None): super(DBAdapter, self).__init__(config) self._client = None - async def _on_prepare(self): + def _on_prepare(self): raise NotImplementedError - async def connect(self): + def connect(self): raise NotImplementedError - async def _open(self, filename, *args, **kwargs): + def _open(self, filename, *args, **kwargs): raise NotImplementedError - async def _save(self, id, data): + def _save(self, id, data): raise NotImplementedError - async def _replace_if_equals(self, id, sample, data): + def _replace_if_equals(self, id, sample, data): raise NotImplementedError - async def _get(self, id): + def _get(self, id): raise NotImplementedError - async def _list_dir(self, path): + def _list_dir(self, path): raise NotImplementedError - async def _glob(self, path, pattern): + def _glob(self, path, pattern): raise NotImplementedError - async def _path_exists(self, path): + def _path_exists(self, path): + raise NotImplementedError + + def _mtime(self, path): + raise NotImplementedError + + def open(self, filename, *args, **kwargs): + return self._run(self._open, filename, *args, **kwargs) + + def glob(self, path, pattern): + return self._run(self._glob, path, pattern) + + def path_exists(self, path): + return self._run(self._path_exists, path) + + def mtime(self, path): + return self._run(self._mtime, path) + + @monitoring.got_histogram("save_time") + def save(self, id, data): + return self._run(self._save, id, data) + + @monitoring.got_histogram("save_time") + def replace_if_equals(self, id, sample, data): + return self._run(self._replace_if_equals, id, sample, data) + + @monitoring.got_histogram("get_time") + def get(self, id): + return self._run(self._get, id) + + def list_dir(self, path): + return self._run(self._list_dir, path) + + @property + def _handled_exception(self): + return Exception + + def _on_all_tries_fail(self): + raise + + +class AsyncDBAdapter(DBAdapter): + IS_ASYNC = True + + async def _on_all_tries_fail(self): + raise + + async def _save(self, id, data): raise NotImplementedError - async def _mtime(self, path): + async def _replace_if_equals(self, id, sample, data): raise NotImplementedError - async def open(self, filename, *args, **kwargs): - return await self._run(self._open, filename, *args, **kwargs) + async def _get(self, id): + raise NotImplementedError - async def glob(self, path, pattern): - return await self._run(self._glob, path, pattern) + async def _path_exists(self, path): + raise NotImplementedError async def path_exists(self, path): return await self._run(self._path_exists, path) - async def mtime(self, path): - return await self._run(self._mtime, path) - @monitoring.got_histogram("save_time") async def save(self, id, data): return await self._run(self._save, id, data) @@ -71,12 +121,28 @@ async def replace_if_equals(self, id, sample, data): async def get(self, id): return await self._run(self._get, id) - async def list_dir(self, path): - return await self._run(self._list_dir, path) - - @property - async def _handled_exception(self): - return Exception - - async def _on_all_tries_fail(self): - raise + async def _run(self, action, *args, _try_count=None, **kwargs): + if _try_count is None: + _try_count = self.try_count + if _try_count <= 0: + await self._on_all_tries_fail() + _try_count = _try_count - 1 + try: + result = await action(*args, **kwargs) if asyncio.iscoroutinefunction(action) \ + else action(*args, **kwargs) + except self._handled_exception as e: + params = { + "class_name": str(self.__class__), + "exception": str(e), + "try_count": _try_count, + log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE + } + log("%(class_name)s run failed with %(exception)s.\n Got %(try_count)s tries left.", + params=params, + level="ERROR") + self._on_prepare() + result = await self._run(action, *args, _try_count=_try_count, **kwargs) + counter_name = self._get_counter_name() + if counter_name: + monitoring.got_counter(f"{counter_name}_exception") + return result diff --git a/core/db_adapter/ignite_adapter.py b/core/db_adapter/ignite_adapter.py index 987de927..bce3742b 100644 --- a/core/db_adapter/ignite_adapter.py +++ b/core/db_adapter/ignite_adapter.py @@ -9,13 +9,12 @@ import core.logging.logger_constants as log_const from core.db_adapter import error -from core.db_adapter.db_adapter import DBAdapter +from core.db_adapter.db_adapter import AsyncDBAdapter from core.logging.logger_utils import log from core.monitoring.monitoring import monitoring -class IgniteAdapter(DBAdapter): - IS_ASYNC = True +class IgniteAdapter(AsyncDBAdapter): _client: AioClient _cache = AioCache @@ -89,40 +88,3 @@ def _on_prepare(self): def _get_counter_name(self): return "ignite_async_adapter" - - @monitoring.got_histogram("save_time") - async def save(self, id, data): - return await self._async_run(self._save, id, data) - - @monitoring.got_histogram("save_time") - async def replace_if_equals(self, id, sample, data): - return await self._async_run(self._replace_if_equals, id, sample, data) - - @monitoring.got_histogram("get_time") - async def get(self, id): - return await self._async_run(self._get, id) - - async def _async_run(self, action, *args, _try_count=None, **kwargs): - if _try_count is None: - _try_count = self.try_count - if _try_count <= 0: - await self._on_all_tries_fail() - _try_count = _try_count - 1 - try: - result = await action(*args, **kwargs) - except self._handled_exception as e: - params = { - "class_name": str(self.__class__), - "exception": str(e), - "try_count": _try_count, - log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE - } - log("%(class_name)s run failed with %(exception)s.\n Got %(try_count)s tries left.", - params=params, - level="ERROR") - self._on_prepare() - result = await self._async_run(action, *args, _try_count=_try_count, **kwargs) - counter_name = self._get_counter_name() - if counter_name: - monitoring.got_counter(f"{counter_name}_exception") - return result diff --git a/core/db_adapter/memory_adapter.py b/core/db_adapter/memory_adapter.py index 1466babb..5491b7cc 100644 --- a/core/db_adapter/memory_adapter.py +++ b/core/db_adapter/memory_adapter.py @@ -8,34 +8,34 @@ def __init__(self, config=None): super(DBAdapter, self).__init__(config) self.memory_storage = {} - async def _glob(self, path, pattern): + def _glob(self, path, pattern): raise error.NotSupportedOperation - async def _path_exists(self, path): + def _path_exists(self, path): raise error.NotSupportedOperation - async def _on_prepare(self): + def _on_prepare(self): pass - async def connect(self): + def connect(self): pass - async def _open(self, filename, *args, **kwargs): + def _open(self, filename, *args, **kwargs): pass - async def _save(self, id, data): + def _save(self, id, data): self.memory_storage[id] = data - async def _replace_if_equals(self, id, sample, data): + def _replace_if_equals(self, id, sample, data): stored_data = self.memory_storage.get(id) if stored_data == sample: self.memory_storage[id] = data return True return False - async def _get(self, id): + def _get(self, id): data = self.memory_storage.get(id) return data - async def _list_dir(self, path): + def _list_dir(self, path): pass diff --git a/core/db_adapter/os_adapter.py b/core/db_adapter/os_adapter.py index 56faf9d3..14347434 100644 --- a/core/db_adapter/os_adapter.py +++ b/core/db_adapter/os_adapter.py @@ -7,44 +7,44 @@ class OSAdapter(DBAdapter): - async def _save(self, id, data): + def _save(self, id, data): raise error.NotSupportedOperation - async def _replace_if_equals(self, id, sample, data): + def _replace_if_equals(self, id, sample, data): raise error.NotSupportedOperation - async def _get(self, id): + def _get(self, id): raise error.NotSupportedOperation - async def connect(self): + def connect(self): pass - async def _on_prepare(self): + def _on_prepare(self): pass @property def source(self): return self - async def _list_dir(self, path): + def _list_dir(self, path): result = [] for path, subdirs, files in os.walk(path): result.extend([os.path.join(path, name) for name in files if not name.startswith(".")]) return result - async def _open(self, filename, *args, **kwargs): + def _open(self, filename, *args, **kwargs): return io.open(filename, *args, **kwargs) - async def _get_counter_name(self): + def _get_counter_name(self): return "os_adapter" - async def _glob(self, path, pattern): + def _glob(self, path, pattern): files_list = self._list_dir(path) filtered = fnmatch.filter(files_list, pattern) return filtered - async def _path_exists(self, path): + def _path_exists(self, path): return os.path.exists(path) - async def _mtime(self, path): + def _mtime(self, path): return os.path.getmtime(path) diff --git a/core/repositories/base_repository.py b/core/repositories/base_repository.py index 54f15918..fc160efe 100644 --- a/core/repositories/base_repository.py +++ b/core/repositories/base_repository.py @@ -19,7 +19,7 @@ def data(self): def data(self, value): self._data = value - async def load(self): + def load(self): params = { "repository_class_name": self.__class__.__name__, "repository_key": self.key, @@ -28,10 +28,10 @@ async def load(self): log("%(repository_class_name)s.load %(repository_key)s repo loading completed.", params=params, level="WARNING") - async def fill(self, data): + def fill(self, data): self.data = data - async def clear(self): + def clear(self): self.data.clear() log("%(repository_class_name)s.clear %(repository_key)s cleared.", params={"repository_class_name": self.__class__.__name__, @@ -39,8 +39,8 @@ async def clear(self): log_const.KEY_NAME: log_const.REPOSITORY_CLEAR_VALUE}, level="WARNING") - async def save(self, save_parameters): + def save(self, save_parameters): raise NotImplementedError - async def check_load_in_parts(self): + def check_load_in_parts(self): return False diff --git a/core/repositories/classifier_repository.py b/core/repositories/classifier_repository.py index f5a15012..135ebc14 100644 --- a/core/repositories/classifier_repository.py +++ b/core/repositories/classifier_repository.py @@ -60,7 +60,7 @@ def _check_classifier_config(self, classifier_key: str, classifier_params: Dict[ except KeyError: raise Exception(f"Missing field: '{req_param}' for classifier {classifier_key} in classifiers.json") - async def load(self) -> None: + def load(self) -> None: if not self._folder_repository: return None @@ -136,8 +136,8 @@ async def load(self) -> None: ) classifiers_dict[classifier_key] = SkipClassifier.get_nothing() - await super(ClassifierRepository, self).fill(classifiers_dict) + super(ClassifierRepository, self).fill(classifiers_dict) classifiers_initial_launch(classifiers_dict) - async def save(self, save_parameters: Any) -> None: + def save(self, save_parameters: Any) -> None: pass diff --git a/core/repositories/csv_repository.py b/core/repositories/csv_repository.py index afec8142..7af241ea 100644 --- a/core/repositories/csv_repository.py +++ b/core/repositories/csv_repository.py @@ -7,9 +7,9 @@ def __init__(self, filename, source=None, *args, **kwargs): super(CSVRepository, self).__init__(source=source, *args, **kwargs) self.filename = filename - async def load(self): + def load(self): with self.source.open(self.filename, newline='') as stream: reader = csv.DictReader(stream) data = list(reader) - await self.fill(data) - await super(CSVRepository, self).load() + self.fill(data) + super(CSVRepository, self).load() diff --git a/core/repositories/dill_repository.py b/core/repositories/dill_repository.py index 9218139e..36596d2d 100644 --- a/core/repositories/dill_repository.py +++ b/core/repositories/dill_repository.py @@ -19,13 +19,13 @@ def __init__(self, filename, source=None, required=True, *args, **kwargs): self.filename = filename self.required = required - async def load(self): + def load(self): dill._dill._reverse_typemap['ClassType'] = type try: with self.source.open(self.filename, 'rb') as stream: data = dill.load(stream) - await self.fill(data) + self.fill(data) except FileNotFoundError as error: params = { 'error': str(error), @@ -34,4 +34,4 @@ async def load(self): log('DillRepository.load loading failed. Error %(error)s', params=params, level='WARNING') if self.required: raise - await super(DillRepository, self).load() + super(DillRepository, self).load() diff --git a/core/repositories/file_repository.py b/core/repositories/file_repository.py index 7c70ad18..642ef07d 100644 --- a/core/repositories/file_repository.py +++ b/core/repositories/file_repository.py @@ -17,13 +17,13 @@ def __init__(self, filename, loader, source=None, save_target=None, saver=None, self.save_target = save_target self._file_exist = False - async def load(self): - if await self.source.path_exists(self.filename): + def load(self): + if self.source.path_exists(self.filename): self._file_exist = True - with await self.source.open(self.filename, 'rb') as stream: + with self.source.open(self.filename, 'rb') as stream: binary_data = stream.read() data = binary_data.decode() - await self.fill(self.loader(data)) + self.fill(self.loader(data)) else: self._file_exist = False params = { @@ -32,9 +32,9 @@ async def load(self): } log("FileRepository.load loading failed with file %(error_repository_path)s", params=params, level="WARNING") - await super(FileRepository, self).load() + super(FileRepository, self).load() - async def save(self, save_parameters): + def save(self, save_parameters): with self.source.open(self.save_target, 'wb') as stream: stream.write(self.saver(self.data, **save_parameters).encode()) @@ -49,22 +49,21 @@ def __init__(self, *args, update_cooldown=5, **kwargs): raise Exception(f"{self.__class__.__name__} support only OSAdapter") @FileRepository.data.getter - # TODO: добавить await везде, где используется - async def data(self): - if await self._is_outdated: + def data(self): + if self._is_outdated: params = { "filename": self.filename } log("FileRepository.data %(filename)s is outdated. Data will be reloaded.", params=params, level="INFO") - await self.load() + self.load() return self._data - async def load(self): - await super().load() + def load(self): + super().load() if self._file_exist: - self._last_mtime = await self.source.mtime(self.filename) + self._last_mtime = self.source.mtime(self.filename) self._last_update_time = time.time() @property @@ -72,7 +71,7 @@ def expired(self): return self._last_update_time + self.update_cooldown < time.time() @property - async def _is_outdated(self): + def _is_outdated(self): if self._file_exist and self.expired: - return await self.source.mtime(self.filename) > self._last_mtime + return self.source.mtime(self.filename) > self._last_mtime return False diff --git a/core/repositories/folder_repository.py b/core/repositories/folder_repository.py index c052defb..205f8530 100644 --- a/core/repositories/folder_repository.py +++ b/core/repositories/folder_repository.py @@ -8,9 +8,9 @@ class FolderRepository(ShardRepository): def __init__(self, path, loader, source=None, *args, **kwargs): super(FolderRepository, self).__init__(path, loader, source, *args, **kwargs) - async def _load_item(self, name): + def _load_item(self, name): try: - with await self.source.open(name, mode='rb') as shard_stream: + with self.source.open(name, mode='rb') as shard_stream: shard_binary_data = shard_stream.read() shard_data = shard_binary_data.decode() loaded_data = self.loader(shard_data) @@ -25,22 +25,22 @@ async def _load_item(self, name): raise return loaded_data - async def _form_file_upload_map(self, shard_desc): + def _form_file_upload_map(self, shard_desc): filename_to_data = {} for shard in shard_desc: - shard_data = await self._load_item(shard) + shard_data = self._load_item(shard) if shard_data: filename_to_data.update({shard: shard_data}) return filename_to_data - async def load(self): - shard_desc = await self.get_shard_desc() - await self.fill(await self._form_file_upload_map(shard_desc)) - await super(FolderRepository, self).load() + def load(self): + shard_desc = self.get_shard_desc() + self.fill(self._form_file_upload_map(shard_desc)) + super(FolderRepository, self).load() - async def load_in_parts(self, count): - await self.clear() - shard_desc = await self.get_shard_desc() + def load_in_parts(self, count): + self.clear() + shard_desc = self.get_shard_desc() for i in range(0, len(shard_desc), count): desc_slice = shard_desc[i: i + count] self.fill_on_top(self._form_file_upload_map(desc_slice)) @@ -48,10 +48,10 @@ async def load_in_parts(self, count): params={"current_count": i + len(desc_slice), "all_count": len(shard_desc)}, level="WARNING") - await super(FolderRepository, self).load() + super(FolderRepository, self).load() - async def get_shard_desc(self): - shard_desc = await self.source.list_dir(self.path) + def get_shard_desc(self): + shard_desc = self.source.list_dir(self.path) if len(shard_desc) == 0: params = { "error_repository_path": self.path, diff --git a/core/repositories/items_repository.py b/core/repositories/items_repository.py index ac3c87ef..7a7346aa 100644 --- a/core/repositories/items_repository.py +++ b/core/repositories/items_repository.py @@ -15,8 +15,8 @@ def data(self, value): else: self._data = value - async def load(self): - await super(ItemsRepository, self).load() + def load(self): + super(ItemsRepository, self).load() def __iter__(self): return iter(self.data) diff --git a/core/repositories/shard_repository.py b/core/repositories/shard_repository.py index 2ef8e8fa..b6ad62bf 100644 --- a/core/repositories/shard_repository.py +++ b/core/repositories/shard_repository.py @@ -30,10 +30,10 @@ def _get_data_type(data): f" data type {t}") return t - async def load(self): - await super(ShardRepository, self).load() + def load(self): + super(ShardRepository, self).load() - async def fill(self, data): + def fill(self, data): res = None t = self._get_data_type(data) if t == dict: @@ -54,5 +54,5 @@ def fill_on_top(self, data): elif t == list: self.data.extend(v) - async def save(self, save_parameters): + def save(self, save_parameters): raise NotImplementedError diff --git a/core/utils/rerunable.py b/core/utils/rerunable.py index 6cef4c2f..aef66ab9 100644 --- a/core/utils/rerunable.py +++ b/core/utils/rerunable.py @@ -13,26 +13,23 @@ def __init__(self, config=None): self.try_count = self.config.get("try_count") or self.DEFAULT_RERUNABLE_TRY_COUNT @property - async def _handled_exception(self): + def _handled_exception(self): raise NotImplementedError - async def _on_prepare(self): + def _on_prepare(self): raise NotImplementedError - async def _on_all_tries_fail(self): + def _on_all_tries_fail(self): raise NotImplementedError - async def _run(self, action, *args, _try_count=None, **kwargs): + def _run(self, action, *args, _try_count=None, **kwargs): if _try_count is None: _try_count = self.try_count if _try_count <= 0: - await self._on_all_tries_fail() + self._on_all_tries_fail() _try_count = _try_count - 1 try: - if asyncio.iscoroutinefunction(action): - result = await action(*args, **kwargs) - else: - result = action(*args, **kwargs) + result = action(*args, **kwargs) except Exception as e: params = { "class_name": str(self.__class__), @@ -43,12 +40,12 @@ async def _run(self, action, *args, _try_count=None, **kwargs): log("%(class_name)s run failed with %(exception)s.\n Got %(try_count)s tries left.", params=params, level="ERROR") - await self._on_prepare() - result = await self._run(action, *args, _try_count=_try_count, **kwargs) - counter_name = await self._get_counter_name() + self._on_prepare() + result = self._run(action, *args, _try_count=_try_count, **kwargs) + counter_name = self._get_counter_name() if counter_name: monitoring.got_counter(f"{counter_name}_exception") return result - async def _get_counter_name(self): + def _get_counter_name(self): return diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index ca472f70..c8d0a74e 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -72,6 +72,10 @@ def __init__( def get_db(self): db_adapter = db_adapter_factory(self.settings["template_settings"].get("db_adapter", {})) + if not db_adapter.IS_ASYNC: + raise Exception( + f"Blocking adapter {db_adapter.__class__.__name__} is not good for {self.__class__.__name__}" + ) self.loop.run_until_complete(db_adapter.connect()) return db_adapter diff --git a/tests/core_tests/repositories_test/test_repo.py b/tests/core_tests/repositories_test/test_repo.py index 3638b476..9d46eb23 100644 --- a/tests/core_tests/repositories_test/test_repo.py +++ b/tests/core_tests/repositories_test/test_repo.py @@ -22,9 +22,9 @@ class MockBaseRepository(BaseRepository): def __init__(self): super(MockBaseRepository, self).__init__(MockDescriptionItem) - async def load(self): - await self.fill({"test": {"value": 1}}) - await super(MockBaseRepository, self).load() + def load(self): + self.fill({"test": {"value": 1}}) + super(MockBaseRepository, self).load() class MockShardRepository(ShardRepository): diff --git a/tests/core_tests/test_utils/test_rerunable.py b/tests/core_tests/test_utils/test_rerunable.py index abddce2e..44a7b53a 100644 --- a/tests/core_tests/test_utils/test_rerunable.py +++ b/tests/core_tests/test_utils/test_rerunable.py @@ -1,4 +1,4 @@ -from unittest import IsolatedAsyncioTestCase +from unittest import TestCase from unittest.mock import Mock from core.utils.rerunable import Rerunable @@ -22,20 +22,20 @@ def __init__(self, try_count): self._on_prepare_mock = Mock() @property - async def _handled_exception(self): + def _handled_exception(self): return HandledException - async def _on_prepare(self): + def _on_prepare(self): self._on_prepare_mock() - async def _on_all_tries_fail(self): + def _on_all_tries_fail(self): raise AllFailedException - async def run(self, *args, **kwargs): - return await self._run(*args, **kwargs) + def run(self, *args, **kwargs): + return self._run(*args, **kwargs) -class TestRerunable(IsolatedAsyncioTestCase): +class TestRerunable(TestCase): def setUp(self): self.try_count = 2 self.rerunable = RerunableOne(self.try_count) @@ -43,36 +43,36 @@ def setUp(self): self.param = Mock() self.param1 = {"param1_name": Mock()} - async def test_pass(self): + def test_pass(self): self.action = Mock() - await self.rerunable.run(self.action, self.param, **self.param1) + self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_once_with(self.param, **self.param1) # TODO: починить тест - async def test_wrong_exception(self): + def test_wrong_exception(self): self.action = Mock(side_effect=CustomException()) with self.assertRaises(CustomException) as context: - await self.rerunable.run(self.action, self.param, **self.param1) + self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_once_with(self.param, **self.param1) - async def test_all_retry_failed(self): + def test_all_retry_failed(self): self.action = Mock(side_effect=HandledException()) with self.assertRaises(AllFailedException) as context: - await self.rerunable.run(self.action, self.param, **self.param1) + self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_with(self.param, **self.param1) self.assertEqual(self.action.call_count, self.try_count) - async def test_first_try_failed(self): + def test_first_try_failed(self): self.param = Mock() self.param1 = {"param1_name": Mock()} self.action = Mock(side_effect=[HandledException(), self.expected_value]) - result = await self.rerunable.run(self.action, self.param, **self.param1) + result = self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_with(self.param, **self.param1) self.assertEqual(self.action.call_count, self.try_count) self.assertEqual(result, self.expected_value) - async def test_first_try_ok(self): + def test_first_try_ok(self): self.action = Mock(return_value=self.expected_value) - result = await self.rerunable.run(self.action, self.param, **self.param1) + result = self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_once_with(self.param, **self.param1) self.assertEqual(result, self.expected_value) From 135c77e69fc7760319e55ec00b0cc16d0c3b4ddc Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Fri, 26 Nov 2021 22:39:07 +0300 Subject: [PATCH 076/116] fix Rerunable and AsyncDBAdapter --- core/db_adapter/db_adapter.py | 10 +++++----- core/utils/rerunable.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/db_adapter/db_adapter.py b/core/db_adapter/db_adapter.py index ab915bcb..c44c1098 100644 --- a/core/db_adapter/db_adapter.py +++ b/core/db_adapter/db_adapter.py @@ -107,21 +107,21 @@ async def _path_exists(self, path): raise NotImplementedError async def path_exists(self, path): - return await self._run(self._path_exists, path) + return await self._async_run(self._path_exists, path) @monitoring.got_histogram("save_time") async def save(self, id, data): - return await self._run(self._save, id, data) + return await self._async_run(self._save, id, data) @monitoring.got_histogram("save_time") async def replace_if_equals(self, id, sample, data): - return await self._run(self._replace_if_equals, id, sample, data) + return await self._async_run(self._replace_if_equals, id, sample, data) @monitoring.got_histogram("get_time") async def get(self, id): - return await self._run(self._get, id) + return await self._async_run(self._get, id) - async def _run(self, action, *args, _try_count=None, **kwargs): + async def _async_run(self, action, *args, _try_count=None, **kwargs): if _try_count is None: _try_count = self.try_count if _try_count <= 0: diff --git a/core/utils/rerunable.py b/core/utils/rerunable.py index aef66ab9..369a6ee8 100644 --- a/core/utils/rerunable.py +++ b/core/utils/rerunable.py @@ -30,7 +30,7 @@ def _run(self, action, *args, _try_count=None, **kwargs): _try_count = _try_count - 1 try: result = action(*args, **kwargs) - except Exception as e: + except self._handled_exception as e: params = { "class_name": str(self.__class__), "exception": str(e), From 9b9cb34d9017dbf30e7a599898083a00ecb43b82 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Sat, 27 Nov 2021 01:17:58 +0300 Subject: [PATCH 077/116] fix Rerunable --- tests/core_tests/test_utils/test_rerunable.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/core_tests/test_utils/test_rerunable.py b/tests/core_tests/test_utils/test_rerunable.py index 44a7b53a..e3b64f0b 100644 --- a/tests/core_tests/test_utils/test_rerunable.py +++ b/tests/core_tests/test_utils/test_rerunable.py @@ -48,7 +48,6 @@ def test_pass(self): self.rerunable.run(self.action, self.param, **self.param1) self.action.assert_called_once_with(self.param, **self.param1) - # TODO: починить тест def test_wrong_exception(self): self.action = Mock(side_effect=CustomException()) with self.assertRaises(CustomException) as context: From e0df62d931d4bee029a57608bfc05b9a3d3828d1 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Sat, 27 Nov 2021 22:44:34 +0300 Subject: [PATCH 078/116] fix async --- smart_kit/configs/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smart_kit/configs/settings.py b/smart_kit/configs/settings.py index bd96c68e..f740e607 100644 --- a/smart_kit/configs/settings.py +++ b/smart_kit/configs/settings.py @@ -1,5 +1,6 @@ import yaml import os +import asyncio from core.configs.base_config import BaseConfig from core.db_adapter.ceph.ceph_adapter import CephAdapter @@ -18,6 +19,7 @@ def __init__(self, *args, **kwargs): self.secret_path = kwargs.get("secret_path") self.app_name = kwargs.get("app_name") self.adapters = {Settings.CephAdapterKey: CephAdapter, self.OSAdapterKey: OSAdapter} + self.loop = asyncio.get_event_loop() self.repositories = [ FileRepository( self.subfolder_path("template_config.yml"), loader=yaml.safe_load, key="template_settings" From 6c179970146c938389a4cb9083db33656f6983e7 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Sat, 27 Nov 2021 22:53:36 +0300 Subject: [PATCH 079/116] fix async --- smart_kit/configs/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/smart_kit/configs/settings.py b/smart_kit/configs/settings.py index f740e607..6356d00a 100644 --- a/smart_kit/configs/settings.py +++ b/smart_kit/configs/settings.py @@ -62,6 +62,9 @@ def get_source(self): adapter_settings = self.registered_repositories[ adapter_key].data if adapter_key != Settings.OSAdapterKey else None adapter = self.adapters[adapter_key](adapter_settings) - self.loop.run_until_complete(adapter.connect()) + if asyncio.iscoroutinefunction(adapter.connect): + self.loop.run_until_complete(adapter.connect()) + else: + adapter.connect() source = adapter.source return source From 5f55268ffe0c7fb30c1a07913f0d639a1065b162 Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Mon, 29 Nov 2021 07:19:47 +0300 Subject: [PATCH 080/116] fix kafka consumer multithreading --- smart_kit/start_points/main_loop_kafka.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 633290fc..5824de4c 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -5,7 +5,7 @@ import pstats import signal import time -import sys +import concurrent.futures from collections import namedtuple from functools import lru_cache @@ -48,6 +48,9 @@ def __init__(self, *args, **kwargs): "class_name": self.__class__.__name__}) self.health_check_server_future = None super().__init__(*args, **kwargs) + # We have many async loops for messages processing in main thread + # And 1 thread for independent consecutive Kafka reading + self.kafka_executor_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) try: kafka_config = _enrich_config_from_secret( @@ -122,7 +125,7 @@ async def healthcheck_coro(self): async def process_consumer(self, kafka_key): consumer = self.consumers[kafka_key] loop = asyncio.get_event_loop() - max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 1) + max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 10) total_messages = 0 async def msg_loop(iteration): @@ -145,7 +148,7 @@ async def msg_loop(iteration): mq_message = None with StatsTimer() as poll_timer: # Max delay between polls configured in consumer.poll_timeout param - mq_message = await loop.run_in_executor(None, consumer.poll) + mq_message = await loop.run_in_executor(self.kafka_executor_pool, consumer.poll) if mq_message: print(f"\n-- Processing {self.concurrent_messages} msgs at {iteration} iter\n") total_messages += 1 From 991fc3a09883db4136874bbdd007edbafc9b890e Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Wed, 1 Dec 2021 09:45:31 +0300 Subject: [PATCH 081/116] fix kafka init bug --- core/mq/kafka/kafka_consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/mq/kafka/kafka_consumer.py b/core/mq/kafka/kafka_consumer.py index 3e19fbf7..9651d3bf 100644 --- a/core/mq/kafka/kafka_consumer.py +++ b/core/mq/kafka/kafka_consumer.py @@ -59,7 +59,7 @@ def on_assign_log(consumer, partitions): log("KafkaConsumer.subscribe: assign %(partitions)s %(log_level)s", params=params, level=log_level) def subscribe(self, topics=None): - topics = topics or list(self._config["topics"].values()) + topics = list(set(topics or list(self._config["topics"].values()))) self._consumer.subscribe(topics, on_assign=self.get_on_assign_callback() if self.assign_offset_end else From ac28533dd5abf768a55e22bc30aa24a02647b39f Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Wed, 1 Dec 2021 10:42:02 +0300 Subject: [PATCH 082/116] make memory adapter async --- core/db_adapter/memory_adapter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/db_adapter/memory_adapter.py b/core/db_adapter/memory_adapter.py index 5491b7cc..6bed7dda 100644 --- a/core/db_adapter/memory_adapter.py +++ b/core/db_adapter/memory_adapter.py @@ -1,11 +1,11 @@ from core.db_adapter import error -from core.db_adapter.db_adapter import DBAdapter +from core.db_adapter.db_adapter import AsyncDBAdapter -class MemoryAdapter(DBAdapter): +class MemoryAdapter(AsyncDBAdapter): def __init__(self, config=None): - super(DBAdapter, self).__init__(config) + super(AsyncDBAdapter, self).__init__(config) self.memory_storage = {} def _glob(self, path, pattern): @@ -17,7 +17,7 @@ def _path_exists(self, path): def _on_prepare(self): pass - def connect(self): + async def connect(self): pass def _open(self, filename, *args, **kwargs): From c12179ac4b1e738eab4e7ed292b62c007671a74a Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Thu, 2 Dec 2021 10:35:57 +0300 Subject: [PATCH 083/116] fix testing --- smart_kit/testing/suite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smart_kit/testing/suite.py b/smart_kit/testing/suite.py index ec293fc2..2831d09d 100644 --- a/smart_kit/testing/suite.py +++ b/smart_kit/testing/suite.py @@ -145,7 +145,7 @@ def __init__(self, app_model: SmartAppModel, settings: Settings, user_cls: type, self.__user_cls = user_cls self.__from_msg_cls = from_msg_cls - def run(self) -> bool: + async def run(self) -> bool: success = True app_callback_id = None @@ -172,7 +172,7 @@ def run(self) -> bool: parametrizer_cls=self.__parametrizer_cls ) - commands = self.app_model.answer(message, user) or [] + commands = await self.app_model.answer(message, user) or [] answers = self._generate_answers( user=user, commands=commands, message=message From 26702089fac82b1e781442785ebee52c54d2b789 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Thu, 2 Dec 2021 10:43:37 +0300 Subject: [PATCH 084/116] fix testing --- smart_kit/testing/suite.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/smart_kit/testing/suite.py b/smart_kit/testing/suite.py index 2831d09d..e91ef182 100644 --- a/smart_kit/testing/suite.py +++ b/smart_kit/testing/suite.py @@ -1,3 +1,4 @@ +import asyncio import json import os from csv import DictWriter, QUOTE_MINIMAL @@ -34,7 +35,7 @@ def run_testfile(path: AnyStr, file: AnyStr, app_model: SmartAppModel, settings: csv_case_callback = csv_file_callback(test_case) else: csv_case_callback = None - if TestCase( + if asyncio.get_event_loop().run_until_complete(TestCase( app_model, settings, user_cls, @@ -44,7 +45,7 @@ def run_testfile(path: AnyStr, file: AnyStr, app_model: SmartAppModel, settings: storaged_predefined_fields=storaged_predefined_fields, interactive=interactive, csv_case_callback=csv_case_callback, - ).run(): + ).run()): print(f"[+] {test_case} OK") success += 1 print(f"[+] {file} {success}/{len(json_obj)}") From ab5be22ac68f37d96ec682134329dff02d1a23cc Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Thu, 2 Dec 2021 11:55:07 +0300 Subject: [PATCH 085/116] fix testing --- scenarios/actions/action.py | 6 +++--- .../scenario_descriptions/form_filling_scenario.py | 2 +- scenarios/scenario_models/field/field.py | 2 +- smart_kit/handlers/handle_server_action.py | 2 +- tests/scenarios_tests/actions_test/test_action.py | 12 ++++++------ .../action/test_run_scenario_by_project_name.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scenarios/actions/action.py b/scenarios/actions/action.py index f1d65303..03aa25d8 100644 --- a/scenarios/actions/action.py +++ b/scenarios/actions/action.py @@ -311,7 +311,7 @@ async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessing scenario_id = self.scenario.render(params) scenario = user.descriptions["scenarios"].get(scenario_id) if scenario: - return scenario.run(text_preprocessing_result, user, params) + return await scenario.run(text_preprocessing_result, user, params) class RunLastScenarioAction(Action): @@ -320,7 +320,7 @@ async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessing last_scenario_id = user.last_scenarios.last_scenario_name scenario = user.descriptions["scenarios"].get(last_scenario_id) if scenario: - return scenario.run(text_preprocessing_result, user, params) + return await scenario.run(text_preprocessing_result, user, params) class ChoiceScenarioAction(Action): @@ -515,7 +515,7 @@ async def run(self, user: User, text_preprocessing_result: TextPreprocessingResu scenario_id = user.message.project_name scenario = user.descriptions["scenarios"].get(scenario_id) if scenario: - return scenario.run(text_preprocessing_result, user, params) + return await scenario.run(text_preprocessing_result, user, params) else: log("%(class_name)s warning: %(scenario_id)s isn't exist", params={log_const.KEY_NAME: "warning_in_RunScenarioByProjectNameAction", diff --git a/scenarios/scenario_descriptions/form_filling_scenario.py b/scenarios/scenario_descriptions/form_filling_scenario.py index 9975d6d3..be831018 100644 --- a/scenarios/scenario_descriptions/form_filling_scenario.py +++ b/scenarios/scenario_descriptions/form_filling_scenario.py @@ -46,7 +46,7 @@ async def ask_again(self, text_preprocessing_result, user, params): content={HistoryConstants.content_fields.FIELD: question_field.description.id}, results=HistoryConstants.event_results.ASK_QUESTION)) - return question.run(user, text_preprocessing_result, params) + return await question.run(user, text_preprocessing_result, params) async def _check_field(self, text_preprocessing_result, user, params): form = user.forms[self.form_type] diff --git a/scenarios/scenario_models/field/field.py b/scenarios/scenario_models/field/field.py index d9de4ddc..2d2b048e 100644 --- a/scenarios/scenario_models/field/field.py +++ b/scenarios/scenario_models/field/field.py @@ -41,7 +41,7 @@ def can_be_updated(self): async def check_can_be_filled(self, text_preprocessing_result, user): check, run = await asyncio.gather( self.description.requirement.check(text_preprocessing_result, user), - self.description.filler.run(user, text_preprocessing_result)) + await self.description.filler.run(user, text_preprocessing_result)) return check and run is not None @property diff --git a/smart_kit/handlers/handle_server_action.py b/smart_kit/handlers/handle_server_action.py index 8388349d..8be109dd 100644 --- a/smart_kit/handlers/handle_server_action.py +++ b/smart_kit/handlers/handle_server_action.py @@ -35,4 +35,4 @@ async def run(self, payload, user): action_id = self.get_action_name(payload, user) action = user.descriptions["external_actions"][action_id] - return action.run(user, TextPreprocessingResult({}), action_params) + return await action.run(user, TextPreprocessingResult({}), action_params) diff --git a/tests/scenarios_tests/actions_test/test_action.py b/tests/scenarios_tests/actions_test/test_action.py index 3409d95c..81ddde21 100644 --- a/tests/scenarios_tests/actions_test/test_action.py +++ b/tests/scenarios_tests/actions_test/test_action.py @@ -1,6 +1,6 @@ import unittest from typing import Dict, Any, Union, Optional -from unittest.mock import MagicMock, Mock, ANY +from unittest.mock import MagicMock, Mock, ANY, AsyncMock from core.basic_models.actions.basic_actions import Action, action_factory, actions from core.model.registered import registered_factories @@ -354,7 +354,7 @@ async def test_scenario_action(self): action = RunScenarioAction({"scenario": "test"}) user = Mock() user.parametrizer = MockParametrizer(user, {}) - scen = Mock() + scen = AsyncMock() scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"test": scen}} @@ -368,7 +368,7 @@ async def test_scenario_action_with_jinja_good(self): action = RunScenarioAction(items) user = Mock() user.parametrizer = MockParametrizer(user, {"data": params}) - scen = Mock() + scen = AsyncMock() scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"ANNA.pipeline.scenario": scen}} @@ -390,7 +390,7 @@ async def test_scenario_action_without_jinja(self): action = RunScenarioAction({"scenario": "next_scenario"}) user = Mock() user.parametrizer = MockParametrizer(user, {}) - scen = Mock() + scen = AsyncMock() scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"next_scenario": scen}} @@ -403,7 +403,7 @@ class RunLastScenarioActionTest(unittest.IsolatedAsyncioTestCase): async def test_scenario_action(self): action = RunLastScenarioAction({}) user = Mock() - scen = Mock() + scen = AsyncMock() scen_result = 'done' scen.run.return_value = scen_result user.descriptions = {"scenarios": {"test": scen}} @@ -423,7 +423,7 @@ async def mock_and_perform_action(test_items: Dict[str, Any], expected_result: O action = ChoiceScenarioAction(test_items) user = Mock() user.parametrizer = MockParametrizer(user, {}) - scen = Mock() + scen = AsyncMock() scen.run.return_value = expected_result if expected_scen: user.descriptions = {"scenarios": {expected_scen: scen}} diff --git a/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py b/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py index 0c3288fe..8a2c6bf6 100644 --- a/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py +++ b/tests/smart_kit_tests/action/test_run_scenario_by_project_name.py @@ -7,7 +7,7 @@ class TestScenarioDesc(dict): - def run(self, argv1, argv2, params): + async def run(self, argv1, argv2, params): return 'result to run scenario' From 2cea7e75b6a87cda76469bf789b321b184c4ea8e Mon Sep 17 00:00:00 2001 From: SyrexMinus Date: Thu, 2 Dec 2021 12:44:35 +0300 Subject: [PATCH 086/116] fix async field --- scenarios/scenario_models/field/field.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/scenario_models/field/field.py b/scenarios/scenario_models/field/field.py index 2d2b048e..d9de4ddc 100644 --- a/scenarios/scenario_models/field/field.py +++ b/scenarios/scenario_models/field/field.py @@ -41,7 +41,7 @@ def can_be_updated(self): async def check_can_be_filled(self, text_preprocessing_result, user): check, run = await asyncio.gather( self.description.requirement.check(text_preprocessing_result, user), - await self.description.filler.run(user, text_preprocessing_result)) + self.description.filler.run(user, text_preprocessing_result)) return check and run is not None @property From 298aac2c5ee9d3582c1cbbda22cd4c31fc0f153b Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 6 Dec 2021 05:15:45 +0300 Subject: [PATCH 087/116] fix async behaviors --- scenarios/behaviors/behaviors.py | 2 +- smart_kit/start_points/main_loop_http.py | 3 ++- smart_kit/testing/local.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scenarios/behaviors/behaviors.py b/scenarios/behaviors/behaviors.py index d5b239c8..6a7b7617 100644 --- a/scenarios/behaviors/behaviors.py +++ b/scenarios/behaviors/behaviors.py @@ -133,7 +133,7 @@ async def success(self, callback_id: str): callback_action_params, ) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - await behavior.success_action.run(self._user, text_preprocessing_result, callback_action_params) + result = await behavior.success_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result diff --git a/smart_kit/start_points/main_loop_http.py b/smart_kit/start_points/main_loop_http.py index 40adeccd..8425a56f 100644 --- a/smart_kit/start_points/main_loop_http.py +++ b/smart_kit/start_points/main_loop_http.py @@ -1,3 +1,4 @@ +import asyncio import json import typing from collections import defaultdict @@ -69,7 +70,7 @@ def process_message(self, message: SmartAppFromMessage, *args, **kwargs): user = self.load_user(db_uid, message) stats += "Loading time: {} msecs\n".format(load_timer.msecs) with StatsTimer() as script_timer: - commands = self.model.answer(message, user) + commands = asyncio.get_event_loop().run_until_complete(self.model.answer(message, user)) if commands: answer = self._generate_answers(user, commands, message) else: diff --git a/smart_kit/testing/local.py b/smart_kit/testing/local.py index 36e106d6..0569d80e 100644 --- a/smart_kit/testing/local.py +++ b/smart_kit/testing/local.py @@ -1,3 +1,4 @@ +import asyncio import cmd import json import os @@ -146,7 +147,7 @@ def process_message(self, raw_message: str, headers: tuple = ()) -> typing.Tuple user = self.__user_cls(self.environment.user_id, message, self.user_data, self.settings, self.app_model.scenario_descriptions, self.__parametrizer_cls, load_error=False) - answers = self.app_model.answer(message, user) + answers = asyncio.get_event_loop().run_until_complete(self.app_model.answer(message, user)) return user, answers or [] def default(self, _input: str): From fb0c2483f6d18d6cac0bd0301b106a646b41a5e9 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 6 Dec 2021 17:37:23 +0300 Subject: [PATCH 088/116] fix async behaviors --- scenarios/behaviors/behaviors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scenarios/behaviors/behaviors.py b/scenarios/behaviors/behaviors.py index 6a7b7617..9a1d4068 100644 --- a/scenarios/behaviors/behaviors.py +++ b/scenarios/behaviors/behaviors.py @@ -152,7 +152,7 @@ async def fail(self, callback_id: str): smart_kit_metrics.counter_behavior_fail, "fail", callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - await behavior.fail_action.run(self._user, text_preprocessing_result, callback_action_params) + result = await behavior.fail_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result @@ -171,7 +171,7 @@ async def timeout(self, callback_id: str): smart_kit_metrics.counter_behavior_timeout, "timeout", callback_action_params) text_preprocessing_result = TextPreprocessingResult(callback.text_preprocessing_result) - await behavior.timeout_action.run(self._user, text_preprocessing_result, callback_action_params) + result = await behavior.timeout_action.run(self._user, text_preprocessing_result, callback_action_params) self._delete(callback_id) return result From 1e54d47da9bdc89f7830fc4d8771e3aa00e3bbc7 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 13 Dec 2021 06:10:30 +0300 Subject: [PATCH 089/116] fix async tests --- core/db_adapter/redis_adapter.py | 43 ------------------- smart_kit/resources/__init__.py | 2 - .../scenarios_test/test_tree_scenario.py | 9 ++-- 3 files changed, 5 insertions(+), 49 deletions(-) delete mode 100644 core/db_adapter/redis_adapter.py diff --git a/core/db_adapter/redis_adapter.py b/core/db_adapter/redis_adapter.py deleted file mode 100644 index 5f5afd75..00000000 --- a/core/db_adapter/redis_adapter.py +++ /dev/null @@ -1,43 +0,0 @@ -import redis -import typing -from core.db_adapter.db_adapter import DBAdapter -from core.db_adapter import error - - -class RedisAdapter(DBAdapter): - def __init__(self, config=None): - super().__init__(config) - self._redis: typing.Optional[redis.Redis] = None - - try: - del self.config["type"] - except KeyError: - pass - - def connect(self): - self._redis = redis.Redis(**self.config) - - def _open(self, filename, *args, **kwargs): - pass - - def _save(self, id, data): - return self._redis.set(id, data) - - def _replace_if_equals(self, id, sample, data): - return self._redis.set(id, data) - - def _get(self, id): - data = self._redis.get(id) - return data.decode() if data else None - - def _list_dir(self, path): - raise error.NotSupportedOperation - - def _glob(self, path, pattern): - raise error.NotSupportedOperation - - def _path_exists(self, path): - self._redis.exists(path) - - def _on_prepare(self): - pass diff --git a/smart_kit/resources/__init__.py b/smart_kit/resources/__init__.py index 0f19622a..d1c965ef 100644 --- a/smart_kit/resources/__init__.py +++ b/smart_kit/resources/__init__.py @@ -38,7 +38,6 @@ from core.db_adapter.db_adapter import db_adapters from core.db_adapter.ignite_adapter import IgniteAdapter from core.db_adapter.memory_adapter import MemoryAdapter -from core.db_adapter.redis_adapter import RedisAdapter from core.descriptions.descriptions import registered_description_factories from core.model.queued_objects.limited_queued_hashable_objects_description import \ LimitedQueuedHashableObjectsDescriptionsItems @@ -380,7 +379,6 @@ def init_db_adapters(self): db_adapters[None] = MemoryAdapter db_adapters["ignite"] = IgniteAdapter db_adapters["memory"] = MemoryAdapter - db_adapters["redis"] = RedisAdapter db_adapters["aioredis"] = AIORedisAdapter db_adapters["aioredis_sentinel"] = AIORedisSentinelAdapter diff --git a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py index 2a4d3fac..53defcc9 100644 --- a/tests/scenarios_tests/scenarios_test/test_tree_scenario.py +++ b/tests/scenarios_tests/scenarios_test/test_tree_scenario.py @@ -50,13 +50,13 @@ async def test_1(self): "scenario_nodes": {"node_1": node_mock}} field_descriptor = Mock(name="field_descriptor_mock") - field_descriptor.filler.extract = AsyncMock(name="my_field_value_1", return_value=61) + field_descriptor.filler.run = AsyncMock(name="my_field_value_1", return_value=61) field_descriptor.fill_other = False field_descriptor.field_validator.actions = [] + field_descriptor.field_validator.requirement.check = AsyncMock(return_value=True) internal_form = Mock(name="internal_form_mock") internal_form.description.fields.items = Mock(return_value=[("age", field_descriptor)]) - internal_form.field.field_validator.requirement.check = AsyncMock(return_value=True) internal_form.fields = MagicMock() internal_form.fields.values.items = Mock(return_value={"age": 61}) internal_form.is_valid = Mock(return_value=True) @@ -97,6 +97,7 @@ async def test_break(self): actions["test"] = MockAction actions["break"] = MockAction actions["success"] = MockAction + actions["external"] = MockAction form_type = "form for doing smth" internal_form_key = "my form key" @@ -107,15 +108,15 @@ async def test_break(self): "scenario_nodes": {"node_1": node_mock}, "actions": [{"type": "success"}]} field_descriptor = Mock(name="field_descriptor_mock") - field_descriptor.filler.extract = AsyncMock(name="my_field_value_1", return_value=61) + field_descriptor.filler.run = AsyncMock(name="my_field_value_1", return_value=61) field_descriptor.fill_other = False field_descriptor.field_validator.actions = [] + field_descriptor.field_validator.requirement.check = AsyncMock(return_value=True) field_descriptor.on_filled_actions = [BreakAction(), MockAction(command_name="break action result")] field_descriptor.id = "age" internal_form = Mock(name="internal_form_mock") internal_form.description.fields.items = Mock(return_value=[("age", field_descriptor)]) - internal_form.field.field_validator.requirement.check = Mock(return_value=True) field = Mock() field.description = field_descriptor field.value = 61 From d68814283fbab998711bc6bdd357bdd73a845670 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 20 Dec 2021 02:25:17 +0300 Subject: [PATCH 090/116] remove jaeger --- core/jaeger_custom_client/__init__.py | 0 core/jaeger_custom_client/jaeger_config.py | 47 ------- core/jaeger_custom_client/jaeger_utils.py | 39 ------ core/jaeger_custom_client/kafka_codec.py | 61 ---------- setup.py | 1 - smart_kit/start_points/base_main_loop.py | 9 -- smart_kit/start_points/main_loop_kafka.py | 115 ++++++++---------- .../static/configs/template_config.yml | 8 -- .../jaeger_custom_client_test/__init__.py | 0 .../jaeger_custom_client_test/test_codec.py | 62 ---------- 10 files changed, 52 insertions(+), 290 deletions(-) delete mode 100644 core/jaeger_custom_client/__init__.py delete mode 100644 core/jaeger_custom_client/jaeger_config.py delete mode 100644 core/jaeger_custom_client/jaeger_utils.py delete mode 100644 core/jaeger_custom_client/kafka_codec.py delete mode 100644 tests/core_tests/jaeger_custom_client_test/__init__.py delete mode 100644 tests/core_tests/jaeger_custom_client_test/test_codec.py diff --git a/core/jaeger_custom_client/__init__.py b/core/jaeger_custom_client/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/core/jaeger_custom_client/jaeger_config.py b/core/jaeger_custom_client/jaeger_config.py deleted file mode 100644 index 2b2c5aef..00000000 --- a/core/jaeger_custom_client/jaeger_config.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging - -from jaeger_client import Config, ConstSampler -from jaeger_client.reporter import NullReporter - -from core.jaeger_custom_client.kafka_codec import KafkaCodec, KAFKA_MAP - - -class ExtendedConfig(Config): - """ - Конфиг расширяет базовый класс, вводя поддержку KafkaCodec и возможность отключать трасировку - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.logger = logging.getLogger(__name__) - - def initialize_tracer(self, io_loop=None): - """ - Initialize Jaeger Tracer based on the passed `jaeger_client.Config`. - Save it to `opentracing.tracer` global variable. - Only the first call to this method has any effect. - """ - - with Config._initialized_lock: - if Config._initialized: - self.logger.warning('Jaeger tracer already initialized, skipping') - return - Config._initialized = True - - if self.enabled: - tracer = self.new_tracer(io_loop) - else: - reporter = NullReporter() - sampler = ConstSampler(False) - tracer = self.create_tracer(reporter, sampler, None) - - self._initialize_global_tracer(tracer=tracer) - return tracer - - @property - def propagation(self): - codec = KafkaCodec(url_encoding=False, - trace_id_header=self.trace_id_header, - baggage_header_prefix=self.baggage_header_prefix, - debug_id_header=self.debug_id_header) - return {KAFKA_MAP: codec} diff --git a/core/jaeger_custom_client/jaeger_utils.py b/core/jaeger_custom_client/jaeger_utils.py deleted file mode 100644 index 4b67ceef..00000000 --- a/core/jaeger_custom_client/jaeger_utils.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Dict, Any - -from confluent_kafka.cimpl import Message -from opentracing import Span - -from core.jaeger_custom_client.jaeger_config import KAFKA_MAP -from core.message.from_message import SmartAppFromMessage -import core.message.message_constants as const - - -def get_incoming_spam(tracer, from_message: SmartAppFromMessage, mq_message: Message) -> Span: - """ - Метод извлекает информацию об trace из входящего сообщения кафки. Созданный Span - обогащается тегами - :param tracer: Трейсер - :param from_message: Распарщенное сообщение - :param mq_message: Сообщение из кафки - :return: Проинициализированный и активированный Span - """ - span_context = tracer.extract(KAFKA_MAP, mq_message) - span = tracer.start_span( - operation_name=from_message.type, - child_of=span_context) - params = build_log_params(from_message) - for key, value in params.items(): - span.set_tag(key, value) - return span - - -def build_log_params(from_message: SmartAppFromMessage) -> Dict[str, Any]: - logging_dict = { - const.MSG_MESSAGEID_KEY: from_message.incremental_id, - const.MSG_USERCHANNEL_KEY: from_message.channel, - const.MSG_USERID_KEY: from_message.uid - } - chat_id = getattr(from_message, "chat_id", None) - if chat_id: - logging_dict[const.MSG_CHATID_KEY] = chat_id - return logging_dict diff --git a/core/jaeger_custom_client/kafka_codec.py b/core/jaeger_custom_client/kafka_codec.py deleted file mode 100644 index eedd9844..00000000 --- a/core/jaeger_custom_client/kafka_codec.py +++ /dev/null @@ -1,61 +0,0 @@ -import logging -from typing import List, Tuple, Optional - -from confluent_kafka.cimpl import Message -from jaeger_client import SpanContext -from jaeger_client.codecs import TextCodec -from jaeger_client.constants import TRACE_ID_HEADER, BAGGAGE_HEADER_PREFIX, DEBUG_ID_HEADER_KEY, BAGGAGE_HEADER_KEY - -KAFKA_MAP = "kafka_map" - - -class KafkaCodec(TextCodec): - """ - Кодек предназначен для серилизации/десерелизации трейса в/из заголовок(а) кафки - """ - VALUE_POS = 1 - KEY_POS = 0 - ENCODING = 'utf-8' - - def __init__(self, - url_encoding=False, - trace_id_header=TRACE_ID_HEADER, - baggage_header_prefix=BAGGAGE_HEADER_PREFIX, - debug_id_header=DEBUG_ID_HEADER_KEY, - baggage_header=BAGGAGE_HEADER_KEY): - - super().__init__(url_encoding, trace_id_header, baggage_header_prefix, debug_id_header, baggage_header) - - self.target_keys = {self.trace_id_header, self.baggage_prefix, self.debug_id_header, self.baggage_header} - self.__logger = logging.getLogger(__name__) - - def inject(self, span_context, carrier: List[Tuple[str, bytes]]): - if carrier is None: - carrier = [] - existing_keys = {} - for pos, el in enumerate(carrier): - existing_keys[el[KafkaCodec.KEY_POS]] = pos - - carrier_dict = {} - super().inject(span_context, carrier_dict) - - for key, value in carrier_dict.items(): - if key in existing_keys: - carrier[existing_keys[key]] = (key, value.encode(KafkaCodec.ENCODING)) - else: - carrier.append((key, value.encode(KafkaCodec.ENCODING))) - - def extract(self, carrier: Message) -> Optional[SpanContext]: - header = carrier.headers() - header_dict = {} - if header: - for el in header: - if el[KafkaCodec.KEY_POS] in self.target_keys: - header_dict[el[KafkaCodec.KEY_POS]] = el[KafkaCodec.VALUE_POS].decode(KafkaCodec.ENCODING) - span_context: SpanContext = None - try: - span_context = super().extract(header_dict) - except Exception: - self.__logger.exception("Could not extract SpanContext. Using new one") - - return span_context diff --git a/setup.py b/setup.py index d69a2526..c66bc503 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ "dawg==0.8.0", "dill==0.3.3", "ics==0.6", - "jaeger_client==4.3.0", "Jinja2==2.10.1", "keras==2.6.0", "lazy", diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index c8d0a74e..a341c3de 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -7,7 +7,6 @@ import scenarios.logging.logger_constants as log_const from core.db_adapter.db_adapter import DBAdapterException from core.db_adapter.db_adapter import db_adapter_factory -from core.jaeger_custom_client.jaeger_config import ExtendedConfig from core.logging.logger_utils import log from core.monitoring.monitoring import monitoring from core.monitoring.healthcheck_handler import RootResource @@ -56,7 +55,6 @@ def __init__( self.user_save_collisions_tries = max(save_tries, 1) self.health_check_server = self._create_health_check_server(template_settings) - self.tracer = self._create_jaeger_tracer(template_settings) if not template_settings["monitoring"].get("enabled"): monitoring.turn_off() else: @@ -95,13 +93,6 @@ def _create_health_check_server(self, settings): ) return health_check_server - def _create_jaeger_tracer(self, template_settings): - jaeger_config = template_settings["jaeger_config"] - config = ExtendedConfig(config=jaeger_config, service_name=self.app_name, - validate=True) - tracer = config.initialize_tracer() - return tracer - async def load_user(self, db_uid, message): db_data = None load_error = False diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 5824de4c..959e262a 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -13,8 +13,6 @@ from lazy import lazy import scenarios.logging.logger_constants as log_const -from core.jaeger_custom_client import jaeger_utils -from core.jaeger_custom_client import kafka_codec as jaeger_kafka_codec from core.logging.logger_utils import log, UID_STR, MESSAGE_ID_STR from core.message.from_message import SmartAppFromMessage @@ -329,70 +327,61 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para db_uid = message.db_uid - span = jaeger_utils.get_incoming_spam(self.tracer, message, mq_message) - - with self.tracer.scope_manager.activate(span, True) as scope: - with self.tracer.start_span('Loading time', child_of=scope.span) as span: - with StatsTimer() as load_timer: - user = await self.load_user(db_uid, message) - self.check_message_key(message, mq_message.key(), user) - stats += "Loading time from DB time: {} msecs\n".format(load_timer.msecs) - log_params["user_loading"] = load_timer.msecs - smart_kit_metrics.sampling_load_time(self.app_name, load_timer.secs) - - log( - "INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(" - "incoming_data)s", - params={log_const.KEY_NAME: "incoming_message", - "topic": mq_message.topic(), - "message_partition": mq_message.partition(), + with StatsTimer() as load_timer: + user = await self.load_user(db_uid, message) + self.check_message_key(message, mq_message.key(), user) + stats += "Loading time from DB time: {} msecs\n".format(load_timer.msecs) + log_params["user_loading"] = load_timer.msecs + smart_kit_metrics.sampling_load_time(self.app_name, load_timer.secs) + + log( + "INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(" + "incoming_data)s", + params={log_const.KEY_NAME: "incoming_message", + "topic": mq_message.topic(), + "message_partition": mq_message.partition(), + "message_key": mq_message.key(), + "kafka_key": kafka_key, + "incoming_data": str(message.masked_value), + "headers": str(mq_message.headers()), + "waiting_message": waiting_message_time, + "surface": message.device.surface, + MESSAGE_ID_STR: message.incremental_id}, + user=user + ) + + with StatsTimer() as script_timer: + commands = await self.model.answer(message, user) + + answers = self._generate_answers(user=user, commands=commands, message=message, + topic_key=topic_key, + kafka_key=kafka_key) + + stats += "Script time: {} msecs\n".format(script_timer.msecs) + log_params["script_time"] = script_timer.msecs + smart_kit_metrics.sampling_script_time(self.app_name, script_timer.secs) + + with StatsTimer() as save_timer: + user_save_ok = await self.save_user(db_uid, user, message) + + stats += "Saving user to DB time: {} msecs\n".format(save_timer.msecs) + log_params["user_saving"] = save_timer.msecs + smart_kit_metrics.sampling_save_time(self.app_name, save_timer.secs) + if not user_save_ok: + log("MainLoop.iterate: save user got collision on uid %(uid)s db_version %(db_version)s.", + user=user, + params={log_const.KEY_NAME: "ignite_collision", + "db_uid": db_uid, "message_key": mq_message.key(), + "message_partition": mq_message.partition(), "kafka_key": kafka_key, - "incoming_data": str(message.masked_value), - "headers": str(mq_message.headers()), - "waiting_message": waiting_message_time, - "surface": message.device.surface, - MESSAGE_ID_STR: message.incremental_id}, - user=user - ) - - with self.tracer.start_span('Script time', child_of=scope.span) as span: - with StatsTimer() as script_timer: - commands = await self.model.answer(message, user) - - answers = self._generate_answers(user=user, commands=commands, message=message, - topic_key=topic_key, - kafka_key=kafka_key) - - stats += "Script time: {} msecs\n".format(script_timer.msecs) - log_params["script_time"] = script_timer.msecs - smart_kit_metrics.sampling_script_time(self.app_name, script_timer.secs) - - with self.tracer.start_span('Saving time', child_of=scope.span) as span: - with StatsTimer() as save_timer: - user_save_ok = await self.save_user(db_uid, user, message) - - stats += "Saving user to DB time: {} msecs\n".format(save_timer.msecs) - log_params["user_saving"] = save_timer.msecs - smart_kit_metrics.sampling_save_time(self.app_name, save_timer.secs) - if not user_save_ok: - log("MainLoop.iterate: save user got collision on uid %(uid)s db_version %(db_version)s.", - user=user, - params={log_const.KEY_NAME: "ignite_collision", - "db_uid": db_uid, - "message_key": mq_message.key(), - "message_partition": mq_message.partition(), - "kafka_key": kafka_key, - "uid": user.id, - "db_version": str(user.variables.get(user.USER_DB_VERSION))}, - level="WARNING") - continue - - if answers: - self.save_behavior_timeouts(user, mq_message, kafka_key) + "uid": user.id, + "db_version": str(user.variables.get(user.USER_DB_VERSION))}, + level="WARNING") + continue - self.tracer.inject(span_context=span.context, format=jaeger_kafka_codec.KAFKA_MAP, - carrier=mq_message.headers()) + if answers: + self.save_behavior_timeouts(user, mq_message, kafka_key) if answers: for answer in answers: diff --git a/smart_kit/template/static/configs/template_config.yml b/smart_kit/template/static/configs/template_config.yml index b184eaa4..fa00902f 100644 --- a/smart_kit/template/static/configs/template_config.yml +++ b/smart_kit/template/static/configs/template_config.yml @@ -7,14 +7,6 @@ health_check: interface: 0.0.0.0 debug_envs: - local -jaeger_config: - service_name: template-service-name - logging: true - enabled: false - trace_id_header: template-app-trace-id - sampler: - type: const - param: true monitoring: enabled: false masking_fields: diff --git a/tests/core_tests/jaeger_custom_client_test/__init__.py b/tests/core_tests/jaeger_custom_client_test/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/core_tests/jaeger_custom_client_test/test_codec.py b/tests/core_tests/jaeger_custom_client_test/test_codec.py deleted file mode 100644 index fa5819d9..00000000 --- a/tests/core_tests/jaeger_custom_client_test/test_codec.py +++ /dev/null @@ -1,62 +0,0 @@ -import unittest -from unittest.mock import Mock - -from core.jaeger_custom_client.kafka_codec import KafkaCodec - -NLP_BAGGAGE = "nlp-baggage-" - -NLP_TRACE_ID = "nlp-trace-id" - - -class KafkaCodecTest(unittest.TestCase): - - def setUp(self) -> None: - self.codec = KafkaCodec(trace_id_header=NLP_TRACE_ID, baggage_header_prefix=NLP_BAGGAGE) - - def test_extract(self): - # Won't fail if there is a filter in KafkaCode extract method - trace_id = 15441712733567627997 - span_id = 957657810971653503 - parent_id = 0 - flags = 1 - - header = [('d', b'D'), ('cp', b'\xff'), ("b'0xc9'", b'0xc9'), ('q', b''), ('a', b'g '), - (NLP_TRACE_ID, b'd64bfbeadfecb2dd:d4a48b48757f57f:0:1')] - carrier = Mock() - carrier.headers.return_value = header - - actual = self.codec.extract(carrier) - self.assertEqual(actual.trace_id, trace_id) - self.assertEqual(actual.span_id, span_id) - self.assertIsNone(actual.parent_id) - self.assertEqual(actual.flags, flags) - - def test_extract_with_malformed_trace_id(self): - header = [('d', b'D'), ('cp', b'\xff'), ("b'0xc9'", b'0xc9'), ('q', b''), ('a', b'g '), - (NLP_TRACE_ID, b'this_is_malformed_trace_id')] - carrier = Mock() - carrier.headers.return_value = header - - actual = self.codec.extract(carrier) - self.assertIsNone(actual) - - def test_inject(self): - span_context = Mock() - span_context.trace_id = 15441712733567627997 - span_context.span_id = 957657810971658888 - span_context.parent_id = 957657810971653503 - span_context.flags = 1 - span_context.baggage = {"some_key": "56"} - baggage_key = f"{NLP_BAGGAGE}test" - header = [('d', b'D'), ('cp', b'\xff'), ("b'0xc9'", b'0xc9'), ('q', b''), - (NLP_TRACE_ID, b'd64bfbeadfecb2dd:d4a48b48757f57f:0:1'), ('a', b'g '), - (baggage_key, b'test_data')] - - expected_header = [('d', b'D'), ('cp', b'\xff'), ("b'0xc9'", b'0xc9'), ('q', b''), - (NLP_TRACE_ID, b'd64bfbeadfecb2dd:d4a48b487580a88:d4a48b48757f57f:1'), ('a', b'g '), - (baggage_key, b'test_data'), - (f"{NLP_BAGGAGE}some_key", b"56")] - self.codec.inject(span_context, header) - self.assertSequenceEqual(expected_header, header) - - From d451c74f2b227b524dd1363adb43bf5fbf03c0ab Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 20 Dec 2021 02:47:15 +0300 Subject: [PATCH 091/116] fix FirstPersonFiller --- scenarios/scenario_models/field/field_filler_description.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/scenario_models/field/field_filler_description.py b/scenarios/scenario_models/field/field_filler_description.py index 4f5da0e1..7c5df136 100644 --- a/scenarios/scenario_models/field/field_filler_description.py +++ b/scenarios/scenario_models/field/field_filler_description.py @@ -278,7 +278,7 @@ class FirstPersonFiller(FieldFillerDescription): @exc_handler(on_error_obj_method_name="on_extract_error") async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, user: User, - params: Dict[str, Any] = None) -> Optional[str]: + params: Dict[str, Any] = None) -> Optional[Dict[str, str]]: persons = text_preprocessing_result.person_token_values if persons: log_params = self._log_params() From e773dea10d1b6fc982dfb7b53651c63f766cbca8 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Mon, 20 Dec 2021 03:06:44 +0300 Subject: [PATCH 092/116] async DatePeriodFiller --- .../field/field_filler_description.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scenarios/scenario_models/field/field_filler_description.py b/scenarios/scenario_models/field/field_filler_description.py index 14713f93..b3c73f78 100644 --- a/scenarios/scenario_models/field/field_filler_description.py +++ b/scenarios/scenario_models/field/field_filler_description.py @@ -397,16 +397,16 @@ def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> self.max_days_in_period = items.get('max_days_in_period', None) self.future_days_allowed = items.get('future_days_allowed', False) - def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Dict: - if text_preprocessing_result\ - .words_tokenized_set\ - .intersection( - [ - 'TIME_DATE_TOKEN', - 'TIME_DATE_INTERVAL_TOKEN', - 'PERIOD_TOKEN' - ]): + async def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Dict: + if text_preprocessing_result \ + .words_tokenized_set \ + .intersection( + [ + 'TIME_DATE_TOKEN', + 'TIME_DATE_INTERVAL_TOKEN', + 'PERIOD_TOKEN' + ]): words_from_intent: List[Optional[str]] = text_preprocessing_result.human_normalized_text.lower().split() else: words_from_intent: List[Optional[str]] = text_preprocessing_result.original_text.lower().split() @@ -417,7 +417,7 @@ def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User is_determined: bool = False is_error: bool = False if not (begin_str == '' or begin_str == 'error' - or end_str == '' or end_str == 'error'): + or end_str == '' or end_str == 'error'): is_determined = True if begin_str == 'error' or end_str == 'error': @@ -571,6 +571,6 @@ async def extract(self, text_preprocessing_result: BaseTextPreprocessingResult, class ClassifierFillerMeta(ClassifierFiller): - def _get_result(self, answers: List[Dict[str, Union[str, float, bool]]])\ + def _get_result(self, answers: List[Dict[str, Union[str, float, bool]]]) \ -> List[Dict[str, Union[str, float, bool]]]: return answers From 4bd2e547b48c496520974dbd3e9a3b1c6cf4fb83 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 28 Dec 2021 10:27:02 +0300 Subject: [PATCH 093/116] fix DatePeriodFiller --- .../field/field_filler_description.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scenarios/scenario_models/field/field_filler_description.py b/scenarios/scenario_models/field/field_filler_description.py index b3c73f78..0d048225 100644 --- a/scenarios/scenario_models/field/field_filler_description.py +++ b/scenarios/scenario_models/field/field_filler_description.py @@ -398,7 +398,7 @@ def __init__(self, items: Optional[Dict[str, Any]], id: Optional[str] = None) -> self.future_days_allowed = items.get('future_days_allowed', False) async def extract(self, text_preprocessing_result: TextPreprocessingResult, user: User, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Dict: + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Dict[str, str]: if text_preprocessing_result \ .words_tokenized_set \ .intersection( @@ -423,10 +423,12 @@ async def extract(self, text_preprocessing_result: TextPreprocessingResult, user if begin_str == 'error' or end_str == 'error': is_error = True - user.variables.set('date_period__is_determined', str(is_determined)) - user.variables.set('date_period__is_error', str(is_error)) - user.variables.set('date_period__begin_date', begin_str) - user.variables.set('date_period__end_date', end_str) + return { + 'date_period__is_determined': is_determined, + 'date_period__is_error': is_error, + 'date_period__begin_date': begin_str, + 'date_period__end_date': end_str + } class IntersectionOriginalTextFiller(FieldFillerDescription): From 3c33f2e4ed8de27acdca47582225025e7cac7a4b Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 28 Dec 2021 10:46:31 +0300 Subject: [PATCH 094/116] refactoring of AIOHttpMainLoop --- .../start_points/main_loop_async_http.py | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/smart_kit/start_points/main_loop_async_http.py b/smart_kit/start_points/main_loop_async_http.py index 2184d2c9..e34c30ca 100644 --- a/smart_kit/start_points/main_loop_async_http.py +++ b/smart_kit/start_points/main_loop_async_http.py @@ -2,7 +2,6 @@ import os import asyncio -import concurrent.futures import aiohttp import aiohttp.web @@ -18,14 +17,13 @@ class AIOHttpMainLoop(BaseHttpMainLoop): def __init__(self, *args, **kwargs): - self.app = aiohttp.web.Application() - self.app.add_routes([aiohttp.web.route('*', '/{tail:.*}', self.iterate)]) super().__init__(*args, **kwargs) max_workers = self.settings["template_settings"].get("max_workers", (os.cpu_count() or 1) * 5) - self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) + self.app = aiohttp.web.Application() + self.app.add_routes([aiohttp.web.route('*', '/health', self.get_health_check)]) + self.app.add_routes([aiohttp.web.route('*', '/{tail:.*}', self.iterate)]) - async def async_init(self): - await self.db_adapter.connect() + async def async_init(self):await self.db_adapter.connect() def get_db(self): db_adapter = db_adapter_factory(self.settings["template_settings"].get("db_adapter", {})) @@ -113,19 +111,33 @@ def stop(self, signum, frame): async def handle_message(self, message: SmartAppFromMessage) -> typing.Tuple[int, str, SmartAppToMessage]: if not message.validate(): - return 400, "BAD REQUEST", SmartAppToMessage(self.BAD_REQUEST_COMMAND, message=message, request=None) + answer = SmartAppToMessage(self.BAD_REQUEST_COMMAND, message=message, request=None) + code = 400 + log(f"OUTGOING DATA: {answer.value} with code: {code}", + params={log_const.KEY_NAME: "outgoing_policy_message", "msg_id": message.incremental_id}) + return code, "BAD REQUEST", answer - answer, stats = await self.process_message(message) + answer, stats, user = await self.process_message(message) if not answer: - return 204, "NO CONTENT", SmartAppToMessage(self.NO_ANSWER_COMMAND, message=message, request=None) + answer = SmartAppToMessage(self.NO_ANSWER_COMMAND, message=message, request=None) + code = 204 + log(f"OUTGOING DATA: {answer.value} with code: {code}", + params={log_const.KEY_NAME: "outgoing_policy_message"}, user=user) + return code, "NO CONTENT", answer - answer_message = SmartAppToMessage( - answer, message, request=None, - validators=self.to_msg_validators) + answer_message = SmartAppToMessage(answer, message, request=None, validators=self.to_msg_validators) if answer_message.validate(): - return 200, "OK", answer_message + code = 200 + log_answer = str(answer_message.value).replace("%", "%%") + log(f"OUTGOING DATA: {log_answer} with code: {code}", + params={log_const.KEY_NAME: "outgoing_policy_message"}, user=user) + return code, "OK", answer_message else: - return 500, "BAD ANSWER", SmartAppToMessage(self.BAD_ANSWER_COMMAND, message=message, request=None) + code = 500 + answer = SmartAppToMessage(self.BAD_ANSWER_COMMAND, message=message, request=None) + log(f"OUTGOING DATA: {answer.value} with code: {code}", + params={log_const.KEY_NAME: "outgoing_policy_message"}, user=user) + return code, "BAD ANSWER", answer async def process_message(self, message: SmartAppFromMessage, *args, **kwargs): stats = "" @@ -137,7 +149,7 @@ async def process_message(self, message: SmartAppFromMessage, *args, **kwargs): user = await self.load_user(db_uid, message) stats += "Loading time: {} msecs\n".format(load_timer.msecs) with StatsTimer() as script_timer: - commands = await self.app.loop.run_in_executor(self.pool, self.model.answer, message, user) + commands = await self.model.answer(message, user) if commands: answer = self._generate_answers(user, commands, message) else: @@ -148,7 +160,13 @@ async def process_message(self, message: SmartAppFromMessage, *args, **kwargs): await self.save_user(db_uid, user, message) stats += "Saving time: {} msecs\n".format(save_timer.msecs) log(stats, params={log_const.KEY_NAME: "timings"}) - return answer, stats + return answer, stats, user + + async def get_health_check(self, request: aiohttp.web.Request): + status, reason, answer = 200, "OK", "ok" + return aiohttp.web.json_response( + status=status, reason=reason, data=answer, + ) async def iterate(self, request: aiohttp.web.Request): headers = self._get_headers(request.headers) From 5165e6fa35c2ea3c77d7eee8502621b48ae32bbf Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Tue, 28 Dec 2021 10:47:16 +0300 Subject: [PATCH 095/116] refactoring of AIOHttpMainLoop --- smart_kit/start_points/main_loop_async_http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/smart_kit/start_points/main_loop_async_http.py b/smart_kit/start_points/main_loop_async_http.py index e34c30ca..8bea82d6 100644 --- a/smart_kit/start_points/main_loop_async_http.py +++ b/smart_kit/start_points/main_loop_async_http.py @@ -18,7 +18,6 @@ class AIOHttpMainLoop(BaseHttpMainLoop): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - max_workers = self.settings["template_settings"].get("max_workers", (os.cpu_count() or 1) * 5) self.app = aiohttp.web.Application() self.app.add_routes([aiohttp.web.route('*', '/health', self.get_health_check)]) self.app.add_routes([aiohttp.web.route('*', '/{tail:.*}', self.iterate)]) From ccffc857f22a006bbfa19e635ff79b2e27511ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D0=B5=D0=B2=D1=87=D0=B5=D0=BD=D0=BA=D0=BE=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D0=B0=D1=80=20=D0=98=D0=BB=D1=8C=D0=B8=D1=87?= Date: Mon, 7 Feb 2022 10:56:39 +0300 Subject: [PATCH 096/116] DPNLPF-1300: sync kafka main loop with DP version --- core/message/from_message.py | 4 + core/mq/kafka/async_kafka_publisher.py | 66 +++++ core/utils/memstats.py | 26 ++ core/utils/utils.py | 10 + smart_kit/start_points/base_main_loop.py | 1 - smart_kit/start_points/main_loop_kafka.py | 322 +++++++++++++++------- 6 files changed, 321 insertions(+), 108 deletions(-) create mode 100644 core/mq/kafka/async_kafka_publisher.py diff --git a/core/message/from_message.py b/core/message/from_message.py index a0a80607..df034773 100644 --- a/core/message/from_message.py +++ b/core/message/from_message.py @@ -226,6 +226,10 @@ def callback_id(self): def callback_id(self, value): self._callback_id = value + @property + def has_callback_id(self): + return self._callback_id is not None or self.headers.get(self._callback_id_header_name) is not None + # noinspection PyMethodMayBeStatic def generate_new_callback_id(self): return str(uuid.uuid4()) diff --git a/core/mq/kafka/async_kafka_publisher.py b/core/mq/kafka/async_kafka_publisher.py new file mode 100644 index 00000000..76ad269b --- /dev/null +++ b/core/mq/kafka/async_kafka_publisher.py @@ -0,0 +1,66 @@ +# coding: utf-8 +from threading import Thread + +import core.logging.logger_constants as log_const +from core.logging.logger_utils import log +from core.monitoring.monitoring import monitoring +from core.mq.kafka.kafka_publisher import KafkaPublisher + + +class AsyncKafkaPublisher(KafkaPublisher): + def __init__(self, config): + super().__init__(config) + self._cancelled = False + self._poll_thread = Thread(target=self._poll_for_callbacks) + self._poll_thread.start() + + def send(self, value, key=None, topic_key=None, headers=None): + try: + topic = self._config["topic"] + if topic_key is not None: + topic = topic[topic_key] + producer_params = dict() + if key is not None: + producer_params["key"] = key + self._producer.produce(topic=topic, value=value, headers=headers or [], **producer_params) + except BufferError as e: + params = { + "queue_amount": len(self._producer), + log_const.KEY_NAME: log_const.EXCEPTION_VALUE + } + log("KafkaProducer: Local producer queue is full (%(queue_amount)s messages awaiting delivery):" + " try again\n", params=params, level="ERROR") + monitoring.got_counter("kafka_producer_exception") + + def send_to_topic(self, value, key=None, topic=None, headers=None): + try: + if topic is None: + params = { + "message": str(value), + log_const.KEY_NAME: log_const.EXCEPTION_VALUE + } + log("KafkaProducer: Failed sending message %{message}s. Topic is not defined", params=params, + level="ERROR") + producer_params = dict() + if key is not None: + producer_params["key"] = key + self._producer.produce(topic=topic, value=value, headers=headers or [], **producer_params) + except BufferError as e: + params = { + "queue_amount": len(self._producer), + log_const.KEY_NAME: log_const.EXCEPTION_VALUE + } + log("KafkaProducer: Local producer queue is full (%(queue_amount)s messages awaiting delivery):" + " try again\n", params=params, level="ERROR") + monitoring.got_counter("kafka_producer_exception") + + def _poll_for_callbacks(self): + poll_timeout = self._config.get("poll_timeout", 1) + while not self._cancelled: + self._producer.poll(poll_timeout) + + def close(self): + self._producer.flush(self._config["flush_timeout"]) + self._cancelled = True + self._poll_thread.join() + log(f"KafkaProducer.close: producer to {self._config['topic']} flushed, poll_thread joined.") diff --git a/core/utils/memstats.py b/core/utils/memstats.py index 501c2c8c..040c99a5 100644 --- a/core/utils/memstats.py +++ b/core/utils/memstats.py @@ -1,4 +1,6 @@ import os +import tracemalloc + import objgraph import psutil import time @@ -22,6 +24,30 @@ def get_leaking_objects(file=None, limit=5): objgraph.show_refs(roots[:limit], refcounts=True, shortnames=False, output=file) +def get_top_malloc(trace_limit=3): + snapshot = tracemalloc.take_snapshot() + snapshot = snapshot.filter_traces(( + tracemalloc.Filter(False, ""), + tracemalloc.Filter(False, ""), + )) + top_stats = snapshot.statistics('traceback') + msg = "" + + if trace_limit > 0: + msg += f"Top malloc {trace_limit} lines\n" + for index, stat in enumerate(top_stats[:trace_limit], 1): + msg += f"#{index}: {stat.size // 1024} KB, {stat.count} times\n" + for line in stat.traceback.format(limit=16): + msg += f"{line}\n" + other = top_stats[trace_limit:] + if other: + size = sum(stat.size for stat in other) + msg += f"{len(other)} other: {size // 1024} KB\n" + total = sum(stat.size for stat in top_stats) + msg += f"Total allocated size: {total // 1024 // 1024} MB" + return msg + + if __name__ == "__main__": while 1 : print(show_most_common_types()) diff --git a/core/utils/utils.py b/core/utils/utils.py index 273191be..4e97273e 100644 --- a/core/utils/utils.py +++ b/core/utils/utils.py @@ -1,14 +1,18 @@ # coding=utf-8 import datetime +import gc import json import os import re +import weakref from collections import OrderedDict from math import isnan, isinf from typing import Optional from time import time +from scenarios.user.user_model import User + def convert_version_to_list_of_int(version): if not version or re.search("[0-9]", version) is None: @@ -126,3 +130,9 @@ def current_time_ms(): def time_check(begin_time, reject_timeout): return current_time_ms() - begin_time <= reject_timeout if begin_time is not None else True + +def get_user_number(): + return sum( + not isinstance(o, weakref.ProxyType) and isinstance(o, User) + for o in gc.get_objects() + ) diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index a341c3de..4f4715ad 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -121,7 +121,6 @@ async def save_user(self, db_uid, user, message): log("User %(uid)s will not saved", user=user, params={"uid": user.id, log_const.KEY_NAME: "user_will_not_saved"}) else: - no_collisions = True try: str_data = user.raw_str diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 8cb7ff8f..00d9d5e2 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -1,11 +1,15 @@ # coding=utf-8 import asyncio import cProfile +import gc +import hashlib import json import pstats +import random import signal import time import concurrent.futures +import tracemalloc from collections import namedtuple from functools import lru_cache @@ -17,10 +21,12 @@ from core.message.from_message import SmartAppFromMessage from core.model.heapq.heapq_storage import HeapqKV +from core.mq.kafka.async_kafka_publisher import AsyncKafkaPublisher from core.mq.kafka.kafka_consumer import KafkaConsumer -from core.mq.kafka.kafka_publisher import KafkaPublisher +from core.utils.memstats import get_top_malloc from core.utils.stats_timer import StatsTimer from core.basic_models.actions.command import Command +from core.utils.utils import get_user_number from smart_kit.compatibility.commands import combine_commands from smart_kit.message.get_to_message import get_to_message from smart_kit.message.smartapp_to_message import SmartAppToMessage @@ -39,17 +45,24 @@ def _enrich_config_from_secret(kafka_config, secret_config): class MainLoop(BaseMainLoop): + # in milliseconds. log event if elapsed time more than value MAX_LOG_TIME = 20 BAD_ANSWER_COMMAND = Command(message_names.ERROR, {"code": -1, "description": "Invalid Answer Message"}) def __init__(self, *args, **kwargs): log("%(class_name)s.__init__ started.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) + self.loop = asyncio.get_event_loop() + # We have many async loops for messages processing in main thread + # And 1 thread for independent consecutive Kafka reading self.health_check_server_future = None super().__init__(*args, **kwargs) # We have many async loops for messages processing in main thread # And 1 thread for independent consecutive Kafka reading self.kafka_executor_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) + self._timers = dict() # stores aio timers for callbacks + self.template_settings = self.settings["template_settings"] + self.worker_tasks = [] try: kafka_config = _enrich_config_from_secret( @@ -66,7 +79,7 @@ def __init__(self, *args, **kwargs): if config.get("consumer"): consumers.update({key: KafkaConsumer(kafka_config[key])}) if config.get("publisher"): - publishers.update({key: KafkaPublisher(kafka_config[key])}) + publishers.update({key: AsyncKafkaPublisher(kafka_config[key])}) log( "%(class_name)s FINISHED CONSUMERS/PUBLISHERS CREATE", params={"class_name": self.__class__.__name__}, level="WARNING" @@ -79,9 +92,11 @@ def __init__(self, *args, **kwargs): self.publishers = publishers self.concurrent_messages = 0 + # Missing in DP self.behaviors_timeouts_value_cls = namedtuple('behaviors_timeouts_value', 'db_uid, callback_id, mq_message, kafka_key') self.behaviors_timeouts = HeapqKV(value_to_key_func=lambda val: val.callback_id) + # End: Missing in DP log("%(class_name)s.__init__ completed.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) @@ -91,22 +106,24 @@ def __init__(self, *args, **kwargs): level="ERROR", exc_info=True) raise - # TODO find where it should be used - async def pre_handle(self): - await self.iterate_behavior_timeouts() - def run(self): signal.signal(signal.SIGINT, self.stop) signal.signal(signal.SIGTERM, self.stop) log("%(class_name)s.run started", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) - try: - self.loop.run_until_complete(self.general_coro()) - except (SystemExit,): - self.loop.stop() - log("MainLoop stopped") - log("%(class_name)s.run stopped", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, - "class_name": self.__class__.__name__}) + # try: + loop = asyncio.get_event_loop() + loop.run_until_complete(self.general_coro()) + + log("MainLoop stopping kafka", level="WARNING") + + for kafka_key in self.consumers: + self.consumers[kafka_key].close() + for kafka_key in self.publishers: + self.publishers[kafka_key].close() + # except (SystemExit,) as e: + log("MainLoop EXIT.", level="WARNING") + # raise e async def general_coro(self): tasks = [self.process_consumer(kafka_key) for kafka_key in self.consumers] @@ -120,42 +137,62 @@ async def healthcheck_coro(self): self.health_check_server_future.cancelled(): self.health_check_server_future = self.loop.run_in_executor(None, self.health_check_server.iterate) await asyncio.sleep(0.5) + log("healthcheck_coro stopped") async def process_consumer(self, kafka_key): consumer = self.consumers[kafka_key] loop = asyncio.get_event_loop() - max_concurrent_messages = self.settings["template_settings"].get("max_concurrent_messages", 10) + max_concurrent_messages = self.template_settings.get("max_concurrent_messages", 1) total_messages = 0 - async def msg_loop(iteration): + profiling_settings = self.template_settings.get("profiling", {}) + profile_cpu = profiling_settings.get("cpu", False) + profile_cpu_path = profiling_settings.get("cpu_path", "/tmp/dp.cpu.prof") + profile_memory = profiling_settings.get("memory", False) + profile_memory_log_delta = profiling_settings.get("memory_log_delta", 30) + profile_memory_depth = profiling_settings.get("memory_depth", 4) + + async def worker(iteration, queue): nonlocal total_messages message_value = None - last_poll_begin_time = time.time() - print(f"\n-- Starting {iteration} iter\n") + user = None + validation_failed = False + last_poll_begin_time = self.loop.time() + last_mem_log = self.loop.time() + log(f"-- Starting {iteration} iter") while self.is_work: - from_last_poll_begin_ms = int((time.time() - last_poll_begin_time) * 1000) - stats = "From last poll time: {} msecs\n".format(from_last_poll_begin_ms) + if profile_memory and iteration == 0 and self.loop.time() - last_mem_log > profile_memory_log_delta: + top = get_top_malloc(trace_limit=0) + async_counts = len(self.loop._ready), len(self.loop._scheduled), len(self.loop._asyncgens) + async_values = " + ".join(map(str, async_counts)) + log( + f"Total memory: {top}; " + f"Async: {async_values} = {sum(async_counts)}; " + f"User number: {get_user_number()}; " + f"Trash: {gc.get_count()} ", + level="DEBUG" + ) + last_mem_log = self.loop.time() + + from_last_poll_begin_ms = int((self.loop.time() - last_poll_begin_time) * 1000) + stats = f"From last message coro {iteration} time: {from_last_poll_begin_ms} msecs\n" log_params = { log_const.KEY_NAME: "timings", - "from_last_poll_begin_ms": from_last_poll_begin_ms + "from_last_poll_begin_ms": from_last_poll_begin_ms, + "iteration": iteration } - last_poll_begin_time = time.time() + last_poll_begin_time = self.loop.time() try: + mq_message = await queue.get() self.concurrent_messages += 1 - mq_message = None - with StatsTimer() as poll_timer: - # Max delay between polls configured in consumer.poll_timeout param - mq_message = await loop.run_in_executor(self.kafka_executor_pool, consumer.poll) if mq_message: print(f"\n-- Processing {self.concurrent_messages} msgs at {iteration} iter\n") total_messages += 1 headers = mq_message.headers() if headers is None: raise Exception("No incoming message headers found.") - stats += "Polling time: {} msecs\n".format(poll_timer.msecs) - log_params["kafka_polling"] = poll_timer.msecs message_value = json.loads(mq_message.value()) await self.process_message(mq_message, consumer, kafka_key, stats, log_params) @@ -166,6 +203,7 @@ async def msg_loop(iteration): "kafka_exp": str(kafka_exp), log_const.REQUEST_VALUE: str(message_value)}, level="ERROR", exc_info=True) + queue.task_done() except Exception: self.concurrent_messages -= 1 @@ -179,23 +217,84 @@ async def msg_loop(iteration): except Exception: log("Error handling worker fail exception.", level="ERROR", exc_info=True) raise + queue.task_done() else: self.concurrent_messages -= 1 + queue.task_done() - print(f"-- Process Consumer enter with {max_concurrent_messages} loops") - start_time = time.time() - pr = cProfile.Profile() - pr.enable() + # END while self.is_work + log(f"-- Stop {iteration} iter") - await asyncio.gather(*(msg_loop(i) for i in range(max_concurrent_messages))) + start_time = self.loop.time() + if profile_cpu: + cpu_pr = cProfile.Profile() + cpu_pr.enable() + else: + cpu_pr = None + if profile_memory: + tracemalloc.start(profile_memory_depth) - time_delta = time.time() - start_time - print(f"-- Process Consumer exit: {total_messages} msg in {int(time_delta)} sec") - pr.disable() - stats = pstats.Stats(pr) - stats.sort_stats(pstats.SortKey.TIME) - stats.print_stats(10) - stats.dump_stats(filename="dp.prof.log") + log(f"Starting %(class_name)s in {max_concurrent_messages} coro", + params={"class_name": self.__class__.__name__}) + + # TODO: think about queue maxsize + queues = [asyncio.Queue() for _ in range(max_concurrent_messages)] + + for i, queue in enumerate(queues): + task = asyncio.create_task(worker(f'worker-{i}', queue)) + self.worker_tasks.append(task) + + await self.poll_kafka(consumer, queues) # blocks while self.is_works + + log("waiting for process unfinished tasks in queues") + await asyncio.gather(*(queue.join() for queue in queues)) + + time_delta = self.loop.time() - start_time + log(f"Process Consumer exit: {total_messages} msg in {int(time_delta)} sec", level="DEBUG") + + t = self.loop.time() + delay = self.template_settings.get("behavior_timers_tear_down_delay", 15) + log(f"wait timers to do their jobs for {delay} secs...") + while self._timers and (self.loop.time() - t) < delay: + await asyncio.sleep(1) + + for task in self.worker_tasks: + cancell_status = task.cancel() + log(f"{task} cancell status: {cancell_status} ") + + log(f"Stop consuming messages. All workers closed, erasing {len(self._timers)} timers.") + + if profile_memory: + log(f"{get_top_malloc(trace_limit=16)}") + tracemalloc.stop() + if cpu_pr is not None: + cpu_pr.disable() + stats = pstats.Stats(cpu_pr) + stats.sort_stats(pstats.SortKey.TIME) + stats.print_stats(10) + stats.dump_stats(filename=profile_cpu_path) + + async def poll_kafka(self, consumer, queues): + while self.is_work: + with StatsTimer() as poll_timer: + # Max delay between polls configured in consumer.poll_timeout param + mq_message = consumer.poll() + if poll_timer.msecs > self.MAX_LOG_TIME: + log_params = {"kafka_polling": poll_timer.msecs} + log(f"Long poll time: %(kafka_polling)s msecs\n", params=log_params, level="WARNING") + + if mq_message: + key = mq_message.key() + if key: + queue_index = int(hashlib.sha1(key).hexdigest(), 16) % len(queues) + else: + queue_index = random.randrange(len(queues)) + + # this will block if queue is full! + await queues[queue_index].put(mq_message) + else: + await asyncio.sleep(self.template_settings.get("no_kafka_messages_poll_time", 0.01)) + log(f"Stop poll_kafka consumer.") def _generate_answers(self, user, commands, message, **kwargs): topic_key = kwargs["topic_key"] @@ -304,25 +403,28 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para topic_key = self._get_topic_key(mq_message, kafka_key) save_tries = 0 user_save_ok = False + skip_timeout = False db_uid = None validation_failed = False + message_handled_ok = False while save_tries < self.user_save_collisions_tries and not user_save_ok: save_tries += 1 message_value = mq_message.value() - message = SmartAppFromMessage(message_value, - headers=mq_message.headers(), + message = SmartAppFromMessage(message_value, headers=mq_message.headers(), masking_fields=self.masking_fields, creation_time=consumer.get_msg_create_time(mq_message)) - # TODO вернуть проверку ключа!!! if message.validate(): + log( + "Incoming RAW message: %(message)s", params={"message": message.masked_value}, + level="DEBUG") waiting_message_time = 0 if message.creation_time: - waiting_message_time = time.time() * 1000 - message.creation_time - stats += "Waiting message: {} msecs\n".format(waiting_message_time) + waiting_message_time = self.loop.time() * 1000 - message.creation_time + stats += f"Waiting message: {waiting_message_time} msecs\n" log_params["waiting_message"] = waiting_message_time - stats += "Mid: {}\n".format(message.incremental_id) + stats += f"Mid: {message.incremental_id}\n" log_params[MESSAGE_ID_STR] = message.incremental_id smart_kit_metrics.sampling_mq_waiting_time(self.app_name, waiting_message_time / 1000) @@ -336,34 +438,19 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para with StatsTimer() as load_timer: user = await self.load_user(db_uid, message) self.check_message_key(message, mq_message.key(), user) - stats += "Loading time from DB time: {} msecs\n".format(load_timer.msecs) + stats += f"Loading user time from DB time: {load_timer.msecs} msecs\n" log_params["user_loading"] = load_timer.msecs smart_kit_metrics.sampling_load_time(self.app_name, load_timer.secs) - log( - "INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(" - "incoming_data)s", - params={log_const.KEY_NAME: "incoming_message", - "topic": mq_message.topic(), - "message_partition": mq_message.partition(), - "message_key": mq_message.key(), - "kafka_key": kafka_key, - "incoming_data": str(message.masked_value), - "headers": str(mq_message.headers()), - "waiting_message": waiting_message_time, - "surface": message.device.surface, - MESSAGE_ID_STR: message.incremental_id}, - user=user - ) + self._incoming_message_log(user, mq_message, message, kafka_key, waiting_message_time) with StatsTimer() as script_timer: commands = await self.model.answer(message, user) - answers = self._generate_answers(user=user, commands=commands, message=message, - topic_key=topic_key, + answers = self._generate_answers(user=user, commands=commands, message=message, topic_key=topic_key, kafka_key=kafka_key) - stats += "Script time: {} msecs\n".format(script_timer.msecs) + stats += f"Script time: {script_timer.msecs} msecs\n" log_params["script_time"] = script_timer.msecs smart_kit_metrics.sampling_script_time(self.app_name, script_timer.secs) @@ -378,7 +465,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, - "message_key": mq_message.key(), + "message_key": (mq_message.key() or b"").decode('utf-8', 'backslashreplace'), "message_partition": mq_message.partition(), "kafka_key": kafka_key, "uid": user.id, @@ -386,6 +473,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para level="WARNING") continue + message_handled_ok = True if answers: self.save_behavior_timeouts(user, mq_message, kafka_key) @@ -394,18 +482,21 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para with StatsTimer() as publish_timer: self._send_request(user, answer, mq_message) smart_kit_metrics.counter_outgoing(self.app_name, answer.command.name, answer, user) - stats += "Publishing to Kafka time: {} msecs".format(publish_timer.msecs) + stats += "Publishing to Kafka time: {publish_timer.msecs} msecs\n" log_params["kafka_publishing"] = publish_timer.msecs - log(stats) else: validation_failed = True + data = None + mid = None try: data = message.masked_value + mid = message.incremental_id except: - data = "" + pass log(f"Message validation failed, skip message handling.", params={log_const.KEY_NAME: "invalid_message", - "data": data}, level="ERROR") + "data": data, + MESSAGE_ID_STR: mid}, level="ERROR") smart_kit_metrics.counter_invalid_message(self.app_name) break if stats: @@ -416,7 +507,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para user=user, params={log_const.KEY_NAME: "ignite_collision", "db_uid": db_uid, - "message_key": mq_message.key(), + "message_key": (mq_message.key() or b"").decode('utf-8', 'backslashreplace'), "message_partition": mq_message.partition(), "kafka_key": kafka_key, "uid": user.id, @@ -425,6 +516,16 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para smart_kit_metrics.counter_save_collision_tries_left(self.app_name) consumer.commit_offset(mq_message) + if message_handled_ok: + self.remove_timer(message) + + def remove_timer(self, kafka_message): + if kafka_message.has_callback_id: + timer = self._timers.pop(kafka_message.callback_id, None) + if timer is not None: + log(f"Removing aio timer for callback {kafka_message.callback_id}. Have {len(self._timers)} running " + f"timers.", level="DEBUG") + timer.cancel() def _is_message_timeout_to_skip(self, message, waiting_message_time): # Returns True if timeout is found @@ -460,31 +561,28 @@ def check_message_key(self, from_message, message_key, user): sub = from_message.sub channel = from_message.channel uid = from_message.uid - message_key = message_key or b"" + valid_key = "_".join([i for i in [channel, sub, uid] if i]) + try: - params = [channel, sub, uid] - valid_key = "" - for value in params: - if value: - valid_key = "{}{}{}".format(valid_key, "_", value) if valid_key else "{}".format(value) - key_str = message_key.decode() - - message_key_is_valid = key_str == valid_key - if not message_key_is_valid: - log(f"Failed to check Kafka message key {message_key} != {valid_key}", - params={ - log_const.KEY_NAME: "check_kafka_key_validation", - MESSAGE_ID_STR: from_message.incremental_id, - UID_STR: uid - }, user=user, - level="WARNING") - except: - log(f"Exception to check Kafka message key {message_key}", + message_key = message_key or b"" + if isinstance(message_key, bytes): + message_key = message_key.decode() + except UnicodeDecodeError: + log(f"Decode error to check Kafka message key {message_key}", params={log_const.KEY_NAME: "check_kafka_key_error", MESSAGE_ID_STR: from_message.incremental_id, UID_STR: uid }, user=user, level="ERROR") + if message_key != valid_key: + log(f"Failed to check Kafka message key {message_key} != {valid_key}", + params={ + log_const.KEY_NAME: "check_kafka_key_validation", + MESSAGE_ID_STR: from_message.incremental_id, + UID_STR: uid + }, user=user, + level="WARNING") + def _send_request(self, user, answer, mq_message): kafka_broker_settings = self.settings["template_settings"].get( "route_kafka_broker" @@ -527,35 +625,26 @@ def masking_fields(self): return self.settings["template_settings"].get("masking_fields") def save_behavior_timeouts(self, user, mq_message, kafka_key): - for i, (timeout, callback_id) in enumerate(user.behaviors.get_behavior_timeouts()): + for i, (expire_time_us, callback_id) in enumerate(user.behaviors.get_behavior_timeouts()): + # TODO: remove "- time()" when framework will be modified to asyncio only # если колбеков много, разносим их на 1 секунду друг от друга во избежание коллизий - timeout = timeout + i + when = expire_time_us - time.time() + i log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout in %(timeout)s seconds.", params={log_const.KEY_NAME: "adding_local_timeout", "class_name": self.__class__.__name__, "callback_id": callback_id, - "timeout": timeout}) + "when": when}) - self.loop.call_later( - timeout, - self.loop.create_task, + self._timers[callback_id] = self.loop.call_later( + when, self.loop.create_task, self.do_behavior_timeout(user.message.db_uid, callback_id, mq_message, kafka_key) ) def stop(self, signum, frame): - log("Stop call!") - log("Stopping Kafka handler", level="WARNING") - for kafka_key in self.consumers: - self.consumers[kafka_key].close() - log("Kafka consumer connection is closed", level="WARNING") - self.publishers[kafka_key].close() - log("Kafka publisher connection is closed", level="WARNING") - log("Kafka handler is stopped", level="WARNING") + log("Stop signal handler!") self.is_work = False async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): - if not self.is_work: - return try: save_tries = 0 user_save_ok = False @@ -563,8 +652,7 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): user = None while save_tries < self.user_save_collisions_tries and not user_save_ok: callback_found = False - log( - f"MainLoop.do_behavior_timeout: handling callback {callback_id}. for db_uid {db_uid}. try " + log(f"MainLoop.do_behavior_timeout: handling callback {callback_id}. for db_uid {db_uid}. try " f"{save_tries}.") save_tries += 1 @@ -577,6 +665,9 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): user = await self.load_user(db_uid, timeout_from_message) # TODO: not to load user to check behaviors.has_callback ? + + self.remove_timer(timeout_from_message) + if user.behaviors.has_callback(callback_id): callback_found = True commands = await self.model.answer(timeout_from_message, user) @@ -624,3 +715,20 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): "class_name": self.__class__.__name__, log_const.REQUEST_VALUE: str(mq_message.value())}, level="ERROR", exc_info=True) + + def _incoming_message_log(self, user, mq_message, message, kafka_key, waiting_message_time): + log( + "INCOMING FROM TOPIC: %(topic)s partition %(message_partition)s HEADERS: %(headers)s DATA: %(" + "incoming_data)s", + params={log_const.KEY_NAME: "incoming_message", + "topic": mq_message.topic(), + "message_partition": mq_message.partition(), + "message_key": mq_message.key(), + "kafka_key": kafka_key, + "incoming_data": str(message.masked_value), + "headers": str(mq_message.headers()), + "waiting_message": waiting_message_time, + "surface": message.device.surface, + MESSAGE_ID_STR: message.incremental_id}, + user=user + ) From a9ff61775aa896c55a699b2ed6a966d2dd9ebc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D0=B5=D0=B2=D1=87=D0=B5=D0=BD=D0=BA=D0=BE=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D0=B0=D1=80=20=D0=98=D0=BB=D1=8C=D0=B8=D1=87?= Date: Mon, 7 Feb 2022 11:37:53 +0300 Subject: [PATCH 097/116] DPNLPF-1300: fix --- smart_kit/compatibility/commands.py | 2 -- smart_kit/handlers/handler_timeout.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/smart_kit/compatibility/commands.py b/smart_kit/compatibility/commands.py index 69949cce..80e8577f 100644 --- a/smart_kit/compatibility/commands.py +++ b/smart_kit/compatibility/commands.py @@ -9,8 +9,6 @@ def combine_answer_to_user(commands: typing.List[Command]) -> Command: - from smart_kit.configs import get_app_config - config = get_app_config() answer = Command(name=ANSWER_TO_USER, request_data=commands[0].request_data, request_type=commands[0].request_type) summary_pronounce_text = [] diff --git a/smart_kit/handlers/handler_timeout.py b/smart_kit/handlers/handler_timeout.py index da5b3fb7..224bdad9 100644 --- a/smart_kit/handlers/handler_timeout.py +++ b/smart_kit/handlers/handler_timeout.py @@ -28,5 +28,5 @@ async def run(self, payload, user): user, app_info=app_info) callback_id = user.message.callback_id - result = user.behaviors.timeout(callback_id) + result = await user.behaviors.timeout(callback_id) return result From df1c9e10b5a2f29fde968684b926210c95df3938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D0=B5=D0=B2=D1=87=D0=B5=D0=BD=D0=BA=D0=BE=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D0=B0=D1=80=20=D0=98=D0=BB=D1=8C=D0=B8=D1=87?= Date: Mon, 7 Feb 2022 14:29:18 +0300 Subject: [PATCH 098/116] DPNLPF-1300: fix --- smart_kit/start_points/main_loop_kafka.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 00d9d5e2..eb18d853 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -629,7 +629,7 @@ def save_behavior_timeouts(self, user, mq_message, kafka_key): # TODO: remove "- time()" when framework will be modified to asyncio only # если колбеков много, разносим их на 1 секунду друг от друга во избежание коллизий when = expire_time_us - time.time() + i - log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout in %(timeout)s seconds.", + log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout in %(when)s seconds.", params={log_const.KEY_NAME: "adding_local_timeout", "class_name": self.__class__.__name__, "callback_id": callback_id, From 4716ee2b731ebcb6af46150c61e76a598b32f872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D0=B5=D0=B2=D1=87=D0=B5=D0=BD=D0=BA=D0=BE=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D0=B0=D1=80=20=D0=98=D0=BB=D1=8C=D0=B8=D1=87?= Date: Tue, 8 Feb 2022 14:11:39 +0300 Subject: [PATCH 099/116] DPNLPF-1300: fix --- core/utils/utils.py | 6 ----- smart_kit/start_points/main_loop_kafka.py | 24 +++++-------------- .../static/configs/template_config.yml | 3 ++- 3 files changed, 8 insertions(+), 25 deletions(-) diff --git a/core/utils/utils.py b/core/utils/utils.py index 4e97273e..26670d50 100644 --- a/core/utils/utils.py +++ b/core/utils/utils.py @@ -130,9 +130,3 @@ def current_time_ms(): def time_check(begin_time, reject_timeout): return current_time_ms() - begin_time <= reject_timeout if begin_time is not None else True - -def get_user_number(): - return sum( - not isinstance(o, weakref.ProxyType) and isinstance(o, User) - for o in gc.get_objects() - ) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index eb18d853..67433346 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -10,7 +10,6 @@ import time import concurrent.futures import tracemalloc -from collections import namedtuple from functools import lru_cache from confluent_kafka.cimpl import KafkaException @@ -20,13 +19,11 @@ from core.logging.logger_utils import log, UID_STR, MESSAGE_ID_STR from core.message.from_message import SmartAppFromMessage -from core.model.heapq.heapq_storage import HeapqKV from core.mq.kafka.async_kafka_publisher import AsyncKafkaPublisher from core.mq.kafka.kafka_consumer import KafkaConsumer from core.utils.memstats import get_top_malloc from core.utils.stats_timer import StatsTimer from core.basic_models.actions.command import Command -from core.utils.utils import get_user_number from smart_kit.compatibility.commands import combine_commands from smart_kit.message.get_to_message import get_to_message from smart_kit.message.smartapp_to_message import SmartAppToMessage @@ -92,12 +89,6 @@ def __init__(self, *args, **kwargs): self.publishers = publishers self.concurrent_messages = 0 - # Missing in DP - self.behaviors_timeouts_value_cls = namedtuple('behaviors_timeouts_value', - 'db_uid, callback_id, mq_message, kafka_key') - self.behaviors_timeouts = HeapqKV(value_to_key_func=lambda val: val.callback_id) - # End: Missing in DP - log("%(class_name)s.__init__ completed.", params={log_const.KEY_NAME: log_const.STARTUP_VALUE, "class_name": self.__class__.__name__}) except Exception: @@ -142,7 +133,7 @@ async def healthcheck_coro(self): async def process_consumer(self, kafka_key): consumer = self.consumers[kafka_key] loop = asyncio.get_event_loop() - max_concurrent_messages = self.template_settings.get("max_concurrent_messages", 1) + max_concurrent_messages = self.template_settings.get("max_concurrent_messages", 100) total_messages = 0 profiling_settings = self.template_settings.get("profiling", {}) @@ -169,7 +160,6 @@ async def worker(iteration, queue): log( f"Total memory: {top}; " f"Async: {async_values} = {sum(async_counts)}; " - f"User number: {get_user_number()}; " f"Trash: {gc.get_count()} ", level="DEBUG" ) @@ -434,7 +424,6 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para break db_uid = message.db_uid - with StatsTimer() as load_timer: user = await self.load_user(db_uid, message) self.check_message_key(message, mq_message.key(), user) @@ -625,18 +614,17 @@ def masking_fields(self): return self.settings["template_settings"].get("masking_fields") def save_behavior_timeouts(self, user, mq_message, kafka_key): - for i, (expire_time_us, callback_id) in enumerate(user.behaviors.get_behavior_timeouts()): - # TODO: remove "- time()" when framework will be modified to asyncio only + for i, (behavior_delay, callback_id) in enumerate(user.behaviors.get_behavior_timeouts()): # если колбеков много, разносим их на 1 секунду друг от друга во избежание коллизий - when = expire_time_us - time.time() + i - log("%(class_name)s: adding local_timeout on callback %(callback_id)s with timeout in %(when)s seconds.", + delay = behavior_delay + i + log("%(class_name)s: adding local_timeout on callback %(callback_id)s with delay in %(delay)s seconds.", params={log_const.KEY_NAME: "adding_local_timeout", "class_name": self.__class__.__name__, "callback_id": callback_id, - "when": when}) + "delay": delay}) self._timers[callback_id] = self.loop.call_later( - when, self.loop.create_task, + delay, self.loop.create_task, self.do_behavior_timeout(user.message.db_uid, callback_id, mq_message, kafka_key) ) diff --git a/smart_kit/template/static/configs/template_config.yml b/smart_kit/template/static/configs/template_config.yml index 7fc92bf3..2d5aa890 100644 --- a/smart_kit/template/static/configs/template_config.yml +++ b/smart_kit/template/static/configs/template_config.yml @@ -17,4 +17,5 @@ masking_fields: - profileId user_save_collisions_tries: 2 self_service_with_state_save_messages: true -project_id: template-app-id \ No newline at end of file +project_id: template-app-id +max_concurrent_messages: 1 \ No newline at end of file From 1a7d9927809ba52cee44a52bc0cc20dffcf1ab03 Mon Sep 17 00:00:00 2001 From: intsynko Date: Wed, 9 Feb 2022 12:33:25 +0300 Subject: [PATCH 100/116] moved BaseHttpRequestAction on async --- smart_kit/action/base_http.py | 25 ++++++++-------- smart_kit/action/http.py | 6 ++-- .../action/test_base_http_action.py | 29 ++++++++++--------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/smart_kit/action/base_http.py b/smart_kit/action/base_http.py index b2904309..256de6e2 100644 --- a/smart_kit/action/base_http.py +++ b/smart_kit/action/base_http.py @@ -1,7 +1,7 @@ -import json from typing import Optional, Dict, Union, List, Any -import requests +import aiohttp +import aiohttp.client_exceptions import core.logging.logger_constants as log_const from core.basic_models.actions.command import Command @@ -49,19 +49,19 @@ def _check_headers_validity(headers: Dict[str, Any]) -> Dict[str, str]: headers[header_name] = str(header_value) return headers - def _make_response(self, request_parameters, user): + async def _make_response(self, request_parameters, user): try: - with requests.request(**request_parameters) as response: + async with aiohttp.request(**request_parameters) as response: response.raise_for_status() try: - data = response.json() - except json.decoder.JSONDecodeError: + data = await response.json() + except aiohttp.client_exceptions.ContentTypeError: data = None self._log_response(user, response, data) return data - except requests.exceptions.Timeout: + except (aiohttp.ClientTimeout, aiohttp.ServerTimeoutError): self.error = self.TIMEOUT - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError): + except aiohttp.ClientError: self.error = self.CONNECTION def _get_requst_params(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, @@ -87,16 +87,15 @@ def _log_request(self, user, request_parameters): def _log_response(self, user, response, data): log(f"{self.__class__.__name__}.run get https response ", user=user, params={ 'headers': dict(response.headers), - 'time': response.elapsed.microseconds, 'cookie': {i.name: i.value for i in response.cookies}, - 'status': response.status_code, + 'status': response.status, 'data': data, log_const.KEY_NAME: "got_http_response", }) - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: params = params or {} request_parameters = self._get_requst_params(user, text_preprocessing_result, params) self._log_request(user, request_parameters) - return self._make_response(request_parameters, user) + return await self._make_response(request_parameters, user) diff --git a/smart_kit/action/http.py b/smart_kit/action/http.py index 7437270c..47090dfe 100644 --- a/smart_kit/action/http.py +++ b/smart_kit/action/http.py @@ -44,8 +44,8 @@ def process_result(self, result, user, text_preprocessing_result, params): action = behavior_description.fail_action return action.run(user, text_preprocessing_result, None) - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: + async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: self.preprocess(user, text_preprocessing_result, params) - result = self.http_action.run(user, text_preprocessing_result, params) + result = await self.http_action.run(user, text_preprocessing_result, params) return self.process_result(result, user, text_preprocessing_result, params) diff --git a/tests/smart_kit_tests/action/test_base_http_action.py b/tests/smart_kit_tests/action/test_base_http_action.py index 581b0a70..90096c0c 100644 --- a/tests/smart_kit_tests/action/test_base_http_action.py +++ b/tests/smart_kit_tests/action/test_base_http_action.py @@ -1,10 +1,10 @@ import unittest -from unittest.mock import Mock, patch +from unittest.mock import Mock, patch, AsyncMock from smart_kit.action.base_http import BaseHttpRequestAction -class BaseHttpRequestActionTest(unittest.TestCase): +class BaseHttpRequestActionTest(unittest.IsolatedAsyncioTestCase): def setUp(self): self.user = Mock(parametrizer=Mock(collect=lambda *args, **kwargs: {})) @@ -12,27 +12,28 @@ def setUp(self): def set_request_mock_attribute(request_mock, return_value=None): return_value = return_value or {} request_mock.return_value = Mock( - __enter__=Mock(return_value=Mock( - json=Mock(return_value=return_value), + __aenter__=AsyncMock(return_value=Mock( + # response + json=AsyncMock(return_value=return_value), cookies={}, headers={}, ),), - __exit__=Mock() + __aexit__=AsyncMock() ) - @patch('requests.request') - def test_simple_request(self, request_mock: Mock): + @patch('aiohttp.request') + async def test_simple_request(self, request_mock: Mock): self.set_request_mock_attribute(request_mock) items = { "method": "POST", "url": "https://my.url.com", } - result = BaseHttpRequestAction(items).run(self.user, None, {}) + result = await BaseHttpRequestAction(items).run(self.user, None, {}) request_mock.assert_called_with(url="https://my.url.com", method='POST') self.assertEqual(result, {}) - @patch('requests.request') - def test_render_params(self, request_mock: Mock): + @patch('aiohttp.request') + async def test_render_params(self, request_mock: Mock): self.set_request_mock_attribute(request_mock) items = { "method": "POST", @@ -46,12 +47,12 @@ def test_render_params(self, request_mock: Mock): "url": "my.url.com", "value": "my_value" } - result = BaseHttpRequestAction(items).run(self.user, None, params) + result = await BaseHttpRequestAction(items).run(self.user, None, params) request_mock.assert_called_with(url="https://my.url.com", method='POST', timeout=3, json={"param": "my_value"}) self.assertEqual(result, {}) - @patch('requests.request') - def test_headers_fix(self, request_mock): + @patch('aiohttp.request') + async def test_headers_fix(self, request_mock): self.set_request_mock_attribute(request_mock) items = { "headers": { @@ -60,7 +61,7 @@ def test_headers_fix(self, request_mock): "header_3": b"d32" }, } - result = BaseHttpRequestAction(items).run(self.user, None, {}) + result = await BaseHttpRequestAction(items).run(self.user, None, {}) request_mock.assert_called_with(headers={ "header_1": "32", "header_2": "32.03", From 87c6e3c2d6ffe4c62a2e42a67c0f754c014c9962 Mon Sep 17 00:00:00 2001 From: intsynko Date: Wed, 9 Feb 2022 12:39:36 +0300 Subject: [PATCH 101/116] moved HttpRequestActionTest on async --- tests/smart_kit_tests/action/test_http.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/smart_kit_tests/action/test_http.py b/tests/smart_kit_tests/action/test_http.py index d418b7cd..6c268a1f 100644 --- a/tests/smart_kit_tests/action/test_http.py +++ b/tests/smart_kit_tests/action/test_http.py @@ -5,7 +5,7 @@ from tests.smart_kit_tests.action.test_base_http_action import BaseHttpRequestActionTest -class HttpRequestActionTest(unittest.TestCase): +class HttpRequestActionTest(unittest.IsolatedAsyncioTestCase): def setUp(self): self.user = Mock( parametrizer=Mock(collect=lambda *args, **kwargs: {}), @@ -16,8 +16,8 @@ def setUp(self): } ) - @patch('requests.request') - def test_simple_request(self, request_mock: Mock): + @patch('aiohttp.request') + async def test_simple_request(self, request_mock: Mock): BaseHttpRequestActionTest.set_request_mock_attribute(request_mock, return_value={'data': 'value'}) items = { "params": { @@ -27,7 +27,7 @@ def test_simple_request(self, request_mock: Mock): "store": "user_variable", "behavior": "my_behavior", } - HTTPRequestAction(items).run(self.user, None, {}) + await HTTPRequestAction(items).run(self.user, None, {}) request_mock.assert_called_with(url="https://my.url.com", method='POST', timeout=3) self.assertTrue(self.user.descriptions["behaviors"]["my_behavior"].success_action.run.called) self.assertTrue(self.user.variables.set.called) From 04c61b5b285764489df7be78abae8d8a428f51f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D0=B5=D0=B2=D1=87=D0=B5=D0=BD=D0=BA=D0=BE=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D0=B0=D1=80=20=D0=98=D0=BB=D1=8C=D0=B8=D1=87?= Date: Fri, 11 Feb 2022 19:38:13 +0300 Subject: [PATCH 102/116] DPNLPF-1300: fix --- scenarios/behaviors/behaviors.py | 17 ++-- .../tree_scenario/tree_scenario.py | 2 +- smart_kit/start_points/main_loop_kafka.py | 86 ++----------------- .../handlers/test_handler_timeout.py | 2 +- 4 files changed, 16 insertions(+), 91 deletions(-) diff --git a/scenarios/behaviors/behaviors.py b/scenarios/behaviors/behaviors.py index 9d4cd9df..972ef982 100644 --- a/scenarios/behaviors/behaviors.py +++ b/scenarios/behaviors/behaviors.py @@ -60,11 +60,7 @@ def add(self, callback_id: str, behavior_id, scenario_id=None, text_preprocessin host = socket.gethostname() text_preprocessing_result_raw = text_preprocessing_result_raw or {} # behavior will be removed after timeout + EXPIRATION_DELAY - expiration_time = ( - int(time()) + - self.descriptions[behavior_id].timeout(self._user) + - self.EXPIRATION_DELAY - ) + expiration_time = int(time()) + self.descriptions[behavior_id].timeout(self._user) + self.EXPIRATION_DELAY action_params = action_params or dict() action_params[LOCAL_VARS] = pickle_deepcopy(self._user.local_vars.values) @@ -238,8 +234,7 @@ def check_misstate(self, callback_id: str): def expire(self): callback_id_for_delete = [] - for callback_id, ( - behavior_id, expiration_time, *_) in self._callbacks.items(): + for callback_id, (behavior_id, expiration_time, *_) in self._callbacks.items(): if expiration_time <= time(): callback_id_for_delete.append(callback_id) for callback_id in callback_id_for_delete: @@ -252,8 +247,8 @@ def expire(self): log_const.BEHAVIOR_DATA_VALUE: str(self._callbacks[callback_id]), "to_message_name": to_message_name} log_params.update(app_info) - log( - f"behavior.expire: if you see this - something went wrong(should be timeout in normal case) callback %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s, with to_message_name %(to_message_name)s", + log(f"behavior.expire: if you see this - something went wrong(should be timeout in normal case) callback " + f"%({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s, with to_message_name %(to_message_name)s", params=log_params, level="WARNING", user=self._user) self._delete(callback_id) @@ -261,8 +256,8 @@ def check_got_saved_id(self, behavior_id): if self.descriptions[behavior_id].loop_def: for callback_id, (_behavior_id, *_) in self._callbacks.items(): if _behavior_id == behavior_id: - log( - f"behavior.check_got_saved_id == True: already got saved behavior %({log_const.BEHAVIOR_ID_VALUE})s for callback_id %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s", + log(f"behavior.check_got_saved_id == True: already got saved behavior " + f"%({log_const.BEHAVIOR_ID_VALUE})s for callback_id %({log_const.BEHAVIOR_CALLBACK_ID_VALUE})s", user=self._user, params={log_const.KEY_NAME: "behavior_got_saved", log_const.BEHAVIOR_CALLBACK_ID_VALUE: callback_id, diff --git a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py index e0a37607..2e8e5004 100644 --- a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py +++ b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py @@ -174,6 +174,6 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No reply_commands.extend(_command) if not reply_commands: - reply_commands = self.get_no_commands_action(user, text_preprocessing_result) + reply_commands = await self.get_no_commands_action(user, text_preprocessing_result) return reply_commands diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 67433346..bb7b9550 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -7,7 +7,6 @@ import pstats import random import signal -import time import concurrent.futures import tracemalloc from functools import lru_cache @@ -24,6 +23,7 @@ from core.utils.memstats import get_top_malloc from core.utils.stats_timer import StatsTimer from core.basic_models.actions.command import Command +from core.utils.utils import current_time_ms from smart_kit.compatibility.commands import combine_commands from smart_kit.message.get_to_message import get_to_message from smart_kit.message.smartapp_to_message import SmartAppToMessage @@ -178,7 +178,7 @@ async def worker(iteration, queue): mq_message = await queue.get() self.concurrent_messages += 1 if mq_message: - print(f"\n-- Processing {self.concurrent_messages} msgs at {iteration} iter\n") + log(f"\n-- Processing {self.concurrent_messages} msgs at {iteration} iter\n") total_messages += 1 headers = mq_message.headers() if headers is None: @@ -318,72 +318,6 @@ def _get_timeout_from_message(self, orig_message_raw, callback_id, headers): timeout_from_message.callback_id = callback_id return timeout_from_message - async def iterate_behavior_timeouts(self): - now = time.time() - while now > (self.behaviors_timeouts.get_head_key() or float("inf")): - _, behavior_timeout_value = self.behaviors_timeouts.pop() - db_uid, callback_id, mq_message, kafka_key = behavior_timeout_value - try: - save_tries = 0 - user_save_no_collisions = False - user = None - while save_tries < self.user_save_collisions_tries and not user_save_no_collisions: - save_tries += 1 - - orig_message_raw = json.loads(mq_message.value()) - orig_message_raw[SmartAppFromMessage.MESSAGE_NAME] = message_names.LOCAL_TIMEOUT - - timeout_from_message = self._get_timeout_from_message(orig_message_raw, callback_id, - headers=mq_message.headers()) - - user = await self.load_user(db_uid, timeout_from_message) - commands = await self.model.answer(timeout_from_message, user) - topic_key = self._get_topic_key(mq_message, kafka_key) - answers = self._generate_answers(user=user, commands=commands, message=timeout_from_message, - topic_key=topic_key, - kafka_key=kafka_key) - - user_save_no_collisions = await self.save_user(db_uid, user, mq_message) - - if user and not user_save_no_collisions: - log( - "MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(" - "db_version)s.", - user=user, - params={log_const.KEY_NAME: "ignite_collision", - "db_uid": db_uid, - "message_key": mq_message.key(), - "kafka_key": kafka_key, - "uid": user.id, - "db_version": str(user.variables.get(user.USER_DB_VERSION))}, - level="WARNING") - - continue - - if not user_save_no_collisions: - log( - "MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s " - "db_version %(db_version)s.", - user=user, - params={log_const.KEY_NAME: "ignite_collision", - "db_uid": db_uid, - "message_key": mq_message.key(), - "message_partition": mq_message.partition(), - "kafka_key": kafka_key, - "uid": user.id, - "db_version": str(user.variables.get(user.USER_DB_VERSION))}, - level="WARNING") - - smart_kit_metrics.counter_save_collision_tries_left(self.app_name) - self.save_behavior_timeouts(user, mq_message, kafka_key) - for answer in answers: - self._send_request(user, answer, mq_message) - except: - log("%(class_name)s error.", params={log_const.KEY_NAME: "error_handling_timeout", - "class_name": self.__class__.__name__, - log_const.REQUEST_VALUE: str(mq_message.value())}, - level="ERROR", exc_info=True) - def _get_topic_key(self, mq_message, kafka_key): topic_names_2_key = self._topic_names_2_key(kafka_key) return self.default_topic_key(kafka_key) or topic_names_2_key[mq_message.topic()] @@ -410,7 +344,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para level="DEBUG") waiting_message_time = 0 if message.creation_time: - waiting_message_time = self.loop.time() * 1000 - message.creation_time + waiting_message_time = current_time_ms() - message.creation_time stats += f"Waiting message: {waiting_message_time} msecs\n" log_params["waiting_message"] = waiting_message_time @@ -614,17 +548,15 @@ def masking_fields(self): return self.settings["template_settings"].get("masking_fields") def save_behavior_timeouts(self, user, mq_message, kafka_key): - for i, (behavior_delay, callback_id) in enumerate(user.behaviors.get_behavior_timeouts()): - # если колбеков много, разносим их на 1 секунду друг от друга во избежание коллизий - delay = behavior_delay + i + for (behavior_delay, callback_id) in user.behaviors.get_behavior_timeouts(): log("%(class_name)s: adding local_timeout on callback %(callback_id)s with delay in %(delay)s seconds.", params={log_const.KEY_NAME: "adding_local_timeout", "class_name": self.__class__.__name__, "callback_id": callback_id, - "delay": delay}) + "delay": behavior_delay}) self._timers[callback_id] = self.loop.call_later( - delay, self.loop.create_task, + behavior_delay, self.loop.create_task, self.do_behavior_timeout(user.message.db_uid, callback_id, mq_message, kafka_key) ) @@ -667,8 +599,7 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): user_save_ok = await self.save_user(db_uid, user, mq_message) if not user_save_ok: - log( - "MainLoop.iterate_behavior_timeouts: save user got collision on uid %(uid)s db_version %(" + log("MainLoop.do_behavior_timeout: save user got collision on uid %(uid)s db_version %(" "db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", @@ -680,8 +611,7 @@ async def do_behavior_timeout(self, db_uid, callback_id, mq_message, kafka_key): level="WARNING") if not user_save_ok and callback_found: - log( - "MainLoop.iterate_behavior_timeouts: db_save collision all tries left on uid %(uid)s db_version " + log("MainLoop.do_behavior_timeout: db_save collision all tries left on uid %(uid)s db_version " "%(db_version)s.", user=user, params={log_const.KEY_NAME: "ignite_collision", diff --git a/tests/smart_kit_tests/handlers/test_handler_timeout.py b/tests/smart_kit_tests/handlers/test_handler_timeout.py index 5e6337f7..a4b3ee91 100644 --- a/tests/smart_kit_tests/handlers/test_handler_timeout.py +++ b/tests/smart_kit_tests/handlers/test_handler_timeout.py @@ -26,7 +26,7 @@ def setUp(self): self.test_user.message.device.surface = "surface" self.test_user.behaviors = Mock('behaviors') - self.test_user.behaviors.timeout = lambda x: 120 + self.test_user.behaviors.timeout = mock_behaviors_timeout self.test_user.behaviors.has_callback = lambda *x, **y: PicklableMagicMock() self.test_user.behaviors.get_callback_action_params = lambda *x, **y: {} self.test_payload = Mock('payload') From eca49afd6e869eeaa3c571c557c6ec5254dd3757 Mon Sep 17 00:00:00 2001 From: Andrey Pushnin Date: Fri, 11 Feb 2022 22:20:52 +0300 Subject: [PATCH 103/116] py3.8 3.9 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5a1aa885..dad5ee07 100644 --- a/setup.py +++ b/setup.py @@ -56,8 +56,7 @@ "freezegun==1.1.0", ], classifiers=[ - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9" ] ) From 517aa5699f09c1ebfc51b7b1e58bb6e3e020e172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D0=B5=D0=B2=D1=87=D0=B5=D0=BD=D0=BA=D0=BE=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D0=B0=D1=80=20=D0=98=D0=BB=D1=8C=D0=B8=D1=87?= Date: Wed, 16 Feb 2022 14:06:45 +0300 Subject: [PATCH 104/116] fix --- smart_kit/start_points/main_loop_kafka.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index bb7b9550..ca2cd11b 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -405,7 +405,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para with StatsTimer() as publish_timer: self._send_request(user, answer, mq_message) smart_kit_metrics.counter_outgoing(self.app_name, answer.command.name, answer, user) - stats += "Publishing to Kafka time: {publish_timer.msecs} msecs\n" + stats += f"Publishing to Kafka time: {publish_timer.msecs} msecs\n" log_params["kafka_publishing"] = publish_timer.msecs else: validation_failed = True From 184161874e7c79c06da9e0c82224f7c0c7330737 Mon Sep 17 00:00:00 2001 From: Dan1lD Date: Wed, 16 Feb 2022 23:13:09 +0300 Subject: [PATCH 105/116] async tree_scenario --- .../scenario_descriptions/tree_scenario/tree_scenario.py | 6 +++--- smart_kit/models/dialogue_manager.py | 2 +- smart_kit/template/static/configs/template_config.yml | 3 --- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py index b3878827..a756d02a 100644 --- a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py +++ b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py @@ -42,14 +42,14 @@ def get_current_node(self, user): current_node = self.scenario_nodes[current_node_id] return current_node - def get_next_node(self, user, node, text_preprocessing_result, params): + async def get_next_node(self, user, node, text_preprocessing_result, params): available_node_keys = node.available_nodes for key in available_node_keys: node = self.scenario_nodes[key] log_params = {log_const.KEY_NAME: log_const.CHECKING_NODE_ID_VALUE, log_const.CHECKING_NODE_ID_VALUE: node.id} log(log_const.CHECKING_NODE_ID_MESSAGE, user, log_params) - requirement_result = node.requirement.check(text_preprocessing_result, user, params) + requirement_result = await node.requirement.check(text_preprocessing_result, user, params) if requirement_result: log_params = {log_const.KEY_NAME: log_const.CHOSEN_NODE_ID_VALUE, log_const.CHOSEN_NODE_ID_VALUE: node.id} @@ -156,7 +156,7 @@ async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = No elif not form: form = internal_form self._set_current_node_id(user, current_node.id) - new_node = self.get_next_node(user, current_node, text_preprocessing_result, params) + new_node = await self.get_next_node(user, current_node, text_preprocessing_result, params) field = await self._find_field(form, text_preprocessing_result, user, params) if form else None diff --git a/smart_kit/models/dialogue_manager.py b/smart_kit/models/dialogue_manager.py index 1e33c786..6c0f4324 100644 --- a/smart_kit/models/dialogue_manager.py +++ b/smart_kit/models/dialogue_manager.py @@ -31,7 +31,7 @@ async def run(self, text_preprocessing_result, user): before_action = user.descriptions["external_actions"].get("before_action") if before_action: params = user.parametrizer.collect(text_preprocessing_result) - before_action.run(user, text_preprocessing_result, params) + await before_action.run(user, text_preprocessing_result, params) scenarios_names = user.last_scenarios.scenarios_names scenario_key = user.message.payload[field.INTENT] if scenario_key in scenarios_names: diff --git a/smart_kit/template/static/configs/template_config.yml b/smart_kit/template/static/configs/template_config.yml index e5f56a01..cdef6480 100644 --- a/smart_kit/template/static/configs/template_config.yml +++ b/smart_kit/template/static/configs/template_config.yml @@ -19,8 +19,5 @@ masking_fields: user_save_collisions_tries: 2 self_service_with_state_save_messages: true project_id: template-app-id -<<<<<<< HEAD consumer_topic: "app" -======= max_concurrent_messages: 1 ->>>>>>> eca49afd6e869eeaa3c571c557c6ec5254dd3757 From ce8b314b580fa03e59ebd0b979fe5aefc33d1c46 Mon Sep 17 00:00:00 2001 From: Artem Batalov Date: Tue, 22 Feb 2022 15:23:35 +0300 Subject: [PATCH 106/116] Add DEFAULT_METHOD and _get_response method --- smart_kit/action/base_http.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/smart_kit/action/base_http.py b/smart_kit/action/base_http.py index 256de6e2..f62519d7 100644 --- a/smart_kit/action/base_http.py +++ b/smart_kit/action/base_http.py @@ -49,14 +49,21 @@ def _check_headers_validity(headers: Dict[str, Any]) -> Dict[str, str]: headers[header_name] = str(header_value) return headers + @staticmethod + async def _get_response(response: aiohttp.ClientResponse): + try: + data = await response.json() + except aiohttp.client_exceptions.ContentTypeError: + data = None + return data + async def _make_response(self, request_parameters, user): try: + if 'method' not in request_parameters: + request_parameters['method'] = self.DEFAULT_METHOD async with aiohttp.request(**request_parameters) as response: response.raise_for_status() - try: - data = await response.json() - except aiohttp.client_exceptions.ContentTypeError: - data = None + data = await self._get_response(response) self._log_response(user, response, data) return data except (aiohttp.ClientTimeout, aiohttp.ServerTimeoutError): @@ -64,8 +71,8 @@ async def _make_response(self, request_parameters, user): except aiohttp.ClientError: self.error = self.CONNECTION - def _get_requst_params(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None): + def _get_request_params(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None): collected = user.parametrizer.collect(text_preprocessing_result) params.update(collected) @@ -96,6 +103,6 @@ def _log_response(self, user, response, data): async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: params = params or {} - request_parameters = self._get_requst_params(user, text_preprocessing_result, params) + request_parameters = self._get_request_params(user, text_preprocessing_result, params) self._log_request(user, request_parameters) return await self._make_response(request_parameters, user) From d3fd253050eb5ad7034eaca7813d2840e9a29205 Mon Sep 17 00:00:00 2001 From: intsynko Date: Mon, 28 Feb 2022 12:40:42 +0300 Subject: [PATCH 107/116] combined HttpRequestAction and BaseHttpRequestAction into one --- smart_kit/action/base_http.py | 113 ------------------ smart_kit/action/http.py | 100 ++++++++++++++-- .../action/test_base_http_action.py | 71 ----------- tests/smart_kit_tests/action/test_http.py | 63 +++++++++- 4 files changed, 149 insertions(+), 198 deletions(-) delete mode 100644 smart_kit/action/base_http.py delete mode 100644 tests/smart_kit_tests/action/test_base_http_action.py diff --git a/smart_kit/action/base_http.py b/smart_kit/action/base_http.py deleted file mode 100644 index 1b3ece45..00000000 --- a/smart_kit/action/base_http.py +++ /dev/null @@ -1,113 +0,0 @@ -import json -from typing import Optional, Dict, Union, List, Any - -import requests - -import core.logging.logger_constants as log_const -from core.basic_models.actions.command import Command -from core.basic_models.actions.string_actions import NodeAction -from core.logging.logger_utils import log -from core.model.base_user import BaseUser -from core.text_preprocessing.base import BaseTextPreprocessingResult - - -class BaseHttpRequestAction(NodeAction): - """ - Example: - { - // обязательные параметры - "method": "POST", - "url": "http://some_url.com/...", - - // необязательные параметры - "json": { - "data": "value", - ... - }, - "timeout": 120, - "headers": { - "Content-Type":"application/json" - } - } - """ - POST = "POST" - GET = "GET" - DEFAULT_METHOD = POST - - TIMEOUT = "TIMEOUT" - CONNECTION = "CONNECTION" - - def __init__(self, items, id=None): - super().__init__(items, id) - self.method_params = items - self.error = None - - @staticmethod - def _check_headers_validity(headers: Dict[str, Any], user) -> Dict[str, str]: - for header_name, header_value in list(headers.items()): - if not isinstance(header_value, (str, bytes)): - if isinstance(header_value, (int, float, bool)): - headers[header_name] = str(header_value) - else: - log(f"{__class__.__name__}._check_headers_validity remove header {header_name} because " - f"({type(header_value)}) is not in [int, float, bool, str, bytes]", user=user, params={ - log_const.KEY_NAME: "sent_http_remove_header", - }) - del headers[header_name] - return headers - - def _make_response(self, request_parameters, user): - try: - with requests.request(**request_parameters) as response: - response.raise_for_status() - try: - data = response.json() - except json.decoder.JSONDecodeError: - data = None - self._log_response(user, response, data) - return data - except requests.exceptions.Timeout: - self.error = self.TIMEOUT - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError): - self.error = self.CONNECTION - - def _get_request_params(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None): - collected = user.parametrizer.collect(text_preprocessing_result) - params.update(collected) - - request_parameters = self._get_rendered_tree_recursive(self._get_template_tree(self.method_params), params) - - req_headers = request_parameters.get("headers") - if req_headers: - # Заголовки в запросах должны иметь тип str или bytes. Поэтому добавлена проверка и приведение к типу str, - # на тот случай если в сценарии заголовок указали как int, float и тд - request_parameters["headers"] = self._check_headers_validity(req_headers, user) - return request_parameters - - def _log_request(self, user, request_parameters, additional_params=None): - additional_params = additional_params or {} - log(f"{self.__class__.__name__}.run sent https request ", user=user, params={ - **request_parameters, - log_const.KEY_NAME: "sent_http_request", - **additional_params, - }) - - def _log_response(self, user, response, data, additional_params=None): - additional_params = additional_params or {} - log(f"{self.__class__.__name__}.run get https response ", user=user, params={ - 'headers': dict(response.headers), - 'time': response.elapsed.microseconds, - 'cookie': {i.name: i.value for i in response.cookies}, - 'status': response.status_code, - 'data': data, - log_const.KEY_NAME: "got_http_response", - **additional_params, - }) - - def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, - params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: - params = params or {} - request_parameters = self._get_request_params(user, text_preprocessing_result, params) - self._log_request(user, request_parameters) - return self._make_response(request_parameters, user) diff --git a/smart_kit/action/http.py b/smart_kit/action/http.py index 7437270c..15fdf537 100644 --- a/smart_kit/action/http.py +++ b/smart_kit/action/http.py @@ -1,13 +1,17 @@ -from typing import Optional, Dict, Union, List +import json +from typing import Optional, Dict, Union, List, Any -from core.basic_models.actions.basic_actions import Action +import requests + +import core.logging.logger_constants as log_const from core.basic_models.actions.command import Command +from core.basic_models.actions.string_actions import NodeAction +from core.logging.logger_utils import log from core.model.base_user import BaseUser from core.text_preprocessing.base import BaseTextPreprocessingResult -from smart_kit.action.base_http import BaseHttpRequestAction -class HTTPRequestAction(Action): +class HTTPRequestAction(NodeAction): """ Example: { @@ -21,24 +25,95 @@ class HTTPRequestAction(Action): } """ - HTTP_ACTION = BaseHttpRequestAction + POST = "POST" + GET = "GET" + DEFAULT_METHOD = POST + + TIMEOUT = "TIMEOUT" + CONNECTION = "CONNECTION" def __init__(self, items, id=None): - self.http_action = self.HTTP_ACTION(items["params"], id) + super().__init__(items, id) + self.method_params = items['params'] + self.method_params.setdefault("method", self.DEFAULT_METHOD) + self.error = None + self.init_save_params(items) + + def init_save_params(self, items): self.store = items["store"] self.behavior = items["behavior"] - super().__init__(items, id) def preprocess(self, user, text_processing, params): behavior_description = user.descriptions["behaviors"][self.behavior] - self.http_action.method_params.setdefault("timeout", behavior_description.timeout(user)) + self.method_params.setdefault("timeout", behavior_description.timeout(user)) + + @staticmethod + def _check_headers_validity(headers: Dict[str, Any], user) -> Dict[str, str]: + for header_name, header_value in list(headers.items()): + if not isinstance(header_value, (str, bytes)): + if isinstance(header_value, (int, float, bool)): + headers[header_name] = str(header_value) + else: + log(f"{__class__.__name__}._check_headers_validity remove header {header_name} because " + f"({type(header_value)}) is not in [int, float, bool, str, bytes]", user=user, params={ + log_const.KEY_NAME: "sent_http_remove_header", + }) + del headers[header_name] + return headers + + def _make_response(self, request_parameters, user): + try: + with requests.request(**request_parameters) as response: + response.raise_for_status() + try: + data = response.json() + except json.decoder.JSONDecodeError: + data = None + self._log_response(user, response, data) + return data + except requests.exceptions.Timeout: + self.error = self.TIMEOUT + except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError): + self.error = self.CONNECTION + + def _get_request_params(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, + params: Optional[Dict[str, Union[str, float, int]]] = None): + collected = user.parametrizer.collect(text_preprocessing_result) + params.update(collected) + request_parameters = self._get_rendered_tree_recursive(self._get_template_tree(self.method_params), params) + req_headers = request_parameters.get("headers") + if req_headers: + # Заголовки в запросах должны иметь тип str или bytes. Поэтому добавлена проверка и приведение к типу str, + # на тот случай если в сценарии заголовок указали как int, float и тд + request_parameters["headers"] = self._check_headers_validity(req_headers, user) + return request_parameters + + def _log_request(self, user, request_parameters, additional_params=None): + additional_params = additional_params or {} + log(f"{self.__class__.__name__}.run sent https request ", user=user, params={ + **request_parameters, + log_const.KEY_NAME: "sent_http_request", + **additional_params, + }) + + def _log_response(self, user, response, data, additional_params=None): + additional_params = additional_params or {} + log(f"{self.__class__.__name__}.run get https response ", user=user, params={ + 'headers': dict(response.headers), + 'time': response.elapsed.microseconds, + 'cookie': {i.name: i.value for i in response.cookies}, + 'status': response.status_code, + 'data': data, + log_const.KEY_NAME: "got_http_response", + **additional_params, + }) def process_result(self, result, user, text_preprocessing_result, params): behavior_description = user.descriptions["behaviors"][self.behavior] - if self.http_action.error is None: + if self.error is None: user.variables.set(self.store, result) action = behavior_description.success_action - elif self.http_action.error == self.http_action.TIMEOUT: + elif self.error == self.TIMEOUT: action = behavior_description.timeout_action else: action = behavior_description.fail_action @@ -47,5 +122,8 @@ def process_result(self, result, user, text_preprocessing_result, params): def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: self.preprocess(user, text_preprocessing_result, params) - result = self.http_action.run(user, text_preprocessing_result, params) + params = params or {} + request_parameters = self._get_request_params(user, text_preprocessing_result, params) + self._log_request(user, request_parameters) + result = self._make_response(request_parameters, user) return self.process_result(result, user, text_preprocessing_result, params) diff --git a/tests/smart_kit_tests/action/test_base_http_action.py b/tests/smart_kit_tests/action/test_base_http_action.py deleted file mode 100644 index cab465ad..00000000 --- a/tests/smart_kit_tests/action/test_base_http_action.py +++ /dev/null @@ -1,71 +0,0 @@ -import unittest -from unittest.mock import Mock, patch - -from smart_kit.action.base_http import BaseHttpRequestAction - - -class BaseHttpRequestActionTest(unittest.TestCase): - def setUp(self): - self.user = Mock(parametrizer=Mock(collect=lambda *args, **kwargs: {})) - - @staticmethod - def set_request_mock_attribute(request_mock, return_value=None): - return_value = return_value or {} - request_mock.return_value = Mock( - __enter__=Mock(return_value=Mock( - json=Mock(return_value=return_value), - cookies={}, - headers={}, - ),), - __exit__=Mock() - ) - - @patch('requests.request') - def test_simple_request(self, request_mock: Mock): - self.set_request_mock_attribute(request_mock) - items = { - "method": "POST", - "url": "https://my.url.com", - } - result = BaseHttpRequestAction(items).run(self.user, None, {}) - request_mock.assert_called_with(url="https://my.url.com", method='POST') - self.assertEqual(result, {}) - - @patch('requests.request') - def test_render_params(self, request_mock: Mock): - self.set_request_mock_attribute(request_mock) - items = { - "method": "POST", - "url": "https://{{url}}", - "timeout": 3, - "json": { - "param": "{{value}}" - } - } - params = { - "url": "my.url.com", - "value": "my_value" - } - result = BaseHttpRequestAction(items).run(self.user, None, params) - request_mock.assert_called_with(url="https://my.url.com", method='POST', timeout=3, json={"param": "my_value"}) - self.assertEqual(result, {}) - - @patch('requests.request') - def test_headers_fix(self, request_mock): - self.set_request_mock_attribute(request_mock) - items = { - "headers": { - "header_1": 32, - "header_2": 32.03, - "header_3": b"d32", - "header_4": None, - "header_5": {"data": "value"}, - }, - } - result = BaseHttpRequestAction(items).run(self.user, None, {}) - request_mock.assert_called_with(headers={ - "header_1": "32", - "header_2": "32.03", - "header_3": b"d32" - }) - self.assertEqual(result, {}) diff --git a/tests/smart_kit_tests/action/test_http.py b/tests/smart_kit_tests/action/test_http.py index d418b7cd..d0cd106c 100644 --- a/tests/smart_kit_tests/action/test_http.py +++ b/tests/smart_kit_tests/action/test_http.py @@ -2,23 +2,35 @@ from unittest.mock import Mock, patch from smart_kit.action.http import HTTPRequestAction -from tests.smart_kit_tests.action.test_base_http_action import BaseHttpRequestActionTest class HttpRequestActionTest(unittest.TestCase): + TIMEOUT = 3 + def setUp(self): self.user = Mock( parametrizer=Mock(collect=lambda *args, **kwargs: {}), descriptions={ "behaviors": { - "my_behavior": Mock(timeout=Mock(return_value=3)) + "my_behavior": Mock(timeout=Mock(return_value=self.TIMEOUT)) } } ) + def set_request_mock_attribute(self, request_mock, return_value=None): + return_value = return_value or {} + request_mock.return_value = Mock( + __enter__=Mock(return_value=Mock( + json=Mock(return_value=return_value), + cookies={}, + headers={}, + ), ), + __exit__=Mock() + ) + @patch('requests.request') def test_simple_request(self, request_mock: Mock): - BaseHttpRequestActionTest.set_request_mock_attribute(request_mock, return_value={'data': 'value'}) + self.set_request_mock_attribute(request_mock, return_value={'data': 'value'}) items = { "params": { "method": "POST", @@ -32,3 +44,48 @@ def test_simple_request(self, request_mock: Mock): self.assertTrue(self.user.descriptions["behaviors"]["my_behavior"].success_action.run.called) self.assertTrue(self.user.variables.set.called) self.user.variables.set.assert_called_with("user_variable", {'data': 'value'}) + + @patch('requests.request') + def test_render_params(self, request_mock: Mock): + self.set_request_mock_attribute(request_mock) + items = { + "params": { + "method": "POST", + "url": "https://{{url}}", + "timeout": 3, + "json": { + "param": "{{value}}" + } + }, + "store": "user_variable", + "behavior": "my_behavior", + } + params = { + "url": "my.url.com", + "value": "my_value" + } + HTTPRequestAction(items).run(self.user, None, params) + request_mock.assert_called_with(url="https://my.url.com", method='POST', timeout=3, json={"param": "my_value"}) + + @patch('requests.request') + def test_headers_fix(self, request_mock): + self.set_request_mock_attribute(request_mock) + items = { + "params": { + "headers": { + "header_1": 32, + "header_2": 32.03, + "header_3": b"d32", + "header_4": None, + "header_5": {"data": "value"}, + }, + }, + "store": "user_variable", + "behavior": "my_behavior", + } + HTTPRequestAction(items).run(self.user, None, {}) + request_mock.assert_called_with(headers={ + "header_1": "32", + "header_2": "32.03", + "header_3": b"d32" + }, timeout=self.TIMEOUT) From cd06de1b327f6044fed55c5c8f5a25677d037c18 Mon Sep 17 00:00:00 2001 From: intsynko Date: Mon, 28 Feb 2022 13:12:50 +0300 Subject: [PATCH 108/116] fix tests bug --- tests/smart_kit_tests/action/test_http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/smart_kit_tests/action/test_http.py b/tests/smart_kit_tests/action/test_http.py index d0cd106c..4d847204 100644 --- a/tests/smart_kit_tests/action/test_http.py +++ b/tests/smart_kit_tests/action/test_http.py @@ -88,4 +88,5 @@ def test_headers_fix(self, request_mock): "header_1": "32", "header_2": "32.03", "header_3": b"d32" - }, timeout=self.TIMEOUT) + }, method=HTTPRequestAction.DEFAULT_METHOD, timeout=self.TIMEOUT) + From 81ec81256913e64ef91cd33cc61d114757dd34db Mon Sep 17 00:00:00 2001 From: intsynko Date: Mon, 28 Feb 2022 17:00:23 +0300 Subject: [PATCH 109/116] use behavior params as optional --- smart_kit/action/http.py | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/smart_kit/action/http.py b/smart_kit/action/http.py index 426c9d0c..fab083fd 100644 --- a/smart_kit/action/http.py +++ b/smart_kit/action/http.py @@ -41,7 +41,7 @@ def __init__(self, items, id=None): def init_save_params(self, items): self.store = items["store"] - self.behavior = items["behavior"] + self.behavior = items.get("behavior") def preprocess(self, user, text_processing, params): behavior_description = user.descriptions["behaviors"][self.behavior] @@ -65,12 +65,8 @@ async def _make_response(self, request_parameters, user): try: async with aiohttp.request(**request_parameters) as response: response.raise_for_status() - try: - data = await response.json() - except aiohttp.client_exceptions.ContentTypeError: - data = None - self._log_response(user, response, data) - return data + self._log_response(user, response) + return response except (aiohttp.ClientTimeout, aiohttp.ServerTimeoutError): self.error = self.TIMEOUT except aiohttp.ClientError: @@ -96,27 +92,33 @@ def _log_request(self, user, request_parameters, additional_params=None): **additional_params, }) - def _log_response(self, user, response, data, additional_params=None): + def _log_response(self, user, response, additional_params=None): additional_params = additional_params or {} log(f"{self.__class__.__name__}.run get https response ", user=user, params={ 'headers': dict(response.headers), 'cookie': {i.name: i.value for i in response.cookies}, 'status': response.status, - 'data': data, log_const.KEY_NAME: "got_http_response", **additional_params, }) - async def process_result(self, result, user, text_preprocessing_result, params): - behavior_description = user.descriptions["behaviors"][self.behavior] + async def process_result(self, response, user, text_preprocessing_result, params): + behavior_description = user.descriptions["behaviors"][self.behavior] if self.behavior else None + action = None if self.error is None: - user.variables.set(self.store, result) - action = behavior_description.success_action - elif self.error == self.TIMEOUT: - action = behavior_description.timeout_action - else: - action = behavior_description.fail_action - return await action.run(user, text_preprocessing_result, None) + try: + data = await response.json() + except aiohttp.client_exceptions.ContentTypeError: + data = None + user.variables.set(self.store, data) + action = behavior_description.success_action if behavior_description else None + elif behavior_description is not None: + if self.error == self.TIMEOUT: + action = behavior_description.timeout_action + else: + action = behavior_description.fail_action + if action: + return await action.run(user, text_preprocessing_result, None) async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult, params: Optional[Dict[str, Union[str, float, int]]] = None) -> Optional[List[Command]]: @@ -124,5 +126,5 @@ async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreproces params = params or {} request_parameters = self._get_request_params(user, text_preprocessing_result, params) self._log_request(user, request_parameters) - result = await self._make_response(request_parameters, user) - return await self.process_result(result, user, text_preprocessing_result, params) + respone = await self._make_response(request_parameters, user) + return await self.process_result(respone, user, text_preprocessing_result, params) From cbeda2167b6d778e8acb60c86d1a63a33885627e Mon Sep 17 00:00:00 2001 From: Artem Batalov Date: Wed, 2 Mar 2022 17:58:16 +0300 Subject: [PATCH 110/116] Fix http cookies logging --- smart_kit/action/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart_kit/action/http.py b/smart_kit/action/http.py index fab083fd..506d6b31 100644 --- a/smart_kit/action/http.py +++ b/smart_kit/action/http.py @@ -96,7 +96,7 @@ def _log_response(self, user, response, additional_params=None): additional_params = additional_params or {} log(f"{self.__class__.__name__}.run get https response ", user=user, params={ 'headers': dict(response.headers), - 'cookie': {i.name: i.value for i in response.cookies}, + 'cookie': {k: v.value for k, v in response.cookies.items()}, 'status': response.status, log_const.KEY_NAME: "got_http_response", **additional_params, From 6d6feda25160a3388773d3de19e83ab9cbde64c5 Mon Sep 17 00:00:00 2001 From: Makar Shevchenko Date: Tue, 22 Mar 2022 20:33:52 +0300 Subject: [PATCH 111/116] make http main loop support async --- core/basic_models/requirement/basic_requirements.py | 2 +- smart_kit/action/http.py | 9 ++++++--- smart_kit/start_points/main_loop_http.py | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/basic_models/requirement/basic_requirements.py b/core/basic_models/requirement/basic_requirements.py index 209902ca..1781fe05 100644 --- a/core/basic_models/requirement/basic_requirements.py +++ b/core/basic_models/requirement/basic_requirements.py @@ -175,7 +175,7 @@ async def check(self, text_preprocessing_result: BaseTextPreprocessingResult, us return True if render_result == "False": return False - raise TypeError(f'Template result should be "True" or "False", got: ', + raise TypeError(f'Template result should be "True" or "False", got: ' f'{render_result} for template {self.items["template"]}') diff --git a/smart_kit/action/http.py b/smart_kit/action/http.py index 506d6b31..ce09bb36 100644 --- a/smart_kit/action/http.py +++ b/smart_kit/action/http.py @@ -1,7 +1,9 @@ +import asyncio from typing import Optional, Dict, Union, List, Any import aiohttp import aiohttp.client_exceptions +from aiohttp import ClientTimeout import core.logging.logger_constants as log_const from core.basic_models.actions.command import Command @@ -46,6 +48,7 @@ def init_save_params(self, items): def preprocess(self, user, text_processing, params): behavior_description = user.descriptions["behaviors"][self.behavior] self.method_params.setdefault("timeout", behavior_description.timeout(user)) + self.method_params["timeout"] = ClientTimeout(self.method_params["timeout"]) @staticmethod def _check_headers_validity(headers: Dict[str, Any], user) -> Dict[str, str]: @@ -67,7 +70,7 @@ async def _make_response(self, request_parameters, user): response.raise_for_status() self._log_response(user, response) return response - except (aiohttp.ClientTimeout, aiohttp.ServerTimeoutError): + except (aiohttp.ServerTimeoutError, asyncio.TimeoutError): self.error = self.TIMEOUT except aiohttp.ClientError: self.error = self.CONNECTION @@ -126,5 +129,5 @@ async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreproces params = params or {} request_parameters = self._get_request_params(user, text_preprocessing_result, params) self._log_request(user, request_parameters) - respone = await self._make_response(request_parameters, user) - return await self.process_result(respone, user, text_preprocessing_result, params) + response = await self._make_response(request_parameters, user) + return await self.process_result(response, user, text_preprocessing_result, params) diff --git a/smart_kit/start_points/main_loop_http.py b/smart_kit/start_points/main_loop_http.py index 8425a56f..2fd8f4dd 100644 --- a/smart_kit/start_points/main_loop_http.py +++ b/smart_kit/start_points/main_loop_http.py @@ -67,7 +67,7 @@ def process_message(self, message: SmartAppFromMessage, *args, **kwargs): db_uid = message.db_uid with StatsTimer() as load_timer: - user = self.load_user(db_uid, message) + user = self.loop.run_until_complete(self.load_user(db_uid, message)) stats += "Loading time: {} msecs\n".format(load_timer.msecs) with StatsTimer() as script_timer: commands = asyncio.get_event_loop().run_until_complete(self.model.answer(message, user)) @@ -78,7 +78,7 @@ def process_message(self, message: SmartAppFromMessage, *args, **kwargs): stats += "Script time: {} msecs\n".format(script_timer.msecs) with StatsTimer() as save_timer: - self.save_user(db_uid, user, message) + self.loop.run_until_complete(self.save_user(db_uid, user, message)) stats += "Saving time: {} msecs\n".format(save_timer.msecs) log(stats, params={log_const.KEY_NAME: "timings"}) return answer, stats From 14930ddae5ec709ef648f9e5e18c95461f9d55d7 Mon Sep 17 00:00:00 2001 From: Makar Shevchenko Date: Thu, 24 Mar 2022 21:30:08 +0300 Subject: [PATCH 112/116] DPNLPF-1300: make children of AsyncDBAdapter async --- core/db_adapter/memory_adapter.py | 16 +++--- .../adapters/test_memory_adapter.py | 50 +++++++++---------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/core/db_adapter/memory_adapter.py b/core/db_adapter/memory_adapter.py index 6bed7dda..6f0f3360 100644 --- a/core/db_adapter/memory_adapter.py +++ b/core/db_adapter/memory_adapter.py @@ -8,34 +8,34 @@ def __init__(self, config=None): super(AsyncDBAdapter, self).__init__(config) self.memory_storage = {} - def _glob(self, path, pattern): + async def _glob(self, path, pattern): raise error.NotSupportedOperation - def _path_exists(self, path): + async def _path_exists(self, path): raise error.NotSupportedOperation - def _on_prepare(self): + async def _on_prepare(self): pass async def connect(self): pass - def _open(self, filename, *args, **kwargs): + async def _open(self, filename, *args, **kwargs): pass - def _save(self, id, data): + async def _save(self, id, data): self.memory_storage[id] = data - def _replace_if_equals(self, id, sample, data): + async def _replace_if_equals(self, id, sample, data): stored_data = self.memory_storage.get(id) if stored_data == sample: self.memory_storage[id] = data return True return False - def _get(self, id): + async def _get(self, id): data = self.memory_storage.get(id) return data - def _list_dir(self, path): + async def _list_dir(self, path): pass diff --git a/tests/smart_kit_tests/adapters/test_memory_adapter.py b/tests/smart_kit_tests/adapters/test_memory_adapter.py index a3c10772..ece4090d 100644 --- a/tests/smart_kit_tests/adapters/test_memory_adapter.py +++ b/tests/smart_kit_tests/adapters/test_memory_adapter.py @@ -4,50 +4,50 @@ from core.db_adapter import memory_adapter -class AdapterTest1(unittest.TestCase): +class AdapterTest1(unittest.IsolatedAsyncioTestCase): - def test_memory_adapter_init(self): + async def test_memory_adapter_init(self): obj1 = memory_adapter.MemoryAdapter() obj2 = memory_adapter.MemoryAdapter({'try_count': 3}) self.assertTrue(hasattr(obj1, 'open')) self.assertTrue(hasattr(obj2, 'open')) - self.assertTrue(obj1.memory_storage == {}) - self.assertTrue(obj2.memory_storage == {}) - self.assertTrue(obj1.try_count == 5) # взято из исходников - self.assertTrue(obj2.try_count == 3) + self.assertEqual(obj1.memory_storage, {}) + self.assertEqual(obj2.memory_storage, {}) + self.assertEqual(obj1.try_count, 5) # взято из исходников + self.assertEqual(obj2.try_count, 3) - def test_memory_adapter_connect(self): + async def test_memory_adapter_connect(self): obj = memory_adapter.MemoryAdapter() self.assertTrue(hasattr(obj, 'connect')) - def test_memory_adapter_open(self): + async def test_memory_adapter_open(self): obj = memory_adapter.MemoryAdapter() self.assertTrue(hasattr(obj, '_open')) with self.assertRaises(TypeError): - obj._open() + await obj._open() - def test_memory_adapter_save(self): + async def test_memory_adapter_save(self): obj1 = memory_adapter.MemoryAdapter() obj2 = memory_adapter.MemoryAdapter() - obj1._save(10, {'any_data'}) - obj1._save(11, {'any_data'}) - obj2._save(10, {'any_data'}) - self.assertTrue(obj1.memory_storage == {10: {'any_data'}, 11: {'any_data'}}) - self.assertTrue(obj2.memory_storage == {10: {'any_data'}}) + await obj1._save(10, {'any_data'}) + await obj1._save(11, {'any_data'}) + await obj2._save(10, {'any_data'}) + self.assertEqual(obj1.memory_storage, {10: {'any_data'}, 11: {'any_data'}}) + self.assertEqual(obj2.memory_storage, {10: {'any_data'}}) # метод переписывает значения - obj2._save(10, 'any_data') - self.assertTrue(obj2.memory_storage == {10: 'any_data'}) + await obj2._save(10, 'any_data') + self.assertEqual(obj2.memory_storage, {10: 'any_data'}) - def test_memory_adapter_get(self): + async def test_memory_adapter_get(self): obj1 = memory_adapter.MemoryAdapter() - obj1._save(10, {'any_data'}) - obj1._save(11, 'any_data') - self.assertTrue(obj1._get(10) == {'any_data'}) - self.assertTrue(obj1._get(11) == 'any_data') - self.assertIsNone(obj1._get(12)) + await obj1._save(10, {'any_data'}) + await obj1._save(11, 'any_data') + self.assertEqual(await obj1._get(10), {'any_data'}) + self.assertEqual(await obj1._get(11), 'any_data') + self.assertIsNone(await obj1._get(12)) - def test_memory_adapter_list_dir(self): + async def test_memory_adapter_list_dir(self): obj = memory_adapter.MemoryAdapter() self.assertTrue(hasattr(obj, '_list_dir')) with self.assertRaises(TypeError): - obj._open() + await obj._open() From 642e2da7258cf639594d4fa7775d89f0d5b5bd4d Mon Sep 17 00:00:00 2001 From: Makar Shevchenko Date: Thu, 24 Mar 2022 21:34:58 +0300 Subject: [PATCH 113/116] DPNLPF-1300: make children of AsyncDBAdapter async --- core/db_adapter/aioredis_adapter.py | 14 +++++++------- core/db_adapter/aioredis_sentinel_adapter.py | 16 ++++++++-------- core/db_adapter/db_adapter.py | 2 +- core/db_adapter/ignite_adapter.py | 12 ++++++------ scenarios/actions/action.py | 2 +- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/core/db_adapter/aioredis_adapter.py b/core/db_adapter/aioredis_adapter.py index 3eeee803..38e498b9 100644 --- a/core/db_adapter/aioredis_adapter.py +++ b/core/db_adapter/aioredis_adapter.py @@ -23,18 +23,18 @@ def __init__(self, config=None): @monitoring.got_histogram_decorate("save_time") async def save(self, id, data): - return await self._run(self._save, id, data) + return await self._async_run(self._save, id, data) @monitoring.got_histogram_decorate("save_time") async def replace_if_equals(self, id, sample, data): - return await self._run(self._replace_if_equals, id, sample, data) + return await self._async_run(self._replace_if_equals, id, sample, data) @monitoring.got_histogram_decorate("get_time") async def get(self, id): - return await self._run(self._get, id) + return await self._async_run(self._get, id) async def path_exists(self, path): - return await self._run(self._path_exists, path) + return await self._async_run(self._path_exists, path) async def connect(self): print("Here is the content of REDIS_CONFIG:", self.config) @@ -60,14 +60,14 @@ async def _get(self, id): data = await self._redis.get(id) return data - def _list_dir(self, path): + async def _list_dir(self, path): raise error.NotSupportedOperation - def _glob(self, path, pattern): + async def _glob(self, path, pattern): raise error.NotSupportedOperation async def _path_exists(self, path): return await self._redis.exists(path) - def _on_prepare(self): + async def _on_prepare(self): pass diff --git a/core/db_adapter/aioredis_sentinel_adapter.py b/core/db_adapter/aioredis_sentinel_adapter.py index 1fee05ac..3f890df3 100644 --- a/core/db_adapter/aioredis_sentinel_adapter.py +++ b/core/db_adapter/aioredis_sentinel_adapter.py @@ -24,19 +24,19 @@ def __init__(self, config=None): @monitoring.got_histogram_decorate("save_time") async def save(self, id, data): - return await self._run(self._save, id, data) + return await self._async_run(self._save, id, data) @monitoring.got_histogram_decorate("save_time") async def replace_if_equals(self, id, sample, data): - return await self._run(self._replace_if_equals, id, sample, data) + return await self._async_run(self._replace_if_equals, id, sample, data) @monitoring.got_histogram_decorate("get_time") async def get(self, id): - return await self._run(self._get, id) + return await self._async_run(self._get, id) async def path_exists(self, path): - return await self._run(self._path_exists, path) + return await self._async_run(self._path_exists, path) async def connect(self): @@ -55,7 +55,7 @@ async def connect(self): sentinels_tuples.append(tuple(sent)) self._sentinel = Sentinel(sentinels_tuples, **config) - def _open(self, filename, *args, **kwargs): + async def _open(self, filename, *args, **kwargs): pass async def _save(self, id, data): @@ -71,15 +71,15 @@ async def _get(self, id): data = await redis.get(id) return data - def _list_dir(self, path): + async def _list_dir(self, path): raise error.NotSupportedOperation - def _glob(self, path, pattern): + async def _glob(self, path, pattern): raise error.NotSupportedOperation async def _path_exists(self, path): redis = await self._sentinel.master_for(self.service_name, socket_timeout=self.socket_timeout) return await redis.exists(path) - def _on_prepare(self): + async def _on_prepare(self): pass diff --git a/core/db_adapter/db_adapter.py b/core/db_adapter/db_adapter.py index 2311fa2c..0ff718c3 100644 --- a/core/db_adapter/db_adapter.py +++ b/core/db_adapter/db_adapter.py @@ -141,7 +141,7 @@ async def _async_run(self, action, *args, _try_count=None, **kwargs): params=params, level="ERROR") self._on_prepare() - result = await self._run(action, *args, _try_count=_try_count, **kwargs) + result = await self._async_run(action, *args, _try_count=_try_count, **kwargs) counter_name = self._get_counter_name() if counter_name: monitoring.got_counter(f"{counter_name}_exception") diff --git a/core/db_adapter/ignite_adapter.py b/core/db_adapter/ignite_adapter.py index bce3742b..b5405ec3 100644 --- a/core/db_adapter/ignite_adapter.py +++ b/core/db_adapter/ignite_adapter.py @@ -27,16 +27,16 @@ def __init__(self, config): self._cache = None super(IgniteAdapter, self).__init__(config) - def _open(self, filename, *args, **kwargs): + async def _open(self, filename, *args, **kwargs): pass - def _list_dir(self, path): + async def _list_dir(self, path): raise error.NotSupportedOperation - def _glob(self, path, pattern): + async def _glob(self, path, pattern): raise error.NotSupportedOperation - def _path_exists(self, path): + async def _path_exists(self, path): raise error.NotSupportedOperation async def connect(self): @@ -83,8 +83,8 @@ def _handled_exception(self): # TypeError is raised during reconnection if all nodes are exhausted return OSError, SocketError, ReconnectError, CancelledError - def _on_prepare(self): + async def _on_prepare(self): self._client = None - def _get_counter_name(self): + async def _get_counter_name(self): return "ignite_async_adapter" diff --git a/scenarios/actions/action.py b/scenarios/actions/action.py index e6f3fe5b..88e2ac45 100644 --- a/scenarios/actions/action.py +++ b/scenarios/actions/action.py @@ -23,7 +23,7 @@ import scenarios.logging.logger_constants as log_const from scenarios.actions.action_params_names import TO_MESSAGE_NAME, TO_MESSAGE_PARAMS, SAVED_MESSAGES, \ - REQUEST_FIELD, LOCAL_VARS + REQUEST_FIELD from scenarios.user.parametrizer import Parametrizer from scenarios.user.user_model import User from scenarios.scenario_models.history import Event From ebfea637a99ccb924871e67e5cc92c4d547ce8f4 Mon Sep 17 00:00:00 2001 From: Makar Shevchenko Date: Thu, 24 Mar 2022 21:41:35 +0300 Subject: [PATCH 114/116] DPNLPF-1300: fix httprequestaction tests --- tests/smart_kit_tests/action/test_http.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/smart_kit_tests/action/test_http.py b/tests/smart_kit_tests/action/test_http.py index 03c70a2e..ee61417c 100644 --- a/tests/smart_kit_tests/action/test_http.py +++ b/tests/smart_kit_tests/action/test_http.py @@ -1,6 +1,8 @@ import unittest from unittest.mock import Mock, patch, AsyncMock +from aiohttp import ClientTimeout + from smart_kit.action.http import HTTPRequestAction @@ -41,7 +43,7 @@ async def test_simple_request(self, request_mock: Mock): "behavior": "my_behavior", } await HTTPRequestAction(items).run(self.user, None, {}) - request_mock.assert_called_with(url="https://my.url.com", method='POST', timeout=3) + request_mock.assert_called_with(url="https://my.url.com", method='POST', timeout=ClientTimeout(3)) self.assertTrue(self.user.descriptions["behaviors"]["my_behavior"].success_action.run.called) self.assertTrue(self.user.variables.set.called) self.user.variables.set.assert_called_with("user_variable", {'data': 'value'}) @@ -66,7 +68,7 @@ async def test_render_params(self, request_mock: Mock): "value": "my_value" } await HTTPRequestAction(items).run(self.user, None, params) - request_mock.assert_called_with(url="https://my.url.com", method='POST', timeout=3, json={"param": "my_value"}) + request_mock.assert_called_with(url="https://my.url.com", method='POST', timeout=ClientTimeout(3), json={"param": "my_value"}) @patch('aiohttp.request') async def test_headers_fix(self, request_mock): @@ -89,4 +91,4 @@ async def test_headers_fix(self, request_mock): "header_1": "32", "header_2": "32.03", "header_3": b"d32" - }, method=HTTPRequestAction.DEFAULT_METHOD, timeout=self.TIMEOUT) + }, method=HTTPRequestAction.DEFAULT_METHOD, timeout=ClientTimeout(self.TIMEOUT)) From 7fd1bdb4c4b2c699f31108ade793ddc4cb556505 Mon Sep 17 00:00:00 2001 From: Artem Batalov Date: Tue, 5 Apr 2022 15:54:01 +0300 Subject: [PATCH 115/116] Make postprocessor async --- smart_kit/start_points/main_loop_async_http.py | 2 +- smart_kit/start_points/main_loop_http.py | 2 +- smart_kit/start_points/main_loop_kafka.py | 2 +- smart_kit/start_points/postprocess.py | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/smart_kit/start_points/main_loop_async_http.py b/smart_kit/start_points/main_loop_async_http.py index 45906dbd..1ba27c20 100644 --- a/smart_kit/start_points/main_loop_async_http.py +++ b/smart_kit/start_points/main_loop_async_http.py @@ -155,7 +155,7 @@ async def process_message(self, message: SmartAppFromMessage, *args, **kwargs): await self.save_user(db_uid, user, message) stats += "Saving time: {} msecs\n".format(save_timer.msecs) log(stats, params={log_const.KEY_NAME: "timings"}) - self.postprocessor.postprocess(user, message) + await self.postprocessor.postprocess(user, message) return answer, stats, user async def get_health_check(self, request: aiohttp.web.Request): diff --git a/smart_kit/start_points/main_loop_http.py b/smart_kit/start_points/main_loop_http.py index d139bf19..21cf0b8a 100644 --- a/smart_kit/start_points/main_loop_http.py +++ b/smart_kit/start_points/main_loop_http.py @@ -84,7 +84,7 @@ def process_message(self, message: SmartAppFromMessage, *args, **kwargs): self.loop.run_until_complete(self.save_user(db_uid, user, message)) stats += "Saving time: {} msecs\n".format(save_timer.msecs) log(stats, user=user, params={log_const.KEY_NAME: "timings"}) - self.postprocessor.postprocess(user, message) + self.loop.run_until_complete(self.postprocessor.postprocess(user, message)) return answer, stats def _get_headers(self, environ): diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index e0436a13..3fd05983 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -458,7 +458,7 @@ async def process_message(self, mq_message, consumer, kafka_key, stats, log_para "uid": user.id, "db_version": str(user.variables.get(user.USER_DB_VERSION))}, level="WARNING") - self.postprocessor.postprocess(user, message) + await self.postprocessor.postprocess(user, message) smart_kit_metrics.counter_save_collision_tries_left(self.app_name) consumer.commit_offset(mq_message) diff --git a/smart_kit/start_points/postprocess.py b/smart_kit/start_points/postprocess.py index 08752619..377d97cf 100644 --- a/smart_kit/start_points/postprocess.py +++ b/smart_kit/start_points/postprocess.py @@ -3,19 +3,19 @@ class PostprocessMainLoop: - def postprocess(self, user, message, *args, **kwargs): + async def postprocess(self, user, message, *args, **kwargs): pass class PostprocessCompose(PostprocessMainLoop): postprocessors: List[PostprocessMainLoop] = [] - def postprocess(self, user, message, *args, **kwargs): + async def postprocess(self, user, message, *args, **kwargs): for processor in self.postprocessors: - processor.postprocess(user, message, *args, **kwargs) + await processor.postprocess(user, message, *args, **kwargs) -def postprocessor_compose(*args: List[Type[PostprocessMainLoop]]): +def postprocessor_compose(*args: Type[PostprocessMainLoop]): class Compose(PostprocessCompose): postprocessors = [processor_cls() for processor_cls in args] return Compose From ae30a18db5d8689366d5827a93c60314711416ee Mon Sep 17 00:00:00 2001 From: Artem Batalov Date: Mon, 4 Apr 2022 15:43:57 +0300 Subject: [PATCH 116/116] Access to monitoring singleton only via monitoring module --- core/db_adapter/aioredis_adapter.py | 8 ++++---- core/db_adapter/aioredis_sentinel_adapter.py | 8 ++++---- core/db_adapter/ceph/ceph_adapter.py | 4 ++-- core/db_adapter/db_adapter.py | 16 ++++++++-------- core/db_adapter/ignite_adapter.py | 6 +++--- core/mq/kafka/async_kafka_publisher.py | 6 +++--- core/mq/kafka/kafka_consumer.py | 6 +++--- core/mq/kafka/kafka_publisher.py | 10 +++++----- core/request/rest_request.py | 4 ++-- core/unified_template/unified_template.py | 4 ++-- core/utils/rerunable.py | 4 ++-- .../form_filling_scenario.py | 4 ++-- .../tree_scenario/tree_scenario.py | 4 ++-- smart_kit/start_points/base_main_loop.py | 5 ++--- smart_kit/utils/monitoring.py | 12 ++++++------ 15 files changed, 50 insertions(+), 51 deletions(-) diff --git a/core/db_adapter/aioredis_adapter.py b/core/db_adapter/aioredis_adapter.py index 38e498b9..b85d6084 100644 --- a/core/db_adapter/aioredis_adapter.py +++ b/core/db_adapter/aioredis_adapter.py @@ -5,7 +5,7 @@ from core.db_adapter.db_adapter import AsyncDBAdapter from core.db_adapter import error -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring from core.logging.logger_utils import log @@ -21,15 +21,15 @@ def __init__(self, config=None): except KeyError: pass - @monitoring.got_histogram_decorate("save_time") + @monitoring.monitoring.got_histogram_decorate("save_time") async def save(self, id, data): return await self._async_run(self._save, id, data) - @monitoring.got_histogram_decorate("save_time") + @monitoring.monitoring.got_histogram_decorate("save_time") async def replace_if_equals(self, id, sample, data): return await self._async_run(self._replace_if_equals, id, sample, data) - @monitoring.got_histogram_decorate("get_time") + @monitoring.monitoring.got_histogram_decorate("get_time") async def get(self, id): return await self._async_run(self._get, id) diff --git a/core/db_adapter/aioredis_sentinel_adapter.py b/core/db_adapter/aioredis_sentinel_adapter.py index 3f890df3..5d01a59f 100644 --- a/core/db_adapter/aioredis_sentinel_adapter.py +++ b/core/db_adapter/aioredis_sentinel_adapter.py @@ -5,7 +5,7 @@ from aioredis.sentinel import Sentinel from core.db_adapter.db_adapter import AsyncDBAdapter from core.db_adapter import error -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring from core.logging.logger_utils import log @@ -22,16 +22,16 @@ def __init__(self, config=None): except KeyError: pass - @monitoring.got_histogram_decorate("save_time") + @monitoring.monitoring.got_histogram_decorate("save_time") async def save(self, id, data): return await self._async_run(self._save, id, data) - @monitoring.got_histogram_decorate("save_time") + @monitoring.monitoring.got_histogram_decorate("save_time") async def replace_if_equals(self, id, sample, data): return await self._async_run(self._replace_if_equals, id, sample, data) - @monitoring.got_histogram_decorate("get_time") + @monitoring.monitoring.got_histogram_decorate("get_time") async def get(self, id): return await self._async_run(self._get, id) diff --git a/core/db_adapter/ceph/ceph_adapter.py b/core/db_adapter/ceph/ceph_adapter.py index 97d91bb2..ce75c8b9 100644 --- a/core/db_adapter/ceph/ceph_adapter.py +++ b/core/db_adapter/ceph/ceph_adapter.py @@ -9,7 +9,7 @@ from core.db_adapter.ceph.ceph_io import CephIO from core.db_adapter.db_adapter import DBAdapter from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring ssl._create_default_https_context = ssl._create_unverified_context @@ -35,7 +35,7 @@ def connect(self): params={log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE}, level="ERROR", exc_info=True) - monitoring.got_counter("ceph_connection_exception") + monitoring.monitoring.got_counter("ceph_connection_exception") raise @property diff --git a/core/db_adapter/db_adapter.py b/core/db_adapter/db_adapter.py index 0ff718c3..9582dd85 100644 --- a/core/db_adapter/db_adapter.py +++ b/core/db_adapter/db_adapter.py @@ -5,7 +5,7 @@ from core.logging.logger_utils import log from core.model.factory import build_factory from core.model.registered import Registered -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring from core.utils.rerunable import Rerunable db_adapters = Registered() @@ -65,15 +65,15 @@ def path_exists(self, path): def mtime(self, path): return self._run(self._mtime, path) - @monitoring.got_histogram_decorate("save_time") + @monitoring.monitoring.got_histogram_decorate("save_time") def save(self, id, data): return self._run(self._save, id, data) - @monitoring.got_histogram_decorate("save_time") + @monitoring.monitoring.got_histogram_decorate("save_time") def replace_if_equals(self, id, sample, data): return self._run(self._replace_if_equals, id, sample, data) - @monitoring.got_histogram_decorate("get_time") + @monitoring.monitoring.got_histogram_decorate("get_time") def get(self, id): return self._run(self._get, id) @@ -109,15 +109,15 @@ async def _path_exists(self, path): async def path_exists(self, path): return await self._async_run(self._path_exists, path) - @monitoring.got_histogram("save_time") + @monitoring.monitoring.got_histogram("save_time") async def save(self, id, data): return await self._async_run(self._save, id, data) - @monitoring.got_histogram("save_time") + @monitoring.monitoring.got_histogram("save_time") async def replace_if_equals(self, id, sample, data): return await self._async_run(self._replace_if_equals, id, sample, data) - @monitoring.got_histogram("get_time") + @monitoring.monitoring.got_histogram("get_time") async def get(self, id): return await self._async_run(self._get, id) @@ -144,5 +144,5 @@ async def _async_run(self, action, *args, _try_count=None, **kwargs): result = await self._async_run(action, *args, _try_count=_try_count, **kwargs) counter_name = self._get_counter_name() if counter_name: - monitoring.got_counter(f"{counter_name}_exception") + monitoring.monitoring.got_counter(f"{counter_name}_exception") return result diff --git a/core/db_adapter/ignite_adapter.py b/core/db_adapter/ignite_adapter.py index b5405ec3..53d6795c 100644 --- a/core/db_adapter/ignite_adapter.py +++ b/core/db_adapter/ignite_adapter.py @@ -11,7 +11,7 @@ from core.db_adapter import error from core.db_adapter.db_adapter import AsyncDBAdapter from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring class IgniteAdapter(AsyncDBAdapter): @@ -55,7 +55,7 @@ async def connect(self): params={log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE}, level="ERROR", exc_info=True) - monitoring.got_counter("ignite_connection_exception") + monitoring.monitoring.got_counter("ignite_connection_exception") raise async def _save(self, id, data): @@ -75,7 +75,7 @@ async def get_cache(self): if self._client is None: log('Attempt to recreate ignite instance', level="WARNING") await self.connect() - monitoring.got_counter("ignite_reconnection") + monitoring.monitoring.got_counter("ignite_reconnection") return self._cache @property diff --git a/core/mq/kafka/async_kafka_publisher.py b/core/mq/kafka/async_kafka_publisher.py index 76ad269b..5bf271ae 100644 --- a/core/mq/kafka/async_kafka_publisher.py +++ b/core/mq/kafka/async_kafka_publisher.py @@ -3,7 +3,7 @@ import core.logging.logger_constants as log_const from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring from core.mq.kafka.kafka_publisher import KafkaPublisher @@ -30,7 +30,7 @@ def send(self, value, key=None, topic_key=None, headers=None): } log("KafkaProducer: Local producer queue is full (%(queue_amount)s messages awaiting delivery):" " try again\n", params=params, level="ERROR") - monitoring.got_counter("kafka_producer_exception") + monitoring.monitoring.got_counter("kafka_producer_exception") def send_to_topic(self, value, key=None, topic=None, headers=None): try: @@ -52,7 +52,7 @@ def send_to_topic(self, value, key=None, topic=None, headers=None): } log("KafkaProducer: Local producer queue is full (%(queue_amount)s messages awaiting delivery):" " try again\n", params=params, level="ERROR") - monitoring.got_counter("kafka_producer_exception") + monitoring.monitoring.got_counter("kafka_producer_exception") def _poll_for_callbacks(self): poll_timeout = self._config.get("poll_timeout", 1) diff --git a/core/mq/kafka/kafka_consumer.py b/core/mq/kafka/kafka_consumer.py index afd0cab4..6e8d22d1 100644 --- a/core/mq/kafka/kafka_consumer.py +++ b/core/mq/kafka/kafka_consumer.py @@ -10,7 +10,7 @@ import core.logging.logger_constants as log_const from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring from core.mq.kafka.base_kafka_consumer import BaseKafkaConsumer @@ -102,7 +102,7 @@ def _error_callback(self, err): log_const.KEY_NAME: log_const.EXCEPTION_VALUE } log("KafkaConsumer: Error: %(error)s", params=params, level="WARNING") - monitoring.got_counter("kafka_consumer_exception") + monitoring.monitoring.got_counter("kafka_consumer_exception") # noinspection PyMethodMayBeStatic def _process_message(self, msg: KafkaMessage): @@ -111,7 +111,7 @@ def _process_message(self, msg: KafkaMessage): if err.code() == KafkaError._PARTITION_EOF: return None else: - monitoring.got_counter("kafka_consumer_exception") + monitoring.monitoring.got_counter("kafka_consumer_exception") params = { "code": err.code(), "pid": os.getpid(), diff --git a/core/mq/kafka/kafka_publisher.py b/core/mq/kafka/kafka_publisher.py index 51ba44d7..f87e6dca 100644 --- a/core/mq/kafka/kafka_publisher.py +++ b/core/mq/kafka/kafka_publisher.py @@ -7,7 +7,7 @@ import core.logging.logger_constants as log_const from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring from core.mq.kafka.base_kafka_publisher import BaseKafkaPublisher @@ -41,7 +41,7 @@ def send(self, value, key=None, topic_key=None, headers=None): } log("KafkaProducer: Local producer queue is full (%(queue_amount)s messages awaiting delivery):" " try again\n", params=params, level="ERROR") - monitoring.got_counter("kafka_producer_exception") + monitoring.monitoring.got_counter("kafka_producer_exception") self._poll() def send_to_topic(self, value, key=None, topic=None, headers=None): @@ -64,7 +64,7 @@ def send_to_topic(self, value, key=None, topic=None, headers=None): } log("KafkaProducer: Local producer queue is full (%(queue_amount)s messages awaiting delivery):" " try again\n", params=params, level="ERROR") - monitoring.got_counter("kafka_producer_exception") + monitoring.monitoring.got_counter("kafka_producer_exception") self._poll() def _poll(self): @@ -81,7 +81,7 @@ def _error_callback(self, err): log_const.KEY_NAME: log_const.EXCEPTION_VALUE } log("KafkaProducer: Error: %(error)s", params=params, level="ERROR") - monitoring.got_counter("kafka_producer_exception") + monitoring.monitoring.got_counter("kafka_producer_exception") def _delivery_callback(self, err, msg): if err: @@ -101,7 +101,7 @@ def _delivery_callback(self, err, msg): log_const.KEY_NAME: log_const.EXCEPTION_VALUE}, level="ERROR", exc_info=True) - monitoring.got_counter("kafka_producer_exception") + monitoring.monitoring.got_counter("kafka_producer_exception") def close(self): self._producer.flush(self._config["flush_timeout"]) diff --git a/core/request/rest_request.py b/core/request/rest_request.py index b92b0018..f787614e 100644 --- a/core/request/rest_request.py +++ b/core/request/rest_request.py @@ -2,7 +2,7 @@ from timeout_decorator import timeout_decorator from core.request.base_request import BaseRequest from core.utils.exception_handlers import exc_handler -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring class RestNoMethodSpecifiedException(Exception): @@ -37,7 +37,7 @@ def run(self, data, params=None): return method(data) def on_timeout_error(self, *args, **kwarg): - monitoring.got_counter("core_rest_run_timeout") + monitoring.monitoring.got_counter("core_rest_run_timeout") def _requests_get(self, params): return requests.get(self.url, params=params, **self.rest_args).text diff --git a/core/unified_template/unified_template.py b/core/unified_template/unified_template.py index 1832ba03..3cd2191d 100644 --- a/core/unified_template/unified_template.py +++ b/core/unified_template/unified_template.py @@ -5,7 +5,7 @@ import core.logging.logger_constants as log_const from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring UNIFIED_TEMPLATE_TYPE_NAME = "unified_template" @@ -49,7 +49,7 @@ def render(self, *args, **kwargs): "params_dict_str": str(params_dict)}, level="ERROR", exc_info=True) - monitoring.got_counter("core_jinja_template_error") + monitoring.monitoring.got_counter("core_jinja_template_error") raise return result diff --git a/core/utils/rerunable.py b/core/utils/rerunable.py index 369a6ee8..431cf7b3 100644 --- a/core/utils/rerunable.py +++ b/core/utils/rerunable.py @@ -2,7 +2,7 @@ import core.logging.logger_constants as log_const from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring class Rerunable(): @@ -44,7 +44,7 @@ def _run(self, action, *args, _try_count=None, **kwargs): result = self._run(action, *args, _try_count=_try_count, **kwargs) counter_name = self._get_counter_name() if counter_name: - monitoring.got_counter(f"{counter_name}_exception") + monitoring.monitoring.got_counter(f"{counter_name}_exception") return result def _get_counter_name(self): diff --git a/scenarios/scenario_descriptions/form_filling_scenario.py b/scenarios/scenario_descriptions/form_filling_scenario.py index fc31558a..9fe34b3f 100644 --- a/scenarios/scenario_descriptions/form_filling_scenario.py +++ b/scenarios/scenario_descriptions/form_filling_scenario.py @@ -2,7 +2,7 @@ from typing import Dict, Any from core.basic_models.scenarios.base_scenario import BaseScenario -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring from core.logging.logger_utils import log import scenarios.logging.logger_constants as log_const @@ -174,7 +174,7 @@ async def get_reply(self, user, text_preprocessing_result, reply_actions, field, user.last_scenarios.delete(self.id) return action_messages - @monitoring.got_histogram_decorate("scenario_time") + @monitoring.monitoring.got_histogram_decorate("scenario_time") async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = None): form = self._get_form(user) user.last_scenarios.add(self.id, text_preprocessing_result) diff --git a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py index a756d02a..f69c797b 100644 --- a/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py +++ b/scenarios/scenario_descriptions/tree_scenario/tree_scenario.py @@ -4,7 +4,7 @@ from scenarios.scenario_descriptions.form_filling_scenario import FormFillingScenario from scenarios.scenario_descriptions.tree_scenario.tree_scenario_node import TreeScenarioNode from core.model.factory import dict_factory -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring from core.logging.logger_utils import log import scenarios.logging.logger_constants as log_const from scenarios.scenario_models.history import Event, HistoryConstants @@ -83,7 +83,7 @@ def get_fields_data(self, main_form, form_type): all_forms_fields.update(form_field_data) return all_forms_fields - @monitoring.got_histogram_decorate("scenario_time") + @monitoring.monitoring.got_histogram_decorate("scenario_time") async def run(self, text_preprocessing_result, user, params: Dict[str, Any] = None): main_form = self._get_form(user) user.last_scenarios.add(self.id, text_preprocessing_result) diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index 2e085d99..9b301612 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -8,7 +8,7 @@ from core.db_adapter.db_adapter import DBAdapterException from core.db_adapter.db_adapter import db_adapter_factory from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring from core.monitoring.healthcheck_handler import RootResource from core.monitoring.twisted_server import TwistedServer from core.model.base_user import BaseUser @@ -95,8 +95,7 @@ def _create_health_check_server(self, settings): def _init_monitoring_config(self, template_settings): monitoring_config = template_settings["monitoring"] - monitoring.apply_config(monitoring_config) - smart_kit_metrics.apply_config(monitoring_config) + monitoring.monitoring.apply_config(monitoring_config) smart_kit_metrics.init_metrics(app_name=self.app_name) async def load_user(self, db_uid, message): diff --git a/smart_kit/utils/monitoring.py b/smart_kit/utils/monitoring.py index 2b981fd1..4099ff25 100644 --- a/smart_kit/utils/monitoring.py +++ b/smart_kit/utils/monitoring.py @@ -1,6 +1,6 @@ from core.logging.logger_constants import KEY_NAME from core.logging.logger_utils import log -from core.monitoring.monitoring import monitoring +from core.monitoring import monitoring def _filter_monitoring_msg(msg): @@ -39,7 +39,7 @@ def init_metrics(self, app_name): "Incoming message validation error.") def _get_or_create_counter(self, monitoring_msg, descr, labels=()): - counter = monitoring.get_counter(monitoring_msg, descr, labels) + counter = monitoring.monitoring.get_counter(monitoring_msg, descr, labels) if counter is None: raise MetricDisabled('counter disabled') return counter @@ -182,22 +182,22 @@ def counter_mq_long_waiting(self, app_name): @silence_it def sampling_load_time(self, app_name, value): monitoring_msg = "{}_load_time".format(app_name) - monitoring.got_histogram_observe(_filter_monitoring_msg(monitoring_msg), value) + monitoring.monitoring.got_histogram_observe(_filter_monitoring_msg(monitoring_msg), value) @silence_it def sampling_script_time(self, app_name, value): monitoring_msg = "{}_script_time".format(app_name) - monitoring.got_histogram_observe(_filter_monitoring_msg(monitoring_msg), value) + monitoring.monitoring.got_histogram_observe(_filter_monitoring_msg(monitoring_msg), value) @silence_it def sampling_save_time(self, app_name, value): monitoring_msg = "{}_save_time".format(app_name) - monitoring.got_histogram_observe(_filter_monitoring_msg(monitoring_msg), value) + monitoring.monitoring.got_histogram_observe(_filter_monitoring_msg(monitoring_msg), value) @silence_it def sampling_mq_waiting_time(self, app_name, value): monitoring_msg = "{}_mq_waiting_time".format(app_name) - monitoring.got_histogram_observe(_filter_monitoring_msg(monitoring_msg), value) + monitoring.monitoring.got_histogram_observe(_filter_monitoring_msg(monitoring_msg), value) smart_kit_metrics = Metrics()