diff --git a/.ruff.toml b/.ruff.toml index 69f832536..f5e81157b 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -3,3 +3,6 @@ line-length = 100 ignore = ["E501"] exclude = ["gen", "alembic"] + +[lint.per-file-ignores] +"model-engine/model_engine_server/inference/forwarding/celery_forwarder.py" = ["E402"] \ No newline at end of file diff --git a/model-engine/model_engine_server/inference/forwarding/celery_forwarder.py b/model-engine/model_engine_server/inference/forwarding/celery_forwarder.py index 27007969a..4b387faad 100644 --- a/model-engine/model_engine_server/inference/forwarding/celery_forwarder.py +++ b/model-engine/model_engine_server/inference/forwarding/celery_forwarder.py @@ -1,3 +1,7 @@ +from gevent import monkey + +monkey.patch_all() + import argparse import json from datetime import datetime, timedelta @@ -177,12 +181,7 @@ def start_celery_service( concurrency=concurrency, loglevel="INFO", optimization="fair", - # Don't use pool="solo" so we can send multiple concurrent requests over - # Historically, pool="solo" argument fixes the known issues of celery and some of the libraries. - # Particularly asyncio and torchvision transformers. This isn't relevant since celery-forwarder - # is quite lightweight - # TODO: we should probably use eventlet or gevent for the pool, since - # the forwarder is nearly the most extreme example of IO bound. + pool="gevent", ) worker.start() diff --git a/model-engine/requirements.in b/model-engine/requirements.in index d503f7b83..949413bf5 100644 --- a/model-engine/requirements.in +++ b/model-engine/requirements.in @@ -25,6 +25,7 @@ ddtrace==1.8.3 deprecation~=2.1 docker~=5.0 fastapi~=0.110.0 +gevent~=24.11.1 gitdb2~=2.0 gunicorn~=20.0 httptools==0.5.0 diff --git a/model-engine/requirements.txt b/model-engine/requirements.txt index 6e784ecc9..54af7eac0 100644 --- a/model-engine/requirements.txt +++ b/model-engine/requirements.txt @@ -167,6 +167,8 @@ frozenlist==1.3.3 # aiosignal fsspec==2023.10.0 # via huggingface-hub +gevent==24.11.1 + # via -r model-engine/requirements.in gitdb==4.0.10 # via gitpython gitdb2==2.0.6 @@ -175,8 +177,10 @@ gitpython==3.1.41 # via -r model-engine/requirements.in google-auth==2.21.0 # via kubernetes -greenlet==2.0.2 - # via sqlalchemy +greenlet==3.1.1 + # via + # gevent + # sqlalchemy gunicorn==20.1.0 # via -r model-engine/requirements.in h11==0.14.0 @@ -569,6 +573,10 @@ yarl==1.9.2 # aiohttp zipp==3.16.0 # via importlib-metadata +zope-event==5.0 + # via gevent +zope-interface==7.2 + # via gevent # The following packages are considered to be unsafe in a requirements file: setuptools==69.0.3 @@ -576,3 +584,5 @@ setuptools==69.0.3 # gunicorn # kubernetes # kubernetes-asyncio + # zope-event + # zope-interface