diff --git a/README.md b/README.md index c992a58..ac652f8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # If you want to use conda, not required 1. Make sure that you have `conda` installed. [Recommend this article if on Mac, just do through step 2](https://caffeinedev.medium.com/how-to-install-tensorflow-on-m1-mac-8e9b91d93706). -2. Create and activate a new conda environment, e.g., `transformers-api` with python 3.10. +2. Create and activate a new conda environment, e.g., `transformers-api` with python 3.12. ```bash conda create --name transformers-api python=3.12 conda activate transformers-api @@ -10,34 +10,181 @@ conda activate transformers-api 3. Run `which pip` and `which python` to verify path to make sure that your `python` and `pip` binaries are coming from your `conda` virtual environment. Note that the order in which you install conda vs. pip matters to set virtual env priorities. # Getting Started Locally (Start here if not using conda, just make sure you have the right version of python and pip installed) + +This procedure runs the c3po-model-server without connecting to Mattermost. +It builds and runs the containers, initializes the data, and runs pytests. + +VITAL: Do not use the Mattermost Token. +Use the integration environment + + +The containers used by c3po-model-server have been built using either: + +• docker-compose up --build +• docker compose build and docker compose up + +WARNING: Support for docker-compose, as opposed to docker compose, officially ended in June 2023. + +ERROR: Sometimes the unsupported version fails. + + 1. Install `poetry` version 1.8.5: `pip install poetry==1.8.5` (or use `pipx` [on link here](https://python-poetry.org/docs/1.4#installing-with-pipx) if you prefer isolated envs and you don't have `conda` managing your env) 2. Create and enter the virtual environment: `poetry shell`. Note: if you use conda, this step may not be necessary. 3. Install the dependencies `poetry install` -4. In `c3po-model-server/app/core/env_var`, create a `secrets.env` file and ensure it is on the `.gitignore`. Add the following for local dev: -```sh -MM_TOKEN="" -``` - -5. Launch postgres, pgadmin, and minio via docker-compose `docker-compose up --build`. +4. Optionally, build postgres, pgadmin, and minio via `docker compose build`. +5. Launch postgres, pgadmin, and minio via docker-compose `docker compose up`. +. 6. Visit `localhost:9001`. Login with user:`miniouser` and password:`minioadmin`. This is the minio console. 7. Visit `localhost:5050`. Login with email:`user@test.com` and password:`admin`. This is the pgadmin console. **See notes below for important details** -8. Run the app db init script `./scripts/init.sh` +8. Run the app db init script `ENVIRONMENT=integration ./scripts/init.sh` -9. Keeping your docker containers running, start the app in a new terminal (activate your conda env first) with `ENVIRONMENT=local uvicorn app.main:versioned_app --reload`. +9. Keeping your docker containers running, start the app in a new terminal (activate your conda env first) with `ENVIRONMENT=integration uvicorn app.main:versioned_app --reload`. 10. Open `localhost:8000/v1/docs` and start interacting with swagger! -11. Run tests and get coverage with `ENVIRONMENT=local pytest --cov`, and get html reports for vs code live server (or any server) with `ENVIRONMENT=local pytest --cov --cov-report=html:coverage_re` +11. Run tests and get coverage with `ENVIRONMENT=integration pytest --cov`, and get html reports for vs code live server (or any server) with `ENVIRONMENT=integration pytest --cov --cov-report=html:coverage_re` 12. You can shut down and your db / minio data will persist via docker volumes. +# Getting Started with Mattermost + +1. In `c3po-model-server/app/core/env_var`, create a `secrets.env` file and ensure it is on the `.gitignore`. Add the following for local dev: +```sh +MM_TOKEN="" +``` +2. Run the app db init script `ENVIRONMENT=local ./scripts/init.sh` + +3. Keeping your docker containers running, start the app in a new terminal (activate your conda env first) with `ENVIRONMENT=local uvicorn app.main:versioned_app --reload`. + +4. Run tests and get coverage with `ENVIRONMENT=local pytest --cov`, and get html reports for vs code live server (or any server) with `ENVIRONMENT=local pytest --cov --cov-report=html:coverage_re` + +# Using VM dgwonub22 +NOTE: “Currently” in this section means Oct. 2025. + +PPG runs on macOS and on Linux. + +CAITT members whose primary computer runs Windows use the VM dgwonub22. +They do not use WSL2. + +Other CAITT members may use the VM in addition to their personal laptops. + +The VM is currently running Ubuntu 22.04.5 LTS. + +## Sudo privileges: + +Because the dgwonub22 installation of Docker requires Root privileges, you must have sudo privileges to build, start, stop, and monitor PPG containers. See IT if necessary. + +## Collisions on VM dgwonub22: + +Currently the following CAITT members have access to VM dgwonub22: + +da32459 Dan Gwon +em23761 Emilie Cowen +jo25802 John Holodnak +pa27879 Paul Gibby +ds14673 Scott Briere IT +ju21882 Justin O’Brien IT +ve22990 Vern Rivet IT + +WARNING: More than one of the above non-IT people may attempt to use the c3po-model-server simultaneously, possibly stopping, starting, and rebuilding the containers. + +Currently there is no formal system for coordinating this. + +Note: An informal way has been to disable sudo privileges for all but one user, thus preventing access to the containers. If you suddenly encounter this, contact Emilie. + +## Disk space on VM dgwonub22: + +VM dgwonub22 has disk space limits that can not be increased. + +Currently, disk usage is not monitored. + +You may get error messages when +• Building or starting the containers +• Running the app db init script, ./scripts/init.sh. +• Running the pytests + +BEST PRACTICE: When running the app db script, always read the displayed event log and grep for “space”. It is easy to overlook this. + +Examples: +• db-1 | 2025-10-22 17:49:06.144 UTC [1] FATAL: could not write lock file "postmaster.pid": No space left on device +• pgadmin-1 | OSError: [Errno 28] No space left on device + +Also, the effects can be progressive. If you start seeing inexplicable errors, check your disk space. + +If you are out of disk space, contact Emilie, who will contact IT. + +## Other Apps on VM dgwonub22: + +Currently the VM hosts: + +• PPG c3po-model-server +• UDL Data Collection + +Currently there are no known dependencies between these apps. + +WARNING, they have different environments, conda or otherwise, so be careful if jumping back and forth between them. + +## Logging in and setting environments: + +On your computer, enter: +$ ssh LincolnId@dgwonub22.llan.ll.mit.edu + +Use your Lincoln password. + +If you are using conda, +If you have not set your shell to automatically load conda: +Enter: +$ eval "$(/home/jo20812/miniconda3/bin/conda shell.bash hook)" + +Then enter: +$ conda activate transformers-api + +## Browsing in to VM dgwonub22: + +PPG uses Chrome for: + +• The minIO Object Store console +• The PostgreSQL console +• Using SWAGGER to access the c3po-model-server API + +Mac users use localhost to access the server running on their Mac +VM users must use an IP address to access the server running on the VM. + +## Browse to MinIO Object Store Console +Chrome: +http://172.25.252.52:9001/login +U: miniouser +P: minioadmin + +Should see MinIO Object Store console. + + +## Browse to the PostgreSQL Console +Chrome: +http://172.25.252.52:5050 +E: user@test.com +P: admin + +Should see PostgreSQL console. + +## Browse to the SWAGGER UI + +You need to expose the server to the external network by giving the host parameter + +Launch the server app with: +ENVIRONMENT=integration uvicorn app.main:versioned_app --reload –host 0.0.0.0 + +Then you can access the swagger UI at http://dgwonub22:llan.ll.mit.edu:8000/v1/docs + + + # Adding a package Note: instructions included in [tutorial linked here](https://realpython.com/dependency-management-python-poetry/) 1. Add the package, e.g., `poetry add transformers` or `poetry add transformers --group ` where `` is the dependency group name, e.g., `test` or `dev`. diff --git a/app/core/config.py b/app/core/config.py index 0505415..bb60b4b 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -87,6 +87,8 @@ def get_env_file(environment_settings_in): elif environment_settings_in.environment == 'development': env_file = (os.path.join(BASEDIR, "development.env"), os.path.join(BASEDIR, "secrets.env")) + elif environment_settings_in.environment == 'integration': + env_file = os.path.join(BASEDIR, "integration.env") else: env_file = os.path.join(BASEDIR, "test.env") diff --git a/app/core/env_var/development.env b/app/core/env_var/development.env index b9a090b..a9b1470 100644 --- a/app/core/env_var/development.env +++ b/app/core/env_var/development.env @@ -20,3 +20,4 @@ MM_NITMRE_BASE_URL="${MM_BASE_URL}/nitmre" MM_TOKEN="overwrite-in-secrets.env" DEFAULT_SHA256_L13B_SNOOZY="997072bd77078c82131e7becf3fc4b090efec43a1f480bbde0e401ffe5145688" DEFAULT_SHA256_Q4_K_M="14789fe0f3a1e9c7b1b92ef4d82d90e7d312f6fdda43b524067569c0795203d8" + diff --git a/app/core/env_var/integration.env b/app/core/env_var/integration.env new file mode 100644 index 0000000..efb8f2d --- /dev/null +++ b/app/core/env_var/integration.env @@ -0,0 +1,18 @@ +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_SERVER=localhost +POSTGRES_PORT=5432 +POSTGRES_DB=postgres + +S3_BUCKET_NAME=test +S3_ACCESS_KEY=miniouser +S3_SECRET_KEY=minioadmin +S3_ENDPOINT_URL=http://localhost:9000 +S3_REGION = "" +S3_SECURE=False + +DOCS_UI_ROOT_PATH="" +LOG_LEVEL="DEBUG" + +DEFAULT_SHA256_L13B_SNOOZY="997072bd77078c82131e7becf3fc4b090efec43a1f480bbde0e401ffe5145688" +DEFAULT_SHA256_Q4_K_M="14789fe0f3a1e9c7b1b92ef4d82d90e7d312f6fdda43b524067569c0795203d8" diff --git a/app/initial_data.py b/app/initial_data.py index 947daae..ce8f9f5 100644 --- a/app/initial_data.py +++ b/app/initial_data.py @@ -58,7 +58,7 @@ def get_db(environment: str, migration_toggle: bool) -> Union[Session, None]: # clear DB if local or staging as long as not actively testing migrating # note: reenabled wipe_db for staging (['local', 'staging']) due to db schema changes, remove staging when schema stable - if (environment in ['local', 'staging'] and migration_toggle is False): + if (environment in ['local', 'integration', 'staging'] and migration_toggle is False): logger.info("Clearing database") drop_constraints() wipe_db() @@ -66,7 +66,7 @@ def get_db(environment: str, migration_toggle: bool) -> Union[Session, None]: # all environments need to initialize the database # prod only if migration toggle is on - if (environment in ['local', 'development', 'test', 'staging'] or (environment == 'production' and migration_toggle is True)): + if (environment in ['local', 'development', 'integration', 'test', 'staging'] or (environment == 'production' and migration_toggle is True)): logger.info("Creating database schema and tables") db = SessionLocal() init() @@ -91,12 +91,12 @@ def get_s3(environment: str, db: Session) -> Union[S3Client, None]: s3 = None # setup s3 client if available (i.e., not in unit tests) - if (environment in ['local', 'development', 'staging', 'production']): + if (environment in ['local', 'development', 'integration', 'staging', 'production']): logger.info("Connecting S3 client") s3 = build_client() logger.info("S3 client connected") - if (environment in ['local', 'development']): + if (environment in ['local', 'development', 'integration']): logger.info("Setting up S3 bucket") init_s3_bucket(s3) logger.info("S3 bucket set up.") @@ -537,7 +537,7 @@ def main() -> None: if len(args) == 1 and args[0] == '--toggle-migration': migration_toggle = True - # environment can be one of 'local', 'test', 'staging', 'production' + # environment can be one of 'local', 'test', 'integration', 'staging', 'production' environment = environment_settings.environment logger.info(f"Using initialization environment: {environment}") @@ -547,10 +547,10 @@ def main() -> None: s3 = get_s3(environment, db) - if (environment != 'test'): + if (environment != 'test') and (environment != 'integration'): init_large_objects(db) - if (environment == 'local'): + if (environment == 'local') or (environment == 'integration'): init_large_objects_local(s3, db) elif (environment == 'staging' or (environment == 'production' and migration_toggle is True)): init_large_objects_p1(s3, db) diff --git a/scripts/init-integration.sh b/scripts/init-integration.sh new file mode 100755 index 0000000..a61b54d --- /dev/null +++ b/scripts/init-integration.sh @@ -0,0 +1,4 @@ +#! /usr/bin/env bash + +# Create initial data in DB +ENVIRONMENT=integration python -m app.initial_data \ No newline at end of file diff --git a/scripts/init.sh b/scripts/init.sh index 9acb632..2543da7 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -8,6 +8,10 @@ elif [ "$ENVIRONMENT" = "development" ] then echo "running development script" ./scripts/init-development.sh +elif [ "$ENVIRONMENT" = "integration" ] +then + echo "running integration script" + ./scripts/init-integration.sh elif [ "$ENVIRONMENT" = "staging" ] then echo "running staging script" diff --git a/tests/mattermost/test_mattermost_crud.py b/tests/mattermost/test_mattermost_crud.py index 5c4617d..f1152f8 100644 --- a/tests/mattermost/test_mattermost_crud.py +++ b/tests/mattermost/test_mattermost_crud.py @@ -166,7 +166,7 @@ def test_crud_mattermost(db: Session): def test_populate_mm_user_team_info_local(db: Session, monkeypatch: MonkeyPatch): # test user info in database - if environment_settings.environment == 'test': + if (environment_settings.environment == 'test') or (environment_settings.environment == 'integration'): return # see note at tests/mattermost/test_mattermost_router.py::test_get_mattermost_user_info diff --git a/tests/mattermost/test_mattermost_router.py b/tests/mattermost/test_mattermost_router.py index f49ffdc..886ea0f 100644 --- a/tests/mattermost/test_mattermost_router.py +++ b/tests/mattermost/test_mattermost_router.py @@ -88,6 +88,10 @@ def test_upload_mattermost_user_info_invalid_format(client: TestClient): # returns 422 def test_upload_mattermost_user_info_invalid_input(client: TestClient): + + if (environment_settings.environment == 'test') or (environment_settings.environment == 'integration'): + return + response = client.post( "/mattermost/user/upload", headers={}, @@ -100,7 +104,7 @@ def test_upload_mattermost_user_info_invalid_input(client: TestClient): # test user upload endpoint def test_upload_mattermost_user_info(client: TestClient, monkeypatch: MonkeyPatch): - if environment_settings.environment == 'test': + if (environment_settings.environment == 'test') or (environment_settings.environment == 'integration'): return # see note at tests/mattermost/test_mattermost_router.py::test_get_mattermost_user_info @@ -139,7 +143,7 @@ def test_get_mattermost_user_info_invalid_input(client: TestClient): # test user get endpoint def test_get_mattermost_user_info(client: TestClient, monkeypatch: MonkeyPatch): - if environment_settings.environment == 'test': + if (environment_settings.environment == 'test') or (environment_settings.environment == 'integration'): return # This test creates db entries for mm user and channel; these are @@ -168,6 +172,10 @@ def test_upload_mattermost_documents_invalid_format(client: TestClient): # returns 422 def test_upload_mattermost_documents_invalid_input(client: TestClient): + + if (environment_settings.environment == 'test') or (environment_settings.environment == 'integration'): + return + response = client.post( "/mattermost/documents/upload", headers={}, diff --git a/tests/mattermost/test_mattermost_utils.py b/tests/mattermost/test_mattermost_utils.py index 0a870b8..7378ff2 100644 --- a/tests/mattermost/test_mattermost_utils.py +++ b/tests/mattermost/test_mattermost_utils.py @@ -6,7 +6,7 @@ def test_mattermost_bot(): # test mattermost api calls for user, team, and channel info - if environment_settings.environment == 'test': + if (environment_settings.environment == 'test') or (environment_settings.environment == 'integration'): return # test get user, teams diff --git a/tests/settings/test_settings.py b/tests/settings/test_settings.py index 52a2ee8..612139b 100644 --- a/tests/settings/test_settings.py +++ b/tests/settings/test_settings.py @@ -18,6 +18,12 @@ def test_env_file_name_test(): env_file = get_env_file(environment_settings) assert env_file == os.path.join(BASEDIR, "test.env") +@mock.patch.dict(os.environ, {"ENVIRONMENT": "integration"}) +def test_env_file_name_test(): + environment_settings = EnvironmentSettings() + env_file = get_env_file(environment_settings) + assert env_file == os.path.join(BASEDIR, "integration.env") + @mock.patch.dict(os.environ, {"ENVIRONMENT": "local"}) def test_env_file_name_local(): environment_settings = EnvironmentSettings()