From 000038b345a1471434e821ec88187fb4ac221958 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 1 Mar 2023 22:15:41 -0800 Subject: [PATCH 1/3] add server_sider_cursors=True to create_engine https://stackoverflow.com/questions/17199113/psycopg2-leaking-memory-after-large-query --- _lib/db.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/_lib/db.py b/_lib/db.py index 23075dd5..dd34bef1 100644 --- a/_lib/db.py +++ b/_lib/db.py @@ -1,15 +1,13 @@ +from contextlib import contextmanager +import os import random +import re import sys import time -import re -import os -from contextlib import contextmanager from uuid import uuid4 -import logbook - import click - +import logbook _DATABASE_URI_RE = re.compile(r"(?P(?Psqlite|postgresql)(\+.*)?):\/\/(?P[^/]*)\/(?P.+)") @@ -24,14 +22,15 @@ def _create_sqlite(path): def _create_postgres(match): import sqlalchemy + from flask_app.models import db uri = match.group(0) db_name = match.group('db') try: - sqlalchemy.create_engine(uri).connect() + sqlalchemy.create_engine(uri, server_sider_cursors=True).connect() except sqlalchemy.exc.OperationalError: - engine = sqlalchemy.create_engine('{}://{}/postgres'.format(match.group('driver'), match.group('host'))) + engine = sqlalchemy.create_engine('{}://{}/postgres'.format(match.group('driver'), match.group('host')), server_sider_cursors=True) conn = engine.connect() conn.execute("commit") conn.execute("create database {} with encoding = 'UTF8'".format(db_name)) @@ -77,7 +76,7 @@ def wait(num_retries=60, retry_sleep_seconds=1): if retry > 0: time.sleep(retry_sleep_seconds) try: - sqlalchemy.create_engine(uri).connect() + sqlalchemy.create_engine(uri, server_sider_cursors=True).connect() except sqlalchemy.exc.OperationalError as e: if 'does not exist' in str(e): break @@ -130,9 +129,10 @@ def downgrade(): @contextmanager def _migrate_context(app=None): + import flask_migrate + from flask_app.app import create_app from flask_app.models import db - import flask_migrate if app is None: app = create_app() From 89679451ef4c007e14b265a93bf51fb2288f4958 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 1 Mar 2023 22:18:27 -0800 Subject: [PATCH 2/3] fix up apt deps --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b304839..ead57f78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev + sudo apt-get install libsasl2-dev libldap2-dev libssl-dev - name: Setup Python uses: actions/setup-python@v2 with: From 52b8161a65ab4abf9901b2d0a217f4707148feca Mon Sep 17 00:00:00 2001 From: Kevin Limburg Date: Thu, 2 Mar 2023 11:43:36 -0800 Subject: [PATCH 3/3] add more logging --- .github/workflows/build_docker.yml | 2 +- flask_app/app.py | 1 + flask_app/auth.py | 4 ++++ flask_app/utils/oauth2.py | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 0c434f54..1c368653 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -70,7 +70,7 @@ jobs: with: context: . file: docker/Dockerfile - push: ${{ github.event.release != null }} + push: true #${{ github.event.release != null }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha diff --git a/flask_app/app.py b/flask_app/app.py index 7b190bdc..fe20163f 100644 --- a/flask_app/app.py +++ b/flask_app/app.py @@ -36,6 +36,7 @@ def create_app(config=None, setup_logging=True): app.config.update(yaml.full_load(yaml_file)) app.config.update(config) + print(app.config) app.before_request(profile_request_start) app.after_request(profile_request_end) diff --git a/flask_app/auth.py b/flask_app/auth.py index d3e91b2a..aa56c41f 100644 --- a/flask_app/auth.py +++ b/flask_app/auth.py @@ -45,6 +45,7 @@ def testing_login(): def login(): credentials = request.get_json(silent=True) + _logger.info(f"Credentials request: {credentials}") if not isinstance(credentials, dict): error_abort('Credentials provided are not a JSON object') @@ -66,11 +67,13 @@ def login(): def _login_with_credentials(credentials): config = get_runtime_config_private_dict() + _logger.info(f"Private conf: {config}") username = credentials.get('username') password = credentials.get('password') email = _fix_email(username, config) user = User.query.filter_by(email=email).first() + _logger.info(f"Found user: {user}") if current_app.config['TESTING']: if user is None: @@ -82,6 +85,7 @@ def _login_with_credentials(credentials): if user is not None and user.password: if verify_password(password, user.password): + _logger.info(f"password: {password} db password: {user.password}") login_user(user) return _make_success_login_response(user) _logger.debug('Could not login user locally (no user or password mismatch)') diff --git a/flask_app/utils/oauth2.py b/flask_app/utils/oauth2.py index 6b72ed00..87eed1c1 100644 --- a/flask_app/utils/oauth2.py +++ b/flask_app/utils/oauth2.py @@ -46,6 +46,7 @@ def get_oauth2_identity_azure(auth_code, redirect_uri): """Gets identity from azure auth_code""" config_dict = config.get_runtime_config_private_dict() + _logger.info(f"PRivate Config: {config_dict}") client_id = config_dict['azure_oauth2_client_id'] client_secret = config_dict['azure_oauth2_client_secret'] authority = f"https://login.microsoftonline.com/{config_dict['azure_oauth2_tenant_id']}"