Align infra/test.sh & infra/lint.sh (#88)

The idea with this commit is that `infra/lint.sh` and `infra/test.sh` are the "source
of truth" for how we want to run linting and tests respectively. We then use
these scripts in `precommit`, `make`, Github Actions, and wherever else we want to
run tests and linting.

For Github Actions, we also now run linting and tests directly in the action
runner, rather than running through Docker. This resulted in a speed-up of the
checks.

Fixes #79, #67

Other changes include:
* Do not install postgresql client
* Install git and use precommit for linting
This should fail if it can't run
* Use development image for tests and lint
* Fix lint as reported by `pre-commit run --all-files`, `black`, and `isort`
* Prepend test commands with `poetry run`
* Ignore missing imports for ruamel
* Update .secrets.baseline to exclude poetry.lock
* Add make commands for format, lint and test
* Remove git check for detect-secrets lint check

Co-authored-by: Mathieu Leplatre <[email protected]>

Author: grahamalama

.github/workflows/lint-build.yaml

@@ -5,24 +5,14 @@ on: pull_request
 jobs:
   run_test:
     runs-on: ubuntu-latest
-
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Build lint image
-        uses: docker/build-push-action@v2
+      - uses: actions/checkout@v3
+      - name: Install poetry
+        run: pipx install poetry
+      - uses: actions/setup-python@v4
         with:
-          context: .
-          file: infra/Dockerfile
-          push: false
-          target: "lint"
-          tags: ghcr.io/${{ github.repository }}-lint:${{ github.sha }}
-
-      - name: Run lint
-        run: |-
-          docker run --rm \
-          -e JIRA_USERNAME \
-          -e JIRA_API_KEY \
-          -e BUGZILLA_API_KEY \
-          ghcr.io/${{ github.repository }}-lint:${{ github.sha }}
+          python-version: "3.9"
+          cache: "poetry"
+      - name: Install dependencies
+        run: poetry install
+      - run: infra/lint.sh

.github/workflows/test-build.yaml

@@ -5,28 +5,19 @@ on: pull_request
 jobs:
   run_test:
     runs-on: ubuntu-latest
-
+    env:
+      JIRA_USERNAME: fake_jira_username
+      JIRA_API_KEY: fake_jira_api_key
+      BUGZILLA_API_KEY: fake_bugzilla_api_key
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Build test image
-        uses: docker/build-push-action@v2
+      - uses: actions/checkout@v3
+      - name: Install poetry
+        run: pipx install poetry
+      - uses: actions/setup-python@v4
         with:
-          context: .
-          file: infra/Dockerfile
-          push: false
-          target: "test"
-          tags: ghcr.io/${{ github.repository }}:${{ github.sha }}
-
+          python-version: "3.9"
+          cache: "poetry"
+      - name: Install dependencies
+        run: poetry install
       - name: Run tests
-        run: |-
-          docker run --rm \
-          -e JIRA_USERNAME \
-          -e JIRA_API_KEY \
-          -e BUGZILLA_API_KEY \
-          ghcr.io/${{ github.repository }}:${{ github.sha }}
-        env:
-          JIRA_USERNAME: "fake_username"
-          JIRA_API_KEY: "fake_api_key"
-          BUGZILLA_API_KEY: "fake_api_key"
+        run: infra/test.sh

.pre-commit-config.yaml

@@ -1,17 +1,4 @@
-exclude: |
-  (?x)^(
-      .*/gunicorn_conf.py|
-      .*/test_gunicorn_conf.py
-  )$
 repos:
-  - repo: local
-    hooks:
-      - id: pylint
-        name: pylint
-        entry: poetry run pylint
-        language: system
-        exclude: ^tests/
-        types: [python]
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.1.0
     hooks:
@@ -27,57 +14,51 @@ repos:
         ]
         exclude: "infra/k8s/secret.json"
       - id: trailing-whitespace
+  - repo: local
+    hooks:
+      - id: pylint
+        name: pylint
+        entry: infra/lint.sh pylint
+        language: script
+        types: [python]
   - repo: local
     hooks:
       - id: mypy
         name: mypy
-        entry: poetry run mypy
-        language: system
+        entry: infra/lint.sh mypy
+        language: script
         types: [python]
   - repo: local
     hooks:
       - id: bandit
         name: bandit
-        entry: poetry run bandit
-        args: [-lll, --recursive]
-        language: system
+        entry: infra/lint.sh bandit
+        language: script
+        types: [python]
   - repo: local
     hooks:
       - id: detect-secrets
         name: detect-secrets
-        entry: poetry run detect-secrets-hook
-        args: ['--baseline', '.secrets.baseline']
-        exclude: "poetry.lock"
-        language: system
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v1.9.3
-    hooks:
-      - id: seed-isort-config
+        entry: infra/lint.sh detect-secrets
+        language: script
   - repo: local
     hooks:
       - id: isort
         name: isort
-        entry: poetry run isort
-        args: ["--recursive", "--settings-path", "./pyproject.toml", "."]
-        language: system
+        entry: infra/lint.sh isort
+        language: script
         types: [python]
-        exclude: |
-          (?x)^(
-              .*/gunicorn_conf.py|
-              .*/test_gunicorn_conf.py
-          )$
   - repo: local
     hooks:
       - id: black
         name: black
-        entry: poetry run black
+        entry: infra/lint.sh black
+        language: script
         types: [python]
-        language: system
   - repo: local
     hooks:
       - id: yamllint
         name: yamllint
-        entry: poetry run yamllint
-        args: ["-c", ".yamllint", 'config/']
-        types: [file, yaml]
-        language: system
+        entry: infra/lint.sh yamllint
+        language: script
+        types: [yaml]

.secrets.baseline

@@ -1,10 +1,10 @@
 {
   "custom_plugin_paths": [],
   "exclude": {
-    "files": "^.secrets.baseline$",
+    "files": "poetry.lock",
     "lines": null
   },
-  "generated_at": "2022-03-18T00:00:55Z",
+  "generated_at": "2022-07-10T01:39:12Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -59,31 +59,11 @@
     }
   ],
   "results": {
-    ".github/workflows/deploy-build.yaml": [
+    "README.md": [
       {
-        "hashed_secret": "24b04b46d769e6f6b8d10c943738ac5c968dea1c",
+        "hashed_secret": "04e78d6e804f2b59e6cb282cb9ed2c7bfd8a9737",
         "is_verified": false,
-        "line_number": 94,
-        "type": "Secret Keyword"
-      },
-      {
-        "hashed_secret": "1481d0a0ceb16ea4672fed76a0710306eb9f3a33",
-        "is_verified": false,
-        "line_number": 96,
-        "type": "Secret Keyword"
-      },
-      {
-        "hashed_secret": "30a1dfa6d09c177dcd147a5f31a5627becd96b23",
-        "is_verified": false,
-        "line_number": 96,
-        "type": "Secret Keyword"
-      }
-    ],
-    "poetry.lock": [
-      {
-        "hashed_secret": "d97b148b47121abb687af133c28a3932202426b0",
-        "is_verified": false,
-        "line_number": 899,
+        "line_number": 152,
         "type": "Hex High Entropy String"
       }
     ]

infra/Dockerfile renamed to Dockerfile

@@ -31,16 +31,13 @@ RUN mkdir -p $POETRY_HOME && \
 
 RUN apt-get update && \
     apt-get install --assume-yes apt-utils && \
-    echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \
-    apt-get install --no-install-recommends -y \
-      libpq5
+    echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
 
 # builder-base is used to build dependencies
 FROM python-base as builder-base
 RUN apt-get install --no-install-recommends -y \
       curl \
-      build-essential \
-      libpq-dev
+      build-essential
 
 # Install Poetry - respects $POETRY_VERSION & $POETRY_HOME
 USER app
@@ -57,6 +54,9 @@ RUN poetry install --no-dev --no-root
 # 'development' stage installs all dev deps and can be used to develop code.
 # For example using docker-compose to mount local volume under /app
 FROM python-base as development
+# to run detect-secrets
+RUN apt-get install --no-install-recommends -y git
+
 ENV FASTAPI_ENV=development
 
 # Copying poetry and venv into image
@@ -80,24 +80,6 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD uvicorn src.app.api:app --reload --host=0.0.0.0 --port=$PORT
 
 
-# 'lint' stage runs similar checks to pre-commit
-# running in check mode means build will fail if any linting errors occur
-FROM development AS lint
-RUN bandit -lll --recursive src --exclude "src/poetry.lock,src/.venv,src/.mypy,src/build"
-RUN mypy src
-RUN black --config ./pyproject.toml --check src tests
-RUN isort --recursive --settings-path ./pyproject.toml --check-only src
-RUN pylint src tests
-RUN yamllint -d "{extends: default, rules: {key-duplicates: enable, key-ordering: enable}}" ./config
-CMD ./infra/detect_secrets_helper.sh
-
-
-# 'test' stage runs our unit tests with pytest and
-# coverage.
-FROM development AS test
-CMD ./infra/test.sh
-
-
 # 'production' stage uses the clean 'python-base' stage and copyies
 # in only our runtime deps that were installed in the 'builder-base'
 FROM python-base as production

Makefile

@@ -12,6 +12,7 @@ help:
 	@echo ""
 	@echo "  build   - build docker containers"
 	@echo "  lint    - lint check for code"
+	@echo "  format  - run formatters (black, isort), fix in place"
 	@echo "  start   - run the API service"
 	@echo ""
 	@echo "  test        - run test suite"
@@ -25,32 +26,30 @@ help:
 
 .PHONY: build
 build:
-	docker-compose -f ./docker-compose.yaml -f ./tests/infra/docker-compose.test.yaml build \
+	docker-compose build \
 		--build-arg userid=${_UID} --build-arg groupid=${_GID}
 
+.PHONY: format
+format:
+	infra/lint.sh black --fix
+	infra/lint.sh isort --fix
+
 .PHONY: lint
 lint:
-	docker-compose -f ./docker-compose.yaml -f ./tests/infra/docker-compose.lint.yaml build \
-		--build-arg userid=${_UID} --build-arg groupid=${_GID} lint
-
+	docker-compose run --rm web infra/lint.sh
 
 .PHONY: shell
 shell:
-	docker-compose -f ./docker-compose.yaml run web
+	docker-compose run web /bin/sh
 
 .PHONY: start
 start:
 	docker-compose up
 
 .PHONY: test
 test:
-	docker-compose -f ./docker-compose.yaml -f ./tests/infra/docker-compose.test.yaml run tests
-ifneq (1, ${MK_KEEP_DOCKER_UP})
-	# Due to https://github.com/docker/compose/issues/2791 we have to explicitly
-	# rm all running containers
-	docker-compose down
-endif
+	docker-compose run --rm web infra/test.sh
 
 .PHONY: test-shell
 test-shell:
-	docker-compose -f ./docker-compose.yaml -f ./tests/infra/docker-compose.test.yaml run web
+	docker-compose run web

docker-compose.yaml

@@ -3,12 +3,9 @@ services:
   web:
     build:
       context: .
-      dockerfile: ./infra/Dockerfile
       target: development
     volumes:
-      - type: bind
-        source: .
-        target: /app
+      - .:/app
     ports:
       - ${PORT:-8000}:${PORT:-8000}
     # Let the init system handle signals for us.