openbikesensor · gluap · Jun 22, 2023 · Mar 12, 2023 · Mar 12, 2023 · Mar 12, 2023
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,28 @@
 # Changelog
 
+## 0.8.0
+
+### Features
+
+* Bulk actions on users owned tracks (reprocess, download, make private, make public, delete) (#269, #38)
+* Easy sorting by device for "multi-device users" (e.g. group lending out OBSes)
+* Region display at higher zoom levels to easily find interesting areas (#112)
+* Export of road statistics on top of the already-existing event statistics (#341)
+
+### Improvements
+
+* Refactored database access to hopefully combat portal crashes (#337)
+* New infrastructure for map imports that makes import of larger maps possible on small VMs (#334)
+* Reference current postgres and postgis versions in docker-compose.yaml files (#286)
+* Configurable terms-and-conditions link (#320)
+* French translation by @cbiteau (#303)
+
+### Bug Fixes
+
+* Logout not working (#285)
+* Duplicate road usage hashes (#335, #253)
+* cannot import name .... (#338)
+
 ## 0.7.0
 
 ### Features

diff --git a/Dockerfile b/Dockerfile
@@ -1,44 +1,5 @@
 # This dockerfile is for the API + Frontend production image
 
-#############################################
-# Build osm2pgsql AS builder
-#############################################
-
-# This image should be the same as final one, because of the lib versions
-FROM python:3.9.7-bullseye as osm2pgsql-builder
-
-ARG DEBIAN_FRONTEND=noninteractive
-ENV TZ=Europe/Berlin
-ENV OSM2PGSQL_VERSION=1.5.1
-
-# Dependencies
-RUN apt-get update &&\
-    apt-get install -y \
-    make \
-    cmake \
-    g++ \
-    libboost-dev \
-    libboost-system-dev \
-    libboost-filesystem-dev \
-    libexpat1-dev \
-    zlib1g-dev \
-    libbz2-dev \
-    libpq-dev \
-    libproj-dev \
-    lua5.3 \
-    liblua5.3-dev \
-    git &&\
-    rm -rf /var/lib/apt/lists/*
-
-# Clone & Build
-RUN git clone  --branch $OSM2PGSQL_VERSION https://github.com/openstreetmap/osm2pgsql.git &&\
-    cd osm2pgsql/ &&\
-    mkdir build &&\
-    cd build &&\
-    cmake .. &&\
-    make -j4 &&\
-    make install
-
 #############################################
 # Build the frontend AS builder
 #############################################
@@ -60,7 +21,7 @@ RUN npm run build
 # Build the API and add the built frontend to it
 #############################################
 
-FROM python:3.9.7-bullseye
+FROM python:3.11.3-bullseye
 
 RUN apt-get update &&\
     apt-get install -y \
@@ -93,11 +54,7 @@ ADD api/obs /opt/obs/api/obs/
 ADD api/tools /opt/obs/api/tools/
 RUN pip install -e /opt/obs/api/
 
-ADD roads_import.lua /opt/obs/api/tools
-ADD osm2pgsql.sh /opt/obs/api/tools
-
 COPY --from=frontend-builder /opt/obs/frontend/build /opt/obs/frontend/build
-COPY --from=osm2pgsql-builder /usr/local/bin/osm2pgsql /usr/local/bin/osm2pgsql
 
 EXPOSE 3000
 

diff --git a/README.md b/README.md
@@ -36,10 +36,11 @@ git submodule update --init --recursive
 
 ## Production setup
 
-There is a guide for a deployment based on docker in the
-[deployment](deployment) folder. Lots of non-docker deployment strategy are
-possible, but they are not "officially" supported, so please do not expect the
-authors of the software to assist in troubleshooting. 
+There is a guide for a deployment based on docker at
+[docs/production-deployment.md](docs/production-deployment.md). Lots of
+non-docker deployment strategies are possible, but they are not "officially"
+supported, so please do not expect the authors of the software to assist in
+troubleshooting. 
 
 This is a rather complex application, and it is expected that you know the
 basics of deploying a modern web application securely onto a production server.
@@ -52,7 +53,8 @@ Please note that you will always need to install your own reverse proxy that
 terminates TLS for you and handles certificates. We do not support TLS directly
 in the application, instead, please use this prefered method. 
 
-Upgrading and migrating is descrube
+Upgrading and migrating is described in [UPGRADING.md](./UPGRADING.md) for each
+version.
 
 ### Migrating (Production)
 
@@ -75,18 +77,6 @@ docker-compose run --rm api alembic upgrade head
 docker-compose run --rm api tools/prepare_sql_tiles
 ```
 
-
-docker-compose run --rm api alembic upgrade head
-
-### Upgrading from v0.2 to v0.3
-
-After v0.2 we switched the underlying technology of the API and the database.
-We now have no more MongoDB, instead, everything has moved to the PostgreSQL
-installation. For development setups, it is advised to just reset the whole
-state (remove the `local` folder) and start fresh. For production upgrades,
-please follow the relevant section in [`UPGRADING.md`](./UPGRADING.md).
-
-
 ## Development setup
 
 We've moved the whole development setup into Docker to make it easy for
@@ -101,7 +91,6 @@ Then clone the repository as described above.
 
 ### Configure Keycloak
 
-
 Login will not be possible until you configure the keycloak realm correctly. Boot your keycloak instance:
 
 ```bash
@@ -164,7 +153,7 @@ You will need to re-run this command after updates, to migrate the database and
 (re-)create the functions in the SQL database that are used when generating
 vector tiles.
 
-You should also import OpenStreetMap data now, see below for instructions.
+You should also [import OpenStreetMap data](docs/osm-import.md) now.
 
 ### Boot the application
 
@@ -190,48 +179,6 @@ docker-compose run --rm api alembic upgrade head
 ```
 
 
-## Import OpenStreetMap data
-
-**Hint:** This step may be skipped if you are using [Lean mode](./docs/lean-mode.md).
-
-You need to import road information from OpenStreetMap for the portal to work.
-This information is stored in your PostgreSQL database and used when processing
-tracks (instead of querying the Overpass API), as well as for vector tile
-generation. The process applies to both development and production setups. For
-development, you should choose a small area for testing, such as your local
-county or city, to keep the amount of data small. For production use you have
-to import the whole region you are serving.
-
-* Install `osm2pgsql`. 
-* Download the area(s) you would like to import from [GeoFabrik](https://download.geofabrik.de). 
-* Import each file like this:
-
-    ```bash
-    osm2pgsql --create --hstore --style roads_import.lua -O flex \
-      -H localhost -d obs -U obs -W \
-      path/to/downloaded/myarea-latest.osm.pbf 
-    ```
-
-You might need to adjust the host, database and username (`-H`, `-d`, `-U`) to
-your setup, and also provide the correct password when queried. For the
-development setup the password is `obs`. For production, you might need to
-expose the containers port and/or create a TCP tunnel, for example with SSH,
-such that you can run the import from your local host and write to the remote
-database.
-
-The import process should take a few seconds to minutes, depending on the area
-size. A whole country might even take one or more hours. You should probably
-not try to import `planet.osm.pbf`. 
-
-You can run the process multiple times, with the same or different area files,
-to import or update the data. However, for this to work, the actual [command
-line arguments](https://osm2pgsql.org/doc/manual.html#running-osm2pgsql) are a
-bit different each time, including when first importing, and the disk space
-required is much higher. 
-
-Refer to the documentation of `osm2pgsql` for assistance. We are using "flex
-mode", the provided script `roads_import.lua` describes the transformations
-and extractions to perform on the original data.
 
 ## Troubleshooting
 

diff --git a/UPGRADING.md b/UPGRADING.md
@@ -1,9 +1,32 @@
 # Upgrading
-
 This document describes the general steps to upgrade between major changes.
 Simple migrations, e.g. for adding schema changes, are not documented
 explicitly. Their general usage is described in the [README](./README.md) (for
-development) and [deployment/README.md](deployment/README.md) (for production).
+development) and [docs/production-deployment.md](docs/production-deployment.md) (for production).
+
+
+## 0.8.0
+Upgrade to `0.7.x` first. See below for details. Then follow these steps:
+
+> **Warning** The update includes a reprocessing of tracks after import. Depending on the number of tracks this can take a few hours. The portal is reachable during that time but events disappear and incrementally reappear during reimport.
+
+> **Info** With this version the import process for OpenStreetMap data has changed: the [new process](docs/osm-import.md) is easier on resources and finally permits to import a full country on a low-end VM.
+
+- Do your [usual backup](docs/production-deployment.md)
+- get the release in your source folder (``git pull; git checkout 0.8.0`` and update submodules ``git submodule update --recursive``)
+- Rebuild images ``docker-compose build``
+- Stop your portal and worker services ``docker-compose stop worker portal``
+- run upgrade
+  ```bash
+  docker-compose run --rm portal tools/upgrade.py
+  ```
+  this automatically does the following
+  - Migration of database schema using alembic.
+  - Upgrade of SQL tile schema to new schema.
+  - Import the nuts-regions from the web into the database.
+  - Trigger a re-import of all tracks.
+- Start your portal and worker services. ``docker-compose up -d worker portal``
+
 
 ## 0.7.0
 
@@ -57,7 +80,7 @@ You can, but do not have to, reimport all tracks. This will generate a GPX file
 for each track and allow the users to download those. If a GPX file has not yet
 been created, the download will fail. To reimport all tracks, log in to your
 PostgreSQL database (instructions are in [README.md](./README.md) for
-development and [deployment/README.md](./deployment/README.md) for production)
+development and [docs/production-deployment.md](./docs/production-deployment.md) for production)
 and run:
 
 ```sql
@@ -77,7 +100,7 @@ Make sure your worker is running to process the queue.
   `POSTGRES_MAX_OVERFLOW`. Check the example config for sane default values.
 * Re-run `tools/prepare_sql_tiles.py` again (see README)
 * It has been made easier to import OSM data, check
-  [deployment/README.md](deployment/README.md) for the sections "Download
+  [docs/production-deployment.md](./docs/production-deployment.md) for the sections "Download
   OpenStreetMap maps" and "Import OpenStreetMap data". You can now download
   multiple .pbf files and then import them at once, using the docker image
   built with the `Dockerfile`. Alternatively, you can choose to enable [lean
@@ -132,5 +155,5 @@ Make sure your worker is running to process the queue.
   `export/users.json` into your realm, it will re-add all the users from the
   old installation. You should delete the file and `export/` folder afterwards.
 * Start `portal`.
-* Consider configuring a worker service. See [deployment/README.md](deployment/README.md).
+* Consider configuring a worker service. See [docs/production-deployment.md](./docs/production-deployment.md).
 
diff --git a/api/Dockerfile b/api/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9.7-bullseye
+FROM python:3.11.3-bullseye
 
 WORKDIR /opt/obs/api
 

diff --git a/api/config.dev.py b/api/config.dev.py
@@ -2,9 +2,8 @@
 PORT = 3000
 DEBUG = True
 VERBOSE = False
-AUTO_RESTART = True
+AUTO_RELOAD = True
 SECRET = "!!!!!!!!!!!!CHANGE ME!!!!!!!!!!!!"
-LEAN_MODE = False
 POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs"
 POSTGRES_POOL_SIZE = 20
 POSTGRES_MAX_OVERFLOW = 2 * POSTGRES_POOL_SIZE
@@ -30,5 +29,7 @@
     "http://localhost:8880/",  # for maputnik on 8880
     "http://localhost:8888/",  # for maputnik on 8888
 ]
+TILE_SEMAPHORE_SIZE = 4
+EXPORT_SEMAPHORE_SIZE = 4
 
 # vim: set ft=python :
diff --git a/api/config.py.example b/api/config.py.example
@@ -5,12 +5,7 @@ PORT = 3000
 # Extended log output, but slower
 DEBUG = False
 VERBOSE = DEBUG
-AUTO_RESTART = DEBUG
-
-# Turn on lean mode to simplify the setup. Lots of features will be
-# unavailable, but you will not need to manage OpenStreetMap data. Please make
-# sure to configure the OBS_FACE_CACHE_DIR correctly for lean mode.
-LEAN_MODE = False
+AUTO_RELOAD = DEBUG
 
 # Required to encrypt or sign sessions, cookies, tokens, etc.
 SECRET = "!!!<<<CHANGEME>>>!!!"
@@ -66,4 +61,13 @@ TILES_FILE = None
 # default. Python list, or whitespace separated string.
 ADDITIONAL_CORS_ORIGINS = None
 
+# How many asynchronous requests may be sent to the database to generate tile
+# information. Should be less than POSTGRES_POOL_SIZE to leave some connections
+# to the other features of the API ;)
+TILE_SEMAPHORE_SIZE = 4
+
+# How many asynchronous requests may generate exported data simultaneously.
+# Keep this small.
+EXPORT_SEMAPHORE_SIZE = 1
+
 # vim: set ft=python :
diff --git a/api/migrations/versions/35e7f1768f9b_create_table_road.py b/api/migrations/versions/35e7f1768f9b_create_table_road.py
@@ -22,13 +22,16 @@ def upgrade():
     op.create_table(
         "road",
         sa.Column(
-            "way_id", sa.BIGINT, autoincrement=True, primary_key=True, index=True
+            "way_id", sa.BIGINT, primary_key=True, index=True, autoincrement=False
         ),
         sa.Column("zone", dbtype("zone_type")),
-        sa.Column("name", sa.String),
-        sa.Column("geometry", dbtype("GEOMETRY"), index=True),
+        sa.Column("name", sa.Text),
+        sa.Column("geometry", dbtype("geometry(LINESTRING,3857)")),
         sa.Column("directionality", sa.Integer),
-        sa.Column("oenway", sa.Boolean),
+        sa.Column("oneway", sa.Boolean),
+    )
+    op.execute(
+        "CREATE INDEX road_geometry_idx ON road USING GIST (geometry) WITH (FILLFACTOR=100);"
     )
 
 

diff --git a/api/migrations/versions/587e69ecb466_transform_overtaking_event_geometry_to_.py b/api/migrations/versions/587e69ecb466_transform_overtaking_event_geometry_to_.py
@@ -0,0 +1,30 @@
+"""transform overtaking_event geometry to 3857
+
+Revision ID: 587e69ecb466
+Revises: f4b0f460254d
+Create Date: 2023-04-01 14:30:49.927505
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "587e69ecb466"
+down_revision = "f4b0f460254d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("UPDATE overtaking_event SET geometry = ST_Transform(geometry, 3857);")
+    op.execute(
+        "ALTER TABLE overtaking_event ALTER COLUMN geometry TYPE geometry(POINT, 3857);"
+    )
+
+
+def downgrade():
+    op.execute(
+        "ALTER TABLE overtaking_event ALTER COLUMN geometry TYPE geometry;"
+    )
+    op.execute("UPDATE overtaking_event SET geometry = ST_Transform(geometry, 4326);")