Merge pull request #1 from uni-intelligence/hourly_partitions

Hourly partitions

Author: slawek-asi

Dockerfile

@@ -0,0 +1,16 @@
+FROM fkrull/multi-python as build
+ENV PYTHONUNBUFFERED 1
+ARG PIP_INDEX_URL
+ENV PIP_INDEX_URL ${PIP_INDEX_URL}
+RUN pip3 --no-cache install --upgrade pip
+COPY setup.py .
+COPY psqlextra/_version.py psqlextra/_version.py
+COPY README.md .
+RUN pip3 install .[test] .[analysis] --no-cache-dir --no-cache --prefix /python-packages --no-warn-script-location
+
+FROM fkrull/multi-python
+ENV PROJECT_DIR /project
+WORKDIR $PROJECT_DIR
+ENV PYTHONUNBUFFERED 1
+COPY --from=build /python-packages /usr/local
+COPY . .

Makefile

@@ -0,0 +1,2 @@
+test:
+	docker compose run --rm tests

docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3.7"
+
+services:
+  postgres:
+    image: postgis/postgis:14-3.2
+    ports:
+    - "5435:5432"
+    volumes:
+    - pgdata:/var/lib/postgresql/data/:delegated
+    environment:
+    - POSTGRES_HOST_AUTH_METHOD=trust
+    - POSTGRES_DB=psqlextra
+    healthcheck:
+      test: "pg_isready -h localhost -p 5432 -q -U postgres"
+      interval: 3s
+      timeout: 5s
+      retries: 3
+
+
+  tests:
+    build:
+      context: ./
+    depends_on:
+      - postgres
+    command:
+      "tox"
+
+volumes:
+  pgdata:

psqlextra/partitioning/shorthands.py

@@ -16,6 +16,7 @@ def partition_by_current_time(
     months: Optional[int] = None,
     weeks: Optional[int] = None,
     days: Optional[int] = None,
+    hours: Optional[int] = None,
     max_age: Optional[relativedelta] = None,
 ) -> PostgresPartitioningConfig:
     """Short-hand for generating a partitioning config that partitions the
@@ -42,6 +43,9 @@ def partition_by_current_time(
         days:
             The amount of days each partition should contain.
 
+        hours:
+            The amount of hours each partition should contain.
+
         max_age:
             The maximum age of a partition (calculated from the
             start of the partition).
@@ -51,7 +55,7 @@ def partition_by_current_time(
     """
 
     size = PostgresTimePartitionSize(
-        years=years, months=months, weeks=weeks, days=days
+        years=years, months=months, weeks=weeks, days=days, hours=hours,
     )
 
     return PostgresPartitioningConfig(

psqlextra/partitioning/time_partition.py

@@ -19,6 +19,7 @@ class PostgresTimePartition(PostgresRangePartition):
         PostgresTimePartitionUnit.MONTHS: "%Y_%b",
         PostgresTimePartitionUnit.WEEKS: "%Y_week_%W",
         PostgresTimePartitionUnit.DAYS: "%Y_%b_%d",
+        PostgresTimePartitionUnit.HOURS: "%Y_%b_%d_%H",
     }
 
     def __init__(
@@ -27,8 +28,8 @@ def __init__(
         end_datetime = start_datetime + size.as_delta()
 
         super().__init__(
-            from_values=start_datetime.strftime("%Y-%m-%d"),
-            to_values=end_datetime.strftime("%Y-%m-%d"),
+            from_values=start_datetime.strftime("%Y-%m-%d %H:00"),
+            to_values=end_datetime.strftime("%Y-%m-%d %H:00"),
         )
 
         self.size = size

psqlextra/partitioning/time_partition_size.py

@@ -13,6 +13,7 @@ class PostgresTimePartitionUnit(enum.Enum):
     MONTHS = "months"
     WEEKS = "weeks"
     DAYS = "days"
+    HOURS = "hours"
 
 
 class PostgresTimePartitionSize:
@@ -27,8 +28,10 @@ def __init__(
         months: Optional[int] = None,
         weeks: Optional[int] = None,
         days: Optional[int] = None,
+        hours: Optional[int] = None,
+
     ) -> None:
-        sizes = [years, months, weeks, days]
+        sizes = [years, months, weeks, days, hours]
 
         if not any(sizes):
             raise PostgresPartitioningError("Partition cannot be 0 in size.")
@@ -50,6 +53,9 @@ def __init__(
         elif days:
             self.unit = PostgresTimePartitionUnit.DAYS
             self.value = days
+        elif hours:
+            self.unit = PostgresTimePartitionUnit.HOURS
+            self.value = hours
         else:
             raise PostgresPartitioningError(
                 "Unsupported time partitioning unit"
@@ -68,25 +74,32 @@ def as_delta(self) -> relativedelta:
         if self.unit == PostgresTimePartitionUnit.DAYS:
             return relativedelta(days=self.value)
 
+        if self.unit == PostgresTimePartitionUnit.HOURS:
+            return relativedelta(hours=self.value)
+
         raise PostgresPartitioningError(
             "Unsupported time partitioning unit: %s" % self.unit
         )
 
     def start(self, dt: datetime) -> datetime:
         if self.unit == PostgresTimePartitionUnit.YEARS:
-            return self._ensure_datetime(dt.replace(month=1, day=1))
+            return self._ensure_datetime(dt.replace(month=1, day=1, hour=0))
 
         if self.unit == PostgresTimePartitionUnit.MONTHS:
-            return self._ensure_datetime(dt.replace(day=1))
+            return self._ensure_datetime(dt.replace(day=1, hour=0))
 
         if self.unit == PostgresTimePartitionUnit.WEEKS:
-            return self._ensure_datetime(dt - relativedelta(days=dt.weekday()))
+            week_dt = dt - relativedelta(days=dt.weekday())
+            return self._ensure_datetime(week_dt.replace(hour=0))
+
+        if self.unit == PostgresTimePartitionUnit.DAYS:
+            return self._ensure_datetime(dt.replace(hour=0))
 
         return self._ensure_datetime(dt)
 
     @staticmethod
     def _ensure_datetime(dt: Union[date, datetime]) -> datetime:
-        return datetime(year=dt.year, month=dt.month, day=dt.day)
+        return datetime(year=dt.year, month=dt.month, day=dt.day, hour=dt.hour)
 
     def __repr__(self) -> str:
         return "PostgresTimePartitionSize<%s, %s>" % (self.unit, self.value)

settings.py

@@ -8,7 +8,7 @@
 TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 
 DATABASES = {
-    'default': dj_database_url.config(default='postgres:///psqlextra'),
+    'default': dj_database_url.config(default='postgresql://postgres/?user=postgres'),
 }
 
 DATABASES['default']['ENGINE'] = 'psqlextra.backend'

tests/test_partitioning_time.py

@@ -215,6 +215,56 @@ def test_partitioning_time_daily_apply():
     assert table.partitions[6].name == "2019_jun_04"
 
 
+@pytest.mark.postgres_version(lt=110000)
+def test_partitioning_time_hourly_apply():
+    """Tests whether automatically creating new partitions ahead hourly works as
+    expected."""
+
+    model = define_fake_partitioned_model(
+        {"timestamp": models.DateTimeField()}, {"key": ["timestamp"]}
+    )
+
+    schema_editor = connection.schema_editor()
+    schema_editor.create_partitioned_model(model)
+
+    # create partitions for the next 4 hours (including the current)
+    with freezegun.freeze_time("2019-1-23 22:00"):
+        manager = PostgresPartitioningManager(
+            [partition_by_current_time(model, hours=1, count=4)]
+        )
+        manager.plan().apply()
+
+    table = _get_partitioned_table(model)
+    assert len(table.partitions) == 4
+    assert table.partitions[0].name == "2019_jan_23_22"
+    assert table.partitions[1].name == "2019_jan_23_23"
+    assert table.partitions[2].name == "2019_jan_24_00"
+    assert table.partitions[3].name == "2019_jan_24_01"
+
+    # re-running it with 5, should just create one additional partition
+    with freezegun.freeze_time("2019-1-23 22:59"):
+        manager = PostgresPartitioningManager(
+            [partition_by_current_time(model, hours=1, count=5)]
+        )
+        manager.plan().apply()
+
+    table = _get_partitioned_table(model)
+    assert len(table.partitions) == 5
+    assert table.partitions[4].name == "2019_jan_24_02"
+
+    # it's june now, we want to partition two hours ahead
+    with freezegun.freeze_time("2019-06-03"):
+        manager = PostgresPartitioningManager(
+            [partition_by_current_time(model, hours=1, count=2)]
+        )
+        manager.plan().apply()
+
+    table = _get_partitioned_table(model)
+    assert len(table.partitions) == 7
+    assert table.partitions[5].name == "2019_jun_03_00"
+    assert table.partitions[6].name == "2019_jun_03_01"
+
+
 @pytest.mark.postgres_version(lt=110000)
 def test_partitioning_time_monthly_apply_insert():
     """Tests whether automatically created monthly partitions line up
@@ -333,6 +383,45 @@ def test_partitioning_time_daily_apply_insert():
     model.objects.create(timestamp=datetime.date(2019, 1, 10))
 
 
+@pytest.mark.postgres_version(lt=110000)
+def test_partitioning_time_hourly_apply_insert():
+    """Tests whether automatically created hourly partitions line up
+    perfectly."""
+
+    model = define_fake_partitioned_model(
+        {"timestamp": models.DateTimeField()}, {"key": ["timestamp"]}
+    )
+
+    schema_editor = connection.schema_editor()
+    schema_editor.create_partitioned_model(model)
+
+    with freezegun.freeze_time("2019-1-07 13:59"):
+        manager = PostgresPartitioningManager(
+            [partition_by_current_time(model, hours=2, count=2)]
+        )
+        manager.plan().apply()
+
+    table = _get_partitioned_table(model)
+    assert len(table.partitions) == 2
+
+    model.objects.create(timestamp=datetime.datetime(2019, 1, 7, 13))
+    model.objects.create(timestamp=datetime.datetime(2019, 1, 7, 16, 59))
+
+    with transaction.atomic():
+        with pytest.raises(IntegrityError):
+            model.objects.create(timestamp=datetime.datetime(2019, 1, 7, 15))
+            model.objects.create(timestamp=datetime.datetime(2019, 1, 8))
+
+    with freezegun.freeze_time("2019-1-07 13:00"):
+        manager = PostgresPartitioningManager(
+            [partition_by_current_time(model, hours=2, count=4)]
+        )
+        manager.plan().apply()
+
+    model.objects.create(timestamp=datetime.datetime(2019, 1, 7, 17))
+    model.objects.create(timestamp=datetime.datetime(2019, 1, 7, 20, 59))
+
+
 @pytest.mark.postgres_version(lt=110000)
 @pytest.mark.parametrize(
     "kwargs,partition_names",

tox.ini

@@ -1,6 +1,10 @@
 [tox]
 envlist = py36-dj{20,21,22,30,31,32}, py37-dj{20,21,22,30,31,32}, py38-dj{20,21,22,30,31,32,40}, py39-dj{21,22,30,31,32,40}, py310-dj{21,22,30,31,32,40}
 
+; evaluation, flight_tracks
+;envlist = py37-dj{21}, py39-dj{32}
+
+
 [testenv]
 deps =
     dj20: Django~=2.0.0
@@ -13,5 +17,5 @@ deps =
     .[test]
 setenv =
     DJANGO_SETTINGS_MODULE=settings
-passenv = DATABASE_URL
+passenv = postgresql://postgres:5432/?user=postgres
 commands = python setup.py test