PEP458: Update TUF repository metadata on project index change

Makefile

@@ -150,7 +150,7 @@ initdb: .state/docker-build-base .state/db-populated
 inittuf: .state/db-migrated
 	docker compose up -d rstuf-api
 	docker compose up -d rstuf-worker
-	docker compose run --rm web python -m warehouse tuf bootstrap dev/rstuf/bootstrap.json --api-server http://rstuf-api
+	docker compose run --rm web rstuf admin --api-server http://rstuf-api send bootstrap dev/rstuf/bootstrap.json
 
 runmigrations: .state/docker-build-base
 	docker compose run --rm web python -m warehouse db upgrade head

dev/environment

@@ -64,6 +64,8 @@ TWOFACTORMANDATE_AVAILABLE=true
 TWOFACTORMANDATE_ENABLED=true
 OIDC_AUDIENCE=pypi
 
+RSTUF_API_URL="http://rstuf-api"
+
 # Default to the reCAPTCHA testing keys from https://developers.google.com/recaptcha/docs/faq
 RECAPTCHA_SITE_KEY=6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI
 RECAPTCHA_SECRET_KEY=6LeIxAcTAAAAAGG-vFI1TnRWxMZNFuojJ4WifJWe

docker-compose.yml

@@ -173,7 +173,7 @@ services:
       SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}"
 
   rstuf-api:
-    image: ghcr.io/repository-service-tuf/repository-service-tuf-api:v0.12.0b1
+    image: ghcr.io/repository-service-tuf/repository-service-tuf-api:v1.0.0b1
     ports:
       - 8001:80
     stop_signal: SIGKILL
@@ -187,7 +187,7 @@ services:
         condition: service_started
 
   rstuf-worker:
-    image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:v0.14.0b1
+    image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:v1.0.0b1
     volumes:
       - rstuf-metadata:/var/opt/repository-service-tuf/storage
       - ./dev/rstuf/keys/online:/keyvault
@@ -200,7 +200,7 @@ services:
       - RSTUF_REDIS_SERVER=redis://redis
       - RSTUF_REDIS_SERVER_DB_RESULT=1
       - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2
-      - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf
+      - RSTUF_DB_SERVER=postgresql://postgres@db:5432/rstuf
     depends_on:
       db:
         condition: service_healthy

docs/dev/development/getting-started.rst

@@ -256,7 +256,7 @@ You should see the following line at the bottom of the output:
 
 .. code-block:: console
 
-    Bootstrap completed using `dev/rstuf/bootstrap.json`. 🔐 🎉
+    Bootstrap completed. 🔐 🎉
 
 
 This command sends a static *bootstrap payload* to the RSTUF API. The payload
@@ -265,19 +265,16 @@ includes the TUF trust root for development and other configuration.
 By calling this API, RSTUF creates the TUF metadata repository, installs the
 TUF trust root for development, and creates the initial set of TUF metadata.
 
-.. note::
+Once this process is finished, TUF metadata will update automatically whenever
+distribution files are uploaded, yanked or removed. To disable this behavior,
+you can unset the ``RSTUF_API_URL`` environment variable. TUF metadata
+is hosted at: http://localhost:9001/tuf-metadata/
 
-    The RSTUF API is exposed only for development purposes and will not be
-    available in production. Currently, no upload hooks or automatic metadata
-    update tasks are configured to interact with RSTUF.
+.. note::
 
-    Take a look at the `RSTUF API documentation
-    <https://repository-service-tuf.readthedocs.io/en/stable/guide/general/usage.html#adding-artifacts>`_
-    to see how you can simulate artifact upload or removal, and how they affect
-    the TUF metadata repository:
+    RSTUF and automated TUF metadata updates are currently only available in the
+    Warehouse development environment.
 
-    * RSTUF API: http://localhost:8001
-    * TUF Metadata Repository: http://localhost:9001/tuf-metadata/
 
 
 Resetting the development database

requirements/dev.txt

@@ -4,3 +4,4 @@ pip-tools>=1.0
 pyramid_debugtoolbar>=2.5
 pip-api
 watchdog
+repository-service-tuf

tests/conftest.py

@@ -63,6 +63,8 @@
 from warehouse.packaging.interfaces import IProjectService
 from warehouse.subscriptions import services as subscription_services
 from warehouse.subscriptions.interfaces import IBillingService, ISubscriptionService
+from warehouse.tuf import services as tuf_services
+from warehouse.tuf.interfaces import ITUFService
 
 from .common.db import Session
 from .common.db.accounts import EmailFactory, UserFactory
@@ -179,6 +181,7 @@ def pyramid_services(
     integrity_service,
     macaroon_service,
     helpdesk_service,
+    tuf_service,
 ):
     services = _Services()
 
@@ -201,6 +204,7 @@ def pyramid_services(
     services.register_service(integrity_service, IIntegrityService, None)
     services.register_service(macaroon_service, IMacaroonService, None, name="")
     services.register_service(helpdesk_service, IHelpDeskService, None)
+    services.register_service(tuf_service, ITUFService, None)
 
     return services
 
@@ -611,6 +615,11 @@ def helpdesk_service():
     return helpdesk_services.ConsoleHelpDeskService()
 
 
+@pytest.fixture
+def tuf_service(db_session):
+    return tuf_services.RSTUFService(db_session)
+
+
 class QueryRecorder:
     def __init__(self):
         self.queries = []

tests/unit/packaging/test_utils.py

@@ -58,17 +58,14 @@ def test_render_simple_detail(db_request, monkeypatch, jinja):
     context = _valid_simple_detail_context(context)
     expected_content = template.render(**context, request=db_request).encode("utf-8")
 
-    content_hash, path = render_simple_detail(project, db_request)
+    content_hash, size = render_simple_detail(project, db_request)
 
     assert fakeblake2b.calls == [pretend.call(digest_size=32)]
     assert fake_hasher.update.calls == [pretend.call(expected_content)]
     assert fake_hasher.hexdigest.calls == [pretend.call()]
 
     assert content_hash == "deadbeefdeadbeefdeadbeefdeadbeef"
-    assert path == (
-        f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef"
-        + f".{project.normalized_name}.html"
-    )
+    assert size == len(expected_content)
 
 
 def test_render_simple_detail_with_store(db_request, monkeypatch, jinja):
@@ -92,9 +89,10 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja):
     fakeblake2b = pretend.call_recorder(lambda *a, **kw: fake_hasher)
     monkeypatch.setattr(hashlib, "blake2b", fakeblake2b)
 
+    expected_size = 225
     fake_named_temporary_file = pretend.stub(
         name="/tmp/wutang",
-        write=pretend.call_recorder(lambda data: None),
+        write=pretend.call_recorder(lambda data: expected_size),
         flush=pretend.call_recorder(lambda: None),
     )
 
@@ -115,7 +113,7 @@ def __exit__(self, type, value, traceback):
     context = _valid_simple_detail_context(context)
     expected_content = template.render(**context, request=db_request).encode("utf-8")
 
-    content_hash, path = render_simple_detail(project, db_request, store=True)
+    content_hash, size = render_simple_detail(project, db_request, store=True)
 
     assert fake_named_temporary_file.write.calls == [pretend.call(expected_content)]
     assert fake_named_temporary_file.flush.calls == [pretend.call()]
@@ -149,7 +147,4 @@ def __exit__(self, type, value, traceback):
     ]
 
     assert content_hash == "deadbeefdeadbeefdeadbeefdeadbeef"
-    assert path == (
-        f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef"
-        + f".{project.normalized_name}.html"
-    )
+    assert size == expected_size

tests/unit/test_config.py

@@ -455,6 +455,7 @@ def __init__(self):
             pretend.call(".helpdesk"),
             pretend.call(".http"),
             pretend.call(".utils.row_counter"),
+            pretend.call(".tuf"),
         ]
         + [pretend.call(x) for x in [configurator_settings.get("warehouse.theme")] if x]
         + [pretend.call(".sanity")]

tests/unit/tuf/test_init.py

@@ -0,0 +1,69 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pretend
+
+from tests.common.db.packaging import ProjectFactory, UserFactory
+from warehouse import tuf
+from warehouse.tuf.interfaces import ITUFService
+from warehouse.tuf.services import rstuf_factory
+
+
+def test_update_metadata_for_project(db_request, monkeypatch):
+    delay = pretend.call_recorder(lambda *a: None)
+    config = pretend.stub(
+        registry=pretend.stub(settings={"rstuf.api_url": "http://rstuf"}),
+        task=pretend.call_recorder(lambda *a: pretend.stub(delay=delay)),
+    )
+
+    project0 = ProjectFactory.create()
+    user0 = UserFactory.create()
+
+    session = pretend.stub(info={}, new={project0, user0}, dirty=set())
+
+    tuf.update_metadata_for_project(config, session, pretend.stub())
+
+    # calls only for Projects
+    assert config.task.calls == [pretend.call(tuf.update_metadata)]
+    assert delay.calls == [pretend.call(project0.id)]
+
+
+def test_update_metadata_for_project_rstuf_disabled(db_request, monkeypatch):
+    delay = pretend.call_recorder(lambda *a: None)
+    config = pretend.stub(
+        registry=pretend.stub(settings={}),
+        task=pretend.call_recorder(lambda *a: pretend.stub(delay=delay)),
+    )
+
+    project0 = ProjectFactory.create()
+
+    session = pretend.stub(info={}, new={project0}, dirty=set())
+
+    tuf.update_metadata_for_project(config, session, pretend.stub())
+
+    assert config.task.calls == []
+    assert delay.calls == []
+
+
+def test_includeme():
+    config = pretend.stub(
+        register_service_factory=pretend.call_recorder(
+            lambda factory, iface, name=None: None
+        ),
+        maybe_dotted=pretend.call_recorder(lambda *a: "http://rstuf"),
+    )
+
+    tuf.includeme(config)
+
+    assert config.register_service_factory.calls == [
+        pretend.call(rstuf_factory, ITUFService),
+    ]