Implement TAP 4 (multiple repository consensus on entrusted targets)

README.rst

@@ -25,6 +25,14 @@ A Framework for Securing Software Update Systems
    :target: https://bestpractices.coreinfrastructure.org/projects/1351
    :alt: CII
 
+.. image:: https://pyup.io/repos/github/theupdateframework/tuf/shield.svg
+     :target: https://pyup.io/repos/github/theupdateframework/tuf/
+     :alt: Updates
+
+.. image:: https://pyup.io/repos/github/theupdateframework/tuf/python-3-shield.svg
+     :target: https://pyup.io/repos/github/theupdateframework/tuf/
+     :alt: Python 3
+
 .. image:: /docs/images/banner_readme.JPG
 
 The Update Framework (TUF) helps developers to secure new or existing

pylintrc

@@ -50,7 +50,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=parameter-unpacking, unpacking-in-except, long-suffix, old-ne-operator, old-octal-literal, import-star-module-level, raw-checker-failed, bad-inline-option, locally-disabled, locally-enabled, file-ignored, suppressed-message, useless-suppression, deprecated-pragma, apply-builtin, basestring-builtin, buffer-builtin, cmp-builtin, coerce-builtin, execfile-builtin, file-builtin, long-builtin, raw_input-builtin, reduce-builtin, standarderror-builtin, unicode-builtin, xrange-builtin, coerce-method, delslice-method, getslice-method, setslice-method, no-absolute-import, old-division, dict-iter-method, dict-view-method, next-method-called, metaclass-assignment, indexing-exception, raising-string, reload-builtin, oct-method, hex-method, nonzero-method, cmp-method, input-builtin, round-builtin, intern-builtin, unichr-builtin, map-builtin-not-iterating, zip-builtin-not-iterating, range-builtin-not-iterating, filter-builtin-not-iterating, using-cmp-argument, eq-without-hash, div-method, idiv-method, rdiv-method, exception-message-attribute, invalid-str-codec, sys-max-int, deprecated-str-translate-call, global-statement, broad-except, C, R
+disable=parameter-unpacking, unpacking-in-except, long-suffix, old-ne-operator, old-octal-literal, import-star-module-level, raw-checker-failed, bad-inline-option, locally-disabled, locally-enabled, file-ignored, suppressed-message, useless-suppression, deprecated-pragma, apply-builtin, basestring-builtin, buffer-builtin, cmp-builtin, coerce-builtin, execfile-builtin, file-builtin, long-builtin, raw_input-builtin, reduce-builtin, standarderror-builtin, unicode-builtin, xrange-builtin, coerce-method, delslice-method, getslice-method, setslice-method, no-absolute-import, old-division, dict-iter-method, dict-view-method, next-method-called, metaclass-assignment, indexing-exception, raising-string, reload-builtin, oct-method, hex-method, nonzero-method, cmp-method, input-builtin, round-builtin, intern-builtin, unichr-builtin, map-builtin-not-iterating, zip-builtin-not-iterating, range-builtin-not-iterating, filter-builtin-not-iterating, using-cmp-argument, eq-without-hash, div-method, idiv-method, rdiv-method, exception-message-attribute, invalid-str-codec, sys-max-int, deprecated-str-translate-call, global-statement, broad-except, logging-not-lazy, C, R
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

setup.py

@@ -108,9 +108,10 @@
     'Topic :: Security',
     'Topic :: Software Development'
   ],
-   install_requires = ['iso8601>=0.1.12', 'six>=1.11.0', 'securesystemslib>=0.10.8'],
+  install_requires = ['iso8601>=0.1.12', 'six>=1.11.0', 'securesystemslib>=0.10.8'],
   packages = find_packages(exclude=['tests']),
   scripts = [
     'tuf/scripts/basic_client.py',
+    'tuf/scripts/simple_server.py'
   ]
 )

tests/repository_data/README.md

@@ -0,0 +1,48 @@
+# Unit and integration testing
+
+## Running the tests
+The unit and integration tests can be executed by invoking `tox` from any
+path under the project directory.
+
+```
+$ tox
+```
+
+Or by invoking `aggregate_tests.py` from the
+[tests](https://github.com/theupdateframework/tuf/tree/develop/tests)
+directory.
+
+```
+$ python aggregate_tests.py
+```
+
+Note: integration tests end in `_integration.py`.
+
+If you wish to run a particular unit test, navigate to the tests directory and
+run that specific unit test.  For example:
+
+```
+$ python test_updater.py
+```
+
+It it also possible to run the test cases of a unit test.  For instance:
+
+```
+$ python -m unittest test_updater.TestMultiRepoUpdater.test_get_one_valid_targetinfo
+```
+
+## Setup
+The unit and integration tests operate on static metadata available in the
+[repository_data
+directory](https://github.com/theupdateframework/tuf/tree/develop/tests/repository_data/).
+Before running the tests, static metadata is first copied to temporary
+directories and modified, as needed, by the tests.
+
+The test modules typically spawn HTTP(S) servers that serve metadata and target
+files for the the unit tests.  The [map
+file](https://github.com/theupdateframework/tuf/tree/develop/tests/repository_data)
+specifies the location of the test repositories and other properties.  For
+specific targets and metadata provided by the tests repositories, please
+inspect their [respective
+metadata](https://github.com/theupdateframework/tuf/tree/develop/tests/repository_data/repository).
+

tests/repository_data/client/map.json

@@ -0,0 +1,33 @@
+{
+ "mapping": [
+  {
+   "paths": [
+    "*1.txt"
+   ],
+   "repositories": [
+    "test_repository1",
+    "test_repository2"
+   ],
+   "terminating": false,
+   "threshold": 1
+  },
+  {
+   "paths": [
+    "*3.txt"
+   ],
+   "repositories": [
+    "test_repository2"
+   ],
+   "terminating": true,
+   "threshold": 1
+  }
+ ],
+ "repositories": {
+  "test_repository1": [
+   "http://localhost:30001"
+  ],
+  "test_repository2": [
+   "http://localhost:30002"
+  ]
+ }
+}

tests/repository_data/client/test_repository2/metadata/current/1.root.json

@@ -0,0 +1,87 @@
+{
+ "signatures": [
+  {
+   "keyid": "4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb",
+   "sig": "344e2399b982470a06bbaeb7cd52eb58dfa7fba65d179a737b3655ce79fe722480f9515163010b714debd46a3f9bd60cd04c3c3093c050a9bb6b6fdd3dd1e902b3871198bf13c7c4ace9813527ac0b6f2d7652c64d7993ef88328bc82bea69ffec13dfd545e3ff0d327613b4eea6ad2c50baffd4d7233104427d0c6066fbec5f0091a8004c7f40c0ee59abf4955ab8682d8401f09adb42b0e8897c96ac7eb37843287c60b631ee67187d01e5b3936d84a36881c8e9726dc30bc62442f3849d94c96c62eb0aa66c9d3869e403940f8f7df527b2d3aa109cbf37d1349a8043395c6d2ad3550a64db0eaa39d45d6e30ea7fd06457bf0f033afba6796fbb7160e094e67da21c2f8da2220d10cef5df691f1f919ff14fb00e2b4057d5c099ce8b2d5d2a9e233dc7eb1dc77b24f4f22b219e67e45b3592a1ac873e96456078d82a12c085186f6b50131f2bad32f6bdf3359b141fb898819c98866f55eff686ad5cda01d2661dff6c39b66e5d83dfb65bb276ea4164b1f86c3371dbe635b31c118067ee"
+  }
+ ],
+ "signed": {
+  "_type": "root",
+  "consistent_snapshot": false,
+  "expires": "2030-01-01T00:00:00Z",
+  "keys": {
+   "4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb": {
+    "keyid_hash_algorithms": [
+     "sha256",
+     "sha512"
+    ],
+    "keytype": "rsa",
+    "keyval": {
+     "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0GjPoVrjS9eCqzoQ8VRe\nPkC0cI6ktiEgqPfHESFzyxyjC490Cuy19nuxPcJuZfN64MC48oOkR+W2mq4pM51i\nxmdG5xjvNOBRkJ5wUCc8fDCltMUTBlqt9y5eLsf/4/EoBU+zC4SW1iPU++mCsity\nfQQ7U6LOn3EYCyrkH51hZ/dvKC4o9TPYMVxNecJ3CL1q02Q145JlyjBTuM3Xdqsa\nndTHoXSRPmmzgB/1dL/c4QjMnCowrKW06mFLq9RAYGIaJWfM/0CbrOJpVDkATmEc\nMdpGJYDfW/sRQvRdlHNPo24ZW7vkQUCqdRxvnTWkK5U81y7RtjLt1yskbWXBIbOV\nz94GXsgyzANyCT9qRjHXDDz2mkLq+9I2iKtEqaEePcWRu3H6RLahpM/TxFzw684Y\nR47weXdDecPNxWyiWiyMGStRFP4Cg9trcwAGnEm1w8R2ggmWphznCd5dXGhPNjfA\na82yNFY8ubnOUVJOf0nXGg3Edw9iY3xyjJb2+nrsk5f3AgMBAAE=\n-----END PUBLIC KEY-----"
+    },
+    "scheme": "rsassa-pss-sha256"
+   },
+   "59a4df8af818e9ed7abe0764c0b47b4240952aa0d179b5b78346c470ac30278d": {
+    "keyid_hash_algorithms": [
+     "sha256",
+     "sha512"
+    ],
+    "keytype": "ed25519",
+    "keyval": {
+     "public": "edcd0a32a07dce33f7c7873aaffbff36d20ea30787574ead335eefd337e4dacd"
+    },
+    "scheme": "ed25519"
+   },
+   "65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093": {
+    "keyid_hash_algorithms": [
+     "sha256",
+     "sha512"
+    ],
+    "keytype": "ed25519",
+    "keyval": {
+     "public": "89f28bd4ede5ec3786ab923fd154f39588d20881903e69c7b08fb504c6750815"
+    },
+    "scheme": "ed25519"
+   },
+   "8a1c4a3ac2d515dec982ba9910c5fd79b91ae57f625b9cff25d06bf0a61c1758": {
+    "keyid_hash_algorithms": [
+     "sha256",
+     "sha512"
+    ],
+    "keytype": "ed25519",
+    "keyval": {
+     "public": "82ccf6ac47298ff43bfa0cd639868894e305a99c723ff0515ae2e9856eb5bbf4"
+    },
+    "scheme": "ed25519"
+   }
+  },
+  "roles": {
+   "root": {
+    "keyids": [
+     "4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb"
+    ],
+    "threshold": 1
+   },
+   "snapshot": {
+    "keyids": [
+     "59a4df8af818e9ed7abe0764c0b47b4240952aa0d179b5b78346c470ac30278d"
+    ],
+    "threshold": 1
+   },
+   "targets": {
+    "keyids": [
+     "65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093"
+    ],
+    "threshold": 1
+   },
+   "timestamp": {
+    "keyids": [
+     "8a1c4a3ac2d515dec982ba9910c5fd79b91ae57f625b9cff25d06bf0a61c1758"
+    ],
+    "threshold": 1
+   }
+  },
+  "spec_version": "1.0",
+  "version": 1
+ }
+}

tests/repository_data/client/test_repository2/metadata/current/role1.json

@@ -0,0 +1,49 @@
+{
+ "signatures": [
+  {
+   "keyid": "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a",
+   "sig": "749d6373835e9e89a269168d9af22bf7692ee7059a1db5ff1162e07b495ba47ae223e9ece6c27b2981d5d8bc046788d3fad9c2ba83d4be9b6547ed1f909c6204"
+  }
+ ],
+ "signed": {
+  "_type": "targets",
+  "delegations": {
+   "keys": {
+    "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a": {
+     "keyid_hash_algorithms": [
+      "sha256",
+      "sha512"
+     ],
+     "keytype": "ed25519",
+     "keyval": {
+      "public": "fcf224e55fa226056adf113ef1eb3d55e308b75b321c8c8316999d8c4fd9e0d9"
+     },
+     "scheme": "ed25519"
+    }
+   },
+   "roles": [
+    {
+     "keyids": [
+      "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a"
+     ],
+     "name": "role2",
+     "paths": [],
+     "terminating": false,
+     "threshold": 1
+    }
+   ]
+  },
+  "expires": "2030-01-01T00:00:00Z",
+  "spec_version": "1.0",
+  "targets": {
+   "/file3.txt": {
+    "hashes": {
+     "sha256": "141f740f53781d1ca54b8a50af22cbf74e44c21a998fa2a8a05aaac2c002886b",
+     "sha512": "ef5beafa16041bcdd2937140afebd485296cd54f7348ecd5a4d035c09759608de467a7ac0eb58753d0242df873c305e8bffad2454aa48f44480f15efae1cacd0"
+    },
+    "length": 28
+   }
+  },
+  "version": 1
+ }
+}

tests/repository_data/client/test_repository2/metadata/current/role2.json

@@ -0,0 +1,19 @@
+{
+ "signatures": [
+  {
+   "keyid": "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a",
+   "sig": "34ae1e3c897062419722c1747970a632e12060f3aef57314e6e6aa96c3a510a25ec5a8b12022058c768724607dd58106293089c87a7ee4b2ce5b3a8d44deeb06"
+  }
+ ],
+ "signed": {
+  "_type": "targets",
+  "delegations": {
+   "keys": {},
+   "roles": []
+  },
+  "expires": "2030-01-01T00:00:00Z",
+  "spec_version": "1.0",
+  "targets": {},
+  "version": 1
+ }
+}

tests/repository_data/client/test_repository2/metadata/current/root.json

@@ -0,0 +1,87 @@
+{
+ "signatures": [
+  {
+   "keyid": "4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb",
+   "sig": "344e2399b982470a06bbaeb7cd52eb58dfa7fba65d179a737b3655ce79fe722480f9515163010b714debd46a3f9bd60cd04c3c3093c050a9bb6b6fdd3dd1e902b3871198bf13c7c4ace9813527ac0b6f2d7652c64d7993ef88328bc82bea69ffec13dfd545e3ff0d327613b4eea6ad2c50baffd4d7233104427d0c6066fbec5f0091a8004c7f40c0ee59abf4955ab8682d8401f09adb42b0e8897c96ac7eb37843287c60b631ee67187d01e5b3936d84a36881c8e9726dc30bc62442f3849d94c96c62eb0aa66c9d3869e403940f8f7df527b2d3aa109cbf37d1349a8043395c6d2ad3550a64db0eaa39d45d6e30ea7fd06457bf0f033afba6796fbb7160e094e67da21c2f8da2220d10cef5df691f1f919ff14fb00e2b4057d5c099ce8b2d5d2a9e233dc7eb1dc77b24f4f22b219e67e45b3592a1ac873e96456078d82a12c085186f6b50131f2bad32f6bdf3359b141fb898819c98866f55eff686ad5cda01d2661dff6c39b66e5d83dfb65bb276ea4164b1f86c3371dbe635b31c118067ee"
+  }
+ ],
+ "signed": {
+  "_type": "root",
+  "consistent_snapshot": false,
+  "expires": "2030-01-01T00:00:00Z",
+  "keys": {
+   "4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb": {
+    "keyid_hash_algorithms": [
+     "sha256",
+     "sha512"
+    ],
+    "keytype": "rsa",
+    "keyval": {
+     "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0GjPoVrjS9eCqzoQ8VRe\nPkC0cI6ktiEgqPfHESFzyxyjC490Cuy19nuxPcJuZfN64MC48oOkR+W2mq4pM51i\nxmdG5xjvNOBRkJ5wUCc8fDCltMUTBlqt9y5eLsf/4/EoBU+zC4SW1iPU++mCsity\nfQQ7U6LOn3EYCyrkH51hZ/dvKC4o9TPYMVxNecJ3CL1q02Q145JlyjBTuM3Xdqsa\nndTHoXSRPmmzgB/1dL/c4QjMnCowrKW06mFLq9RAYGIaJWfM/0CbrOJpVDkATmEc\nMdpGJYDfW/sRQvRdlHNPo24ZW7vkQUCqdRxvnTWkK5U81y7RtjLt1yskbWXBIbOV\nz94GXsgyzANyCT9qRjHXDDz2mkLq+9I2iKtEqaEePcWRu3H6RLahpM/TxFzw684Y\nR47weXdDecPNxWyiWiyMGStRFP4Cg9trcwAGnEm1w8R2ggmWphznCd5dXGhPNjfA\na82yNFY8ubnOUVJOf0nXGg3Edw9iY3xyjJb2+nrsk5f3AgMBAAE=\n-----END PUBLIC KEY-----"
+    },
+    "scheme": "rsassa-pss-sha256"
+   },
+   "59a4df8af818e9ed7abe0764c0b47b4240952aa0d179b5b78346c470ac30278d": {
+    "keyid_hash_algorithms": [
+     "sha256",
+     "sha512"
+    ],
+    "keytype": "ed25519",
+    "keyval": {
+     "public": "edcd0a32a07dce33f7c7873aaffbff36d20ea30787574ead335eefd337e4dacd"
+    },
+    "scheme": "ed25519"
+   },
+   "65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093": {
+    "keyid_hash_algorithms": [
+     "sha256",
+     "sha512"
+    ],
+    "keytype": "ed25519",
+    "keyval": {
+     "public": "89f28bd4ede5ec3786ab923fd154f39588d20881903e69c7b08fb504c6750815"
+    },
+    "scheme": "ed25519"
+   },
+   "8a1c4a3ac2d515dec982ba9910c5fd79b91ae57f625b9cff25d06bf0a61c1758": {
+    "keyid_hash_algorithms": [
+     "sha256",
+     "sha512"
+    ],
+    "keytype": "ed25519",
+    "keyval": {
+     "public": "82ccf6ac47298ff43bfa0cd639868894e305a99c723ff0515ae2e9856eb5bbf4"
+    },
+    "scheme": "ed25519"
+   }
+  },
+  "roles": {
+   "root": {
+    "keyids": [
+     "4e777de0d275f9d28588dd9a1606cc748e548f9e22b6795b7cb3f63f98035fcb"
+    ],
+    "threshold": 1
+   },
+   "snapshot": {
+    "keyids": [
+     "59a4df8af818e9ed7abe0764c0b47b4240952aa0d179b5b78346c470ac30278d"
+    ],
+    "threshold": 1
+   },
+   "targets": {
+    "keyids": [
+     "65171251a9aff5a8b3143a813481cb07f6e0de4eb197c767837fe4491b739093"
+    ],
+    "threshold": 1
+   },
+   "timestamp": {
+    "keyids": [
+     "8a1c4a3ac2d515dec982ba9910c5fd79b91ae57f625b9cff25d06bf0a61c1758"
+    ],
+    "threshold": 1
+   }
+  },
+  "spec_version": "1.0",
+  "version": 1
+ }
+}

tests/repository_data/client/test_repository2/metadata/current/snapshot.json

@@ -0,0 +1,28 @@
+{
+ "signatures": [
+  {
+   "keyid": "59a4df8af818e9ed7abe0764c0b47b4240952aa0d179b5b78346c470ac30278d",
+   "sig": "d78e9013bab1da2a8425caa48143cd79a21632dce021ad7e1b883d83775035df333a8b26c9c952d832edaf9dc7be2ef612bdb21326fcc9849346d7e3a162050e"
+  }
+ ],
+ "signed": {
+  "_type": "snapshot",
+  "expires": "2030-01-01T00:00:00Z",
+  "meta": {
+   "role1.json": {
+    "version": 1
+   },
+   "role2.json": {
+    "version": 1
+   },
+   "root.json": {
+    "version": 1
+   },
+   "targets.json": {
+    "version": 1
+   }
+  },
+  "spec_version": "1.0",
+  "version": 1
+ }
+}