From 978400897a803350dd11fdc6154434dd77d8f6c1 Mon Sep 17 00:00:00 2001 From: Gabriel Filion Date: Mon, 20 Jan 2025 17:54:34 -0500 Subject: [PATCH 1/5] Hide packages if phasing is being applied (#220) Ubuntu uses a progress upgrade throughout hosts called phasing upgrades. If a package needs to be upgraded but is marked as being delayed because of phasing, it shouldn't be listed as the packages needing upgrades since it's already being taken care of. Signed-off-by: Gabriel Filion --- apt_info.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apt_info.py b/apt_info.py index 19455915..9c520904 100755 --- a/apt_info.py +++ b/apt_info.py @@ -55,7 +55,7 @@ def _convert_candidates_to_upgrade_infos(candidates): def _write_pending_upgrades(registry, cache): candidates = { - p.candidate for p in cache if p.is_upgradable + p.candidate for p in cache if p.is_upgradable and not p.phasing_applied } upgrade_list = _convert_candidates_to_upgrade_infos(candidates) @@ -69,7 +69,11 @@ def _write_pending_upgrades(registry, cache): def _write_held_upgrades(registry, cache): held_candidates = { p.candidate for p in cache - if p.is_upgradable and p._pkg.selected_state == apt_pkg.SELSTATE_HOLD + if ( + p.is_upgradable + and p._pkg.selected_state == apt_pkg.SELSTATE_HOLD + and not p.phasing_applied + ) } upgrade_list = _convert_candidates_to_upgrade_infos(held_candidates) From e25b066edbf3087dc8a8a265c5a9ba826909bb2c Mon Sep 17 00:00:00 2001 From: Gabriel Filion Date: Tue, 21 Jan 2025 17:01:35 -0500 Subject: [PATCH 2/5] Expose number of obsolete packages Packages are obsolete if they either: * don't have a candidate version * their candidate version does not have an origin. * their candidate version has a single origin that points to local storage Signed-off-by: Gabriel Filion --- apt_info.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/apt_info.py b/apt_info.py index 9c520904..f7d809a4 100755 --- a/apt_info.py +++ b/apt_info.py @@ -84,6 +84,20 @@ def _write_held_upgrades(registry, cache): g.labels(change.labels['origin'], change.labels['arch']).set(change.count) +def _write_obsolete_packages(registry, cache): + # This corresponds to the apt filter "?obsolete" + obsoletes = [p for p in cache if p.is_installed and ( + p.candidate is None or + not p.candidate.origins or + (len(p.candidate.origins) == 1 and + p.candidate.origins[0].origin in ['', "/var/lib/dpkg/status"]) + )] + + g = Gauge('apt_packages_obsolete_count', "Apt packages which are obsolete", + registry=registry) + g.set(len(obsoletes)) + + def _write_autoremove_pending(registry, cache): autoremovable_packages = {p for p in cache if p.is_auto_removable} g = Gauge('apt_autoremove_pending', "Apt packages pending autoremoval.", @@ -122,6 +136,7 @@ def _main(): registry = CollectorRegistry() _write_pending_upgrades(registry, cache) _write_held_upgrades(registry, cache) + _write_obsolete_packages(registry, cache) _write_autoremove_pending(registry, cache) _write_cache_timestamps(registry) _write_reboot_required(registry) From 39498fb2efb9022649d89cfb56228a2aac84f12e Mon Sep 17 00:00:00 2001 From: Gabriel Filion Date: Wed, 22 Jan 2025 15:53:56 -0500 Subject: [PATCH 3/5] Show total package installed per origin (#220) With this you can see a progression of how many packages are installed from each source used. Signed-off-by: Gabriel Filion --- apt_info.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/apt_info.py b/apt_info.py index f7d809a4..42222d83 100755 --- a/apt_info.py +++ b/apt_info.py @@ -35,8 +35,13 @@ def _convert_candidates_to_upgrade_infos(candidates): changes_dict = collections.defaultdict(lambda: collections.defaultdict(int)) for candidate in candidates: + # The 'now' archive only shows that packages are not installed. We tend + # to filter the candidates on those kinds of conditions before reaching + # here so here we don't want to include this information in order to + # reduce noise in the data. origins = sorted( - {f"{o.origin}:{o.codename}/{o.archive}" for o in candidate.origins} + {f"{o.origin}:{o.codename}/{o.archive}" for o in candidate.origins + if o.archive != 'now'} ) changes_dict[",".join(origins)][candidate.architecture] += 1 @@ -105,6 +110,17 @@ def _write_autoremove_pending(registry, cache): g.set(len(autoremovable_packages)) +def _write_installed_packages_per_origin(registry, cache): + installed_packages = {p.candidate for p in cache if p.is_installed} + per_origin = _convert_candidates_to_upgrade_infos(installed_packages) + + if per_origin: + g = Gauge('apt_packages_per_origin_count', "Number of packages installed per origin.", + ['origin', 'arch'], registry=registry) + for o in per_origin: + g.labels(o.labels['origin'], o.labels['arch']).set(o.count) + + def _write_cache_timestamps(registry): g = Gauge('apt_package_cache_timestamp_seconds', "Apt update last run time.", registry=registry) apt_pkg.init_config() @@ -138,6 +154,7 @@ def _main(): _write_held_upgrades(registry, cache) _write_obsolete_packages(registry, cache) _write_autoremove_pending(registry, cache) + _write_installed_packages_per_origin(registry, cache) _write_cache_timestamps(registry) _write_reboot_required(registry) print(generate_latest(registry).decode(), end='') From 423e4b763aab58caffeae541540dfa780ad05798 Mon Sep 17 00:00:00 2001 From: Gabriel Filion Date: Tue, 28 Jan 2025 12:06:01 -0500 Subject: [PATCH 4/5] Add debug logging This can help one in identifying what the cache filters identified in each category, which is useful both for debugging the code itself and for identifying what builtin command-line apt search filters may not be identifying in a similar manner. One example of this is the obsolete package filter, which can sometimes catch more packages than `apt list "?obsolete"` Signed-off-by: Gabriel Filion --- apt_info.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/apt_info.py b/apt_info.py index 42222d83..c633e436 100755 --- a/apt_info.py +++ b/apt_info.py @@ -25,7 +25,9 @@ import apt import apt_pkg import collections +import logging import os +import sys from prometheus_client import CollectorRegistry, Gauge, generate_latest _UpgradeInfo = collections.namedtuple("_UpgradeInfo", ["labels", "count"]) @@ -62,6 +64,12 @@ def _write_pending_upgrades(registry, cache): candidates = { p.candidate for p in cache if p.is_upgradable and not p.phasing_applied } + for candidate in candidates: + logging.debug( + "pending upgrade: %s / %s", + candidate.package, + candidate.architecture, + ) upgrade_list = _convert_candidates_to_upgrade_infos(candidates) if upgrade_list: @@ -80,6 +88,12 @@ def _write_held_upgrades(registry, cache): and not p.phasing_applied ) } + for candidate in held_candidates: + logging.debug( + "held upgrade: %s / %s", + candidate.package, + candidate.architecture, + ) upgrade_list = _convert_candidates_to_upgrade_infos(held_candidates) if upgrade_list: @@ -97,6 +111,15 @@ def _write_obsolete_packages(registry, cache): (len(p.candidate.origins) == 1 and p.candidate.origins[0].origin in ['', "/var/lib/dpkg/status"]) )] + for package in obsoletes: + if package.candidate is None: + logging.debug("obsolete package with no candidate: %s", package) + else: + logging.debug( + "obsolete package: %s / %s", + package, + package.candidate.architecture, + ) g = Gauge('apt_packages_obsolete_count', "Apt packages which are obsolete", registry=registry) @@ -104,7 +127,15 @@ def _write_obsolete_packages(registry, cache): def _write_autoremove_pending(registry, cache): - autoremovable_packages = {p for p in cache if p.is_auto_removable} + autoremovable_packages = { + p.candidate for p in cache if p.is_auto_removable + } + for candidate in autoremovable_packages: + logging.debug( + "autoremovable package: %s / %s", + candidate.package, + candidate.architecture, + ) g = Gauge('apt_autoremove_pending', "Apt packages pending autoremoval.", registry=registry) g.set(len(autoremovable_packages)) @@ -147,6 +178,9 @@ def _write_reboot_required(registry): def _main(): + if os.getenv('DEBUG'): + logging.basicConfig(level=logging.DEBUG) + cache = apt.cache.Cache() registry = CollectorRegistry() From c8e8a484e973f22c0ff152d04f81a1ea914ce141 Mon Sep 17 00:00:00 2001 From: Gabriel Filion Date: Tue, 28 Jan 2025 14:16:14 -0500 Subject: [PATCH 5/5] Make it possible to exclude packages by name In some situations some packages may need to be excluded, e.g. if we're expecting some odd situation. In this case it would be nice to exclude the packages by name. Signed-off-by: Gabriel Filion --- apt_info.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/apt_info.py b/apt_info.py index c633e436..c57e0b8c 100755 --- a/apt_info.py +++ b/apt_info.py @@ -24,6 +24,7 @@ import apt import apt_pkg +import argparse import collections import logging import os @@ -60,9 +61,11 @@ def _convert_candidates_to_upgrade_infos(candidates): return changes_list -def _write_pending_upgrades(registry, cache): +def _write_pending_upgrades(registry, cache, exclusions): candidates = { - p.candidate for p in cache if p.is_upgradable and not p.phasing_applied + p.candidate + for p in cache + if p.is_upgradable and not p.phasing_applied and p.name not in exclusions } for candidate in candidates: logging.debug( @@ -79,13 +82,14 @@ def _write_pending_upgrades(registry, cache): g.labels(change.labels['origin'], change.labels['arch']).set(change.count) -def _write_held_upgrades(registry, cache): +def _write_held_upgrades(registry, cache, exclusions): held_candidates = { p.candidate for p in cache if ( p.is_upgradable and p._pkg.selected_state == apt_pkg.SELSTATE_HOLD and not p.phasing_applied + and p.name not in exclusions ) } for candidate in held_candidates: @@ -103,13 +107,14 @@ def _write_held_upgrades(registry, cache): g.labels(change.labels['origin'], change.labels['arch']).set(change.count) -def _write_obsolete_packages(registry, cache): +def _write_obsolete_packages(registry, cache, exclusions): # This corresponds to the apt filter "?obsolete" obsoletes = [p for p in cache if p.is_installed and ( p.candidate is None or not p.candidate.origins or (len(p.candidate.origins) == 1 and p.candidate.origins[0].origin in ['', "/var/lib/dpkg/status"]) + and p.name not in exclusions )] for package in obsoletes: if package.candidate is None: @@ -126,9 +131,11 @@ def _write_obsolete_packages(registry, cache): g.set(len(obsoletes)) -def _write_autoremove_pending(registry, cache): +def _write_autoremove_pending(registry, cache, exclusions): autoremovable_packages = { - p.candidate for p in cache if p.is_auto_removable + p.candidate + for p in cache + if p.is_auto_removable and p.name not in exclusions } for candidate in autoremovable_packages: logging.debug( @@ -181,13 +188,17 @@ def _main(): if os.getenv('DEBUG'): logging.basicConfig(level=logging.DEBUG) + parser = argparse.ArgumentParser() + parser.add_argument("--exclude", nargs='*', default=[]) + args = parser.parse_args(sys.argv[1:]) + cache = apt.cache.Cache() registry = CollectorRegistry() - _write_pending_upgrades(registry, cache) - _write_held_upgrades(registry, cache) - _write_obsolete_packages(registry, cache) - _write_autoremove_pending(registry, cache) + _write_pending_upgrades(registry, cache, args.exclude) + _write_held_upgrades(registry, cache, args.exclude) + _write_obsolete_packages(registry, cache, args.exclude) + _write_autoremove_pending(registry, cache, args.exclude) _write_installed_packages_per_origin(registry, cache) _write_cache_timestamps(registry) _write_reboot_required(registry)