plugins: add clean command to remove inactive plugins/dependencies

lib/pluginmanager/clean.rb

@@ -0,0 +1,59 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require "pluginmanager/command"
+require 'set'
+require 'rubygems/uninstaller'
+
+class LogStash::PluginManager::Clean < LogStash::PluginManager::Command
+
+  option "--dry-run", :flag, "If set, only report what would be deleted", :default => false
+
+  def execute
+    locked_gem_names = ::Bundler::LockfileParser.new(File.read(LogStash::Environment::LOCKFILE)).specs.map(&:full_name).to_set
+    orphan_gem_specs = ::Gem::Specification.each
+                                           .reject(&:stubbed?) # skipped stubbed (uninstalled) gems
+                                           .reject(&:default_gem?) # don't touch jruby-included default gems
+                                           .reject{ |spec| locked_gem_names.include?(spec.full_name) }
+                                           .sort
+
+    inactive_plugins, orphaned_dependencies = orphan_gem_specs.partition { |spec| LogStash::PluginManager.logstash_plugin_gem_spec?(spec) }
+
+    # uninstall plugins first, to limit damage should one fail to uninstall
+    inactive_plugins.each { |plugin| uninstall("inactive plugin", plugin) }
+    orphaned_dependencies.each { |dep| uninstall("orphaned dependency", dep) }
+  end
+
+  def uninstall(desc, spec)
+    full_desc = "#{desc} #{spec.name} (#{spec.version})"
+    if dry_run?
+      puts "would clean #{full_desc}"
+    else
+      uninstall_gem!(spec)
+      puts "cleaned #{full_desc}"
+    end
+  end
+
+  def uninstall_gem!(gem_spec)
+    removal_options = { force: true, executables: true }
+    Gem::DefaultUserInteraction.use_ui(debug? ? Gem::DefaultUserInteraction.ui : Gem::SilentUI.new) do
+      Gem::Uninstaller.new(gem_spec.name, removal_options.merge(version: gem_spec.version)).uninstall
+    end
+  rescue Gem::InstallError => e
+    fail "Failed to uninstall `#{gem_spec.full_name}`"
+  end
+end

lib/pluginmanager/main.rb

@@ -34,6 +34,7 @@ module PluginManager
 require "pluginmanager/remove"
 require "pluginmanager/list"
 require "pluginmanager/update"
+require "pluginmanager/clean"
 require "pluginmanager/pack"
 require "pluginmanager/unpack"
 require "pluginmanager/generate"
@@ -50,6 +51,7 @@ class Main < Clamp::Command
       subcommand "install", "Install a Logstash plugin", LogStash::PluginManager::Install
       subcommand "remove", "Remove a Logstash plugin", LogStash::PluginManager::Remove
       subcommand "update", "Update a plugin", LogStash::PluginManager::Update
+      subcommand "clean", "Remove all inactive plugins", LogStash::PluginManager::Clean
       subcommand "pack", "Package currently installed plugins, Deprecated: Please use prepare-offline-pack instead", LogStash::PluginManager::Pack
       subcommand "unpack", "Unpack packaged plugins, Deprecated: Please use prepare-offline-pack instead", LogStash::PluginManager::Unpack
       subcommand "generate", "Create the foundation for a new plugin", LogStash::PluginManager::Generate

qa/integration/fixtures/clean_spec.yml

@@ -0,0 +1,3 @@
+---
+services:
+  - logstash

qa/integration/services/logstash_service.rb

@@ -343,6 +343,10 @@ def remove(plugin_name, *additional_plugins)
       run("remove #{plugin_list}")
     end
 
+    def clean
+      run("clean")
+    end
+
     def prepare_offline_pack(plugins, output_zip = nil)
       plugins = Array(plugins)
 

qa/integration/specs/cli/clean_spec.rb

@@ -0,0 +1,92 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../../framework/fixture'
+require_relative '../../framework/settings'
+require_relative '../../services/logstash_service'
+require_relative '../../framework/helpers'
+require "logstash/devutils/rspec/spec_helper"
+
+require "bundler/lockfile_parser"
+
+describe "CLI > logstash-plugin clean" do
+
+  let(:fixture) { Fixture.new(__FILE__) }
+  let(:logstash) { fixture.get_service("logstash") }
+  let(:gem_vendor_path) { (Pathname.new(logstash.logstash_home) / "vendor" / "bundle" / "jruby").glob("[0-9]*").first }
+
+  subject(:logstash_plugin) { logstash.plugin_cli }
+
+  ## inspects the Gemfile.lock to get a mapping of active gems
+  # @return [Hash{String=>String}]
+  def activated_gems
+    ::Bundler::LockfileParser.new(File.read(logstash.lock_file)).specs.each_with_object({}) do |spec, memo|
+      memo[spec.name] = spec.version.to_s
+    end
+  end
+
+  def find_vendored_gemspecs(gem_name)
+    (gem_vendor_path / "specifications").glob("#{gem_name}-[0-9]*.gemspec")
+  end
+
+  def find_vendored_gem_files(gem_name)
+    (gem_vendor_path / "gems").glob("#{gem_name}-[0-9]*")
+  end
+
+
+  context "when run after removing a plugin with many gem dependencies" do
+    # we uninstall the AWS integration plugin, because we know that it brings a number of extra dependencies
+    # and we validate in the `before` bock that the regular `remove` operation successfully _deactivates_ those
+    # gems but that they are still present on disk
+    before(:each) do
+      aggregate_failures("setup") do
+        activated_gems_pre_removal = activated_gems.freeze
+
+        logstash_plugin.remove("logstash-integration-aws")
+
+        activated_gems_post_removal = activated_gems.freeze
+
+        # ensure that we have actually deactivated some gems, including known aws-integration dependencies
+        @deactivated_gems = activated_gems_pre_removal.keys - activated_gems_post_removal.keys
+        expect(@deactivated_gems).to_not be_empty
+        expect(@deactivated_gems).to include("logstash-integration-aws"), lambda { "expected `logstash-integration-aws` gems to not be in the active set after plugin removal: #{activated_gems_post_removal}" }
+        expect(@deactivated_gems).to include("aws-sdk-core"), lambda { "expected AWS-related gems dependencies to not be in the active set after plugin removal: #{activated_gems_post_removal}" }
+
+        # we expect the gemspecs and expanded gems to still be present before a `clean`
+        @deactivated_gems.each do |deactivated_gem|
+          expect(find_vendored_gemspecs(deactivated_gem)).to_not be_empty, lambda { "expected remaining gemspec for `#{deactivated_gem}` after normal removal"}
+          expect(find_vendored_gem_files(deactivated_gem)).to_not be_empty, lambda { "expected remaining gem files for `#{deactivated_gem}` after normal removal"}
+        end
+      end
+    end
+
+    it "successfully removes the deactivated plugins and orphaned dependencies from disk" do
+      logstash_plugin.clean
+
+      aggregate_failures do
+        @deactivated_gems.each do |gem_name|
+          find_vendored_gemspecs(gem_name).tap do |found_vendored_gemspecs|
+            expect(find_vendored_gemspecs(gem_name)).to be_empty, lambda { "expected gemspecs for `#{gem_name}` to NOT be present after being cleaned (found: `#{found_vendored_gemspecs}`)"}
+          end
+          find_vendored_gem_files(gem_name).tap do |found_vendored_gem_files|
+            expect(found_vendored_gem_files).to be_empty, lambda { "expected gem files for` #{gem_name}` to NOT be present after being cleaned (found: `#{found_vendored_gem_files}`"}
+          end
+        end
+      end
+    end
+  end
+end