From c77349af9d41289a33c8a81f651b1ddd0035a4da Mon Sep 17 00:00:00 2001 From: Lord Hepipud Date: Wed, 26 Jul 2023 14:46:16 +0200 Subject: [PATCH] Adds IWKB and test/manage Cmdlets for SCOM intercept counters --- doc/100-General/10-Changelog.md | 1 + doc/300-Knowledge-Base.md | 3 +- doc/knowledgebase/IWKB000016.md | 87 +++++++++++++++++++ .../Disable-IcingaInterceptCounter.psm1 | 28 ++++++ .../Enable-IcingaInterceptCounter.psm1 | 27 ++++++ .../Test-IcingaInterceptCounter.psm1 | 36 ++++++++ 6 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 doc/knowledgebase/IWKB000016.md create mode 100644 lib/core/health/interceptcounter/Disable-IcingaInterceptCounter.psm1 create mode 100644 lib/core/health/interceptcounter/Enable-IcingaInterceptCounter.psm1 create mode 100644 lib/core/health/interceptcounter/Test-IcingaInterceptCounter.psm1 diff --git a/doc/100-General/10-Changelog.md b/doc/100-General/10-Changelog.md index b7c9e446..9557346a 100644 --- a/doc/100-General/10-Changelog.md +++ b/doc/100-General/10-Changelog.md @@ -33,6 +33,7 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic * [#640](https://github.com/Icinga/icinga-powershell-framework/issues/640) Adds support to set the flag `-NoSSLValidation` for Cmdlets `icinga` and `Install-Icinga`, to ignore errors on self-signed certificates within the environment * [#643](https://github.com/Icinga/icinga-powershell-framework/pull/643) Adds support for `-RebuildCache` flag on `icinga` cmd to rebuild component cache as well * [#644](https://github.com/Icinga/icinga-powershell-framework/pull/644) Adds progress bar output to repository interaction (sync, update, new) instead of plain text output +* [#655](https://github.com/Icinga/icinga-powershell-framework/pull/655) Adds [IWKB](https://icinga.com/docs/icinga-for-windows/latest/doc/knowledgebase/IWKB000016/) and test/manage Cmdlets for SCOM intercept counters ## 1.10.1 (2022-12-20) diff --git a/doc/300-Knowledge-Base.md b/doc/300-Knowledge-Base.md index e671e00b..ec029ab1 100644 --- a/doc/300-Knowledge-Base.md +++ b/doc/300-Knowledge-Base.md @@ -22,4 +22,5 @@ For this reason you will find a list of Icinga knowledge base entries below. Ent | [IWKB000012](knowledgebase/IWKB000012.md) | Icinga for Windows cannot be used with Microsoft Defender: `Windows Defender Antivirus has detected malware or other potentially unwanted software` | | [IWKB000013](knowledgebase/IWKB000013.md) | The local Icinga Agent certificate seems not to be signed by our Icinga CA yet. Using this certificate for the REST-Api as example might not work yet. Please check the state of the certificate and complete the signing process if required | | [IWKB000014](knowledgebase/IWKB000014.md) | Installing or Updating Icinga for Windows causes error messages regarding `framework_cache.psm1` errors | -| [IWKB000015](knowledgebase/IWKB000015.md) | Got JSON, but not an object, from IfW API on host 'localhost' port '5668': "Exception while calling \\"Fill\\" with 1 arguments: \\"Invalid syntax near \\"`:`\\".\\"" | \ No newline at end of file +| [IWKB000015](knowledgebase/IWKB000015.md) | Got JSON, but not an object, from IfW API on host 'localhost' port '5668': "Exception while calling \\"Fill\\" with 1 arguments: \\"Invalid syntax near \\"`:`\\".\\"" | +| [IWKB000016](knowledgebase/IWKB000016.md) | Checks using Performance Counter fail with various messages like `Exception of type 'System.OutOfMemoryException' was thrown` or `Icinga Invalid Configuration Error was thrown: PerfCounterCategoryMissing: Category "Memory" not found` | diff --git a/doc/knowledgebase/IWKB000016.md b/doc/knowledgebase/IWKB000016.md new file mode 100644 index 00000000..c63cd890 --- /dev/null +++ b/doc/knowledgebase/IWKB000016.md @@ -0,0 +1,87 @@ +# Icinga Knowledge Base - IWKB000016 + +## Short Message + +Checks using Performance Counter fail with various messages like `Exception of type 'System.OutOfMemoryException' was thrown` or `Icinga Invalid Configuration Error was thrown: PerfCounterCategoryMissing: Category "Memory" not found` + +## Example Exception + +### Out-Of-Memory Exception + +```powershell +[UNKNOWN]: Icinga Unhandled Error was thrown: Unhandled Exception + +Unhandled exception occurred: System.Management.Automation.MethodInvocationException: Exception calling "GetInstanceNames" with "0" argument(s): "Exception of type 'System.OutOfMemoryException' was thrown." ---> System.OutOfMemoryException: Exception of type 'System.OutOfMemoryException' was thrown. at Microsoft.Win32.RegistryKey.InternalGetValue(String name, Object defaultValue, Boolean doNotExpand, Boolean checkSecurity) at Microsoft.Win32.RegistryKey.GetValue(String name) at System.Diagnostics.PerformanceMonitor.GetData(String item) at System.Diagnostics.PerformanceCounterLib.GetPerformanceData(String item) at System.Diagnostics.PerformanceCounterLib.get_CategoryTable() at System.Diagnostics.PerformanceCounterLib.GetCategorySample(String category) at System.Diagnostics.PerformanceCounterLib.GetCategorySample(String machine, String category) at System.Diagnostics.PerformanceCounterCategory.GetCounterInstances(String categoryName, String machineName) at CallSite.Target(Closure , CallSite , Object ) --- End of inner exception stack trace --- at System.Management.Automation.ExceptionHandlingOps.CheckActionPreference(FunctionContext funcContext, Exception exception) at System.Management.Automation.Interpreter.ActionCallInstruction`2.Run(InterpretedFrame frame) at System.Management.Automation.Interpreter.EnterTryCatchFinallyInstruction.Run(InterpretedFrame frame) at System.Management.Automation.Interpreter.EnterTryCatchFinallyInstruction.Run(InterpretedFrame frame) +``` + +### Category "..." not found + +```powershell +[UNKNOWN]: Icinga Invalid Configuration Error was thrown: PerfCounterCategoryMissing: Category "Memory" not found + +The specified Performance Counter category was not found on this system. This could either be a configuration error on your local Windows machine or a wrong usage of the plugin. Please check on different Windows machines if this issue persis. In case it only occurs on certain machines it is likely that the counter is simply not present and the plugin can not be processed. +``` + +## Reason + +This issue seem only to appear while running Icinga for Windows **without** [JEA-Profile](../130-JEA/01-JEA-Profiles.md) and installed SCOM-Agent from Microsoft including the `APM` feature on the same machine. + +The reason can be tracked back to the `Intercept Counters` which are installed together with SCOM and `APM` feature. + +To check if you are affected by these counters, you can run the command `Test-IcingaInterceptCounter` starting with Icinga for Windows v1.11.0: + +```powershell +PS> Test-IcingaInterceptCounter; + +[Notice]: Testing for Microsoft SCOM Intercept Counters +[Failed]: Entry "HKLM:\SYSTEM\CurrentControlSet\Services\Intercept CSM Filters\Performance" is present on the system and the intercept counter is NOT disabled +[Passed]: Entry "HKLM:\SYSTEM\CurrentControlSet\Services\Intercept Injector\Performance" is not present on the system +[Passed]: Entry "HKLM:\SYSTEM\CurrentControlSet\Services\Intercept SyncAction Processing\Performance" is not present on the system +[Passed]: Entry "HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance" is not present on the system +[Passed]: Entry "HKLM:\SYSTEM\CurrentControlSet\Services\Backup Exec\Performance" is not present on the system +[Failed]: One or more intercept counters exist on this system which are not disabled. Please take a look at https://icinga.com/docs/icinga-for-windows/latest/doc/knowledgebase/IWKB000016/ for further details +``` + +## Solution + +If the command `Test-IcingaInterceptCounter` is reporting a `Failed` state on any of the tests, they are most likely the cause for issues regarding Performance Counter monitoring. + +### Requiring APM Feature + +Please check beforehand, if you are making use of any of the `Intercept Counters` installed with the SCOM-Agent in your environment. If you are using and requiring the `APM` feature provided by SCOM, the solution would be to reinstall the SCOM-Agent together with the `APM` feature, which should resolve the issue. + +If the error persists after the reinstallation, please open a support case for SCOM with Microsoft. + +### APM is not required + +In case you are not using any of the `APM` features provided the SCOM-Agent, you are save to disable to `Intercept Counters`. + +To disable these counters, you can run the command `Disable-IcingaInterceptCounter`: + +```powershell +PS> Disable-IcingaInterceptCounter; + +[Notice]: Disabling SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\Intercept CSM Filters\Performance" +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\Intercept Injector\Performance" not installed on the system +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\Intercept SyncAction Processing\Performance" not installed on the system +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance" not installed on the system +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance" not installed on the system +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\Backup Exec\Performance" not installed on the system +``` + +Afterwards reboot your server and the monitoring with plugins using Performance Counter should be fine. + +To enable the `Intercept Counters` again, you can run `Enable-IcingaInterceptCounter`: + +```powershell +PS> Enable-IcingaInterceptCounter; + +[Notice]: Enabling SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\Intercept CSM Filters\Performance" +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\Intercept Injector\Performance" not installed on the system +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\Intercept SyncAction Processing\Performance" not installed on the system +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance" not installed on the system +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance" not installed on the system +[Notice]: SCOM intercept counter "HKLM:\SYSTEM\CurrentControlSet\Services\Backup Exec\Performance" not installed on the system +``` + +Again, reboot your server afterwards to complete the changes. diff --git a/lib/core/health/interceptcounter/Disable-IcingaInterceptCounter.psm1 b/lib/core/health/interceptcounter/Disable-IcingaInterceptCounter.psm1 new file mode 100644 index 00000000..5ff26f66 --- /dev/null +++ b/lib/core/health/interceptcounter/Disable-IcingaInterceptCounter.psm1 @@ -0,0 +1,28 @@ +function Disable-IcingaInterceptCounter() +{ + [array]$InterceptCounterList = @( + 'HKLM:\SYSTEM\CurrentControlSet\Services\Intercept CSM Filters\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\Intercept Injector\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\Intercept SyncAction Processing\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\Backup Exec\Performance' + ); + + foreach ($counter in $InterceptCounterList) { + if (Test-Path $counter) { + Write-IcingaConsoleNotice 'Disabling SCOM intercept counter "{0}"' -Objects $counter + + $CounterState = Get-ItemProperty -Path $counter -Name 'Disable Performance Counters' -ErrorAction SilentlyContinue; + + if ($null -eq $CounterState) { + New-ItemProperty -Path $counter -Name 'Disable Performance Counters' -Value 1; + continue; + } + + Set-ItemProperty -Path $counter -Name 'Disable Performance Counters' -Value 1; + } else { + Write-IcingaConsoleNotice 'SCOM intercept counter "{0}" not installed on the system' -Objects $counter + } + } +} diff --git a/lib/core/health/interceptcounter/Enable-IcingaInterceptCounter.psm1 b/lib/core/health/interceptcounter/Enable-IcingaInterceptCounter.psm1 new file mode 100644 index 00000000..61f78f27 --- /dev/null +++ b/lib/core/health/interceptcounter/Enable-IcingaInterceptCounter.psm1 @@ -0,0 +1,27 @@ +function Enable-IcingaInterceptCounter() +{ + [array]$InterceptCounterList = @( + 'HKLM:\SYSTEM\CurrentControlSet\Services\Intercept CSM Filters\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\Intercept Injector\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\Intercept SyncAction Processing\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\Backup Exec\Performance' + ); + + foreach ($counter in $InterceptCounterList) { + if (Test-Path $counter) { + Write-IcingaConsoleNotice 'Enabling SCOM intercept counter "{0}"' -Objects $counter + + $CounterState = Get-ItemProperty -Path $counter -Name 'Disable Performance Counters' -ErrorAction SilentlyContinue; + + if ($null -eq $CounterState) { + continue; + } + + Set-ItemProperty -Path $counter -Name 'Disable Performance Counters' -Value 0; + } else { + Write-IcingaConsoleNotice 'SCOM intercept counter "{0}" not installed on the system' -Objects $counter + } + } +} diff --git a/lib/core/health/interceptcounter/Test-IcingaInterceptCounter.psm1 b/lib/core/health/interceptcounter/Test-IcingaInterceptCounter.psm1 new file mode 100644 index 00000000..9d465669 --- /dev/null +++ b/lib/core/health/interceptcounter/Test-IcingaInterceptCounter.psm1 @@ -0,0 +1,36 @@ +function Test-IcingaInterceptCounter() +{ + Write-IcingaConsoleNotice 'Testing for Microsoft SCOM Intercept Counters'; + + [bool]$TestResult = $TRUE; + [array]$InterceptCounterList = @( + 'HKLM:\SYSTEM\CurrentControlSet\Services\Intercept CSM Filters\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\Intercept Injector\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\Intercept SyncAction Processing\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\InterceptCountersManager\Performance', + 'HKLM:\SYSTEM\CurrentControlSet\Services\Backup Exec\Performance' + ); + + foreach ($counter in $InterceptCounterList) { + if (Test-Path -Path $counter) { + $CounterState = Get-ItemProperty -Path $counter -Name 'Disable Performance Counters' -ErrorAction SilentlyContinue; + + if ($null -eq $CounterState -Or $CounterState.'Disable Performance Counters' -eq 0) { + Write-IcingaTestOutput -Severity 'Failed' -Message ([string]::Format('Entry "{0}" is present on the system and the intercept counter is NOT disabled', $counter)); + $TestResult = $FALSE; + continue; + } + + Write-IcingaTestOutput -Severity 'Passed' -Message ([string]::Format('Entry "{0}" is present on the system and the intercept counter is disabled', $counter)); + } else { + Write-IcingaTestOutput -Severity 'Passed' -Message ([string]::Format('Entry "{0}" is not present on the system', $counter)); + } + } + + if ($TestResult -eq $FALSE) { + Write-IcingaTestOutput -Severity 'Failed' -Message 'One or more intercept counters exist on this system which are not disabled. Please take a look at https://icinga.com/docs/icinga-for-windows/latest/doc/knowledgebase/IWKB000016/ for further details'; + } else { + Write-IcingaTestOutput -Severity 'Passed' -Message 'There are either no intercept counters installed on your system or they are disabled. Monitoring of Performance Counters should work fine'; + } +}