diff --git a/doc/100-General/10-Changelog.md b/doc/100-General/10-Changelog.md index 722273e0..0222eef5 100644 --- a/doc/100-General/10-Changelog.md +++ b/doc/100-General/10-Changelog.md @@ -25,6 +25,7 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic * [#469](https://github.com/Icinga/icinga-powershell-framework/pull/469) Improves plugin doc generator to allow multi-lines in code examples and updates plugin overview as table, adding a short description on what the plugin is for * [#495](https://github.com/Icinga/icinga-powershell-framework/pull/495) Adds feature to check the sign status for the local Icinga Agent certificate and notifying the user, in case the certificate is not yet signed by the Icinga CA * [#496](https://github.com/Icinga/icinga-powershell-framework/pull/496) Improves REST-Api default timeout for internal plugin execution calls from 30s to 120s +* [#498](https://github.com/Icinga/icinga-powershell-framework/pull/498) Adds feature for thread queuing optimisation and frozen thread detection for REST calls ## 1.8.0 (2022-02-08) diff --git a/lib/core/framework/New-IcingaEnvironmentVariable.psm1 b/lib/core/framework/New-IcingaEnvironmentVariable.psm1 index a47ec3da..4134f990 100644 --- a/lib/core/framework/New-IcingaEnvironmentVariable.psm1 +++ b/lib/core/framework/New-IcingaEnvironmentVariable.psm1 @@ -50,6 +50,7 @@ function New-IcingaEnvironmentVariable() $Global:Icinga.Public.Add('Daemons', @{ }); $Global:Icinga.Public.Add('Threads', @{ }); $Global:Icinga.Public.Add('ThreadPools', @{ }); + $Global:Icinga.Public.Add('ThreadAliveHousekeeping', @{ }); } # Session specific configuration which should never be modified by users! @@ -60,5 +61,6 @@ function New-IcingaEnvironmentVariable() $Global:Icinga.Protected.Add('JEAContext', $FALSE); $Global:Icinga.Protected.Add('RunAsDaemon', $FALSE); $Global:Icinga.Protected.Add('Minimal', $FALSE); + $Global:Icinga.Protected.Add('ThreadName', ''); } } diff --git a/lib/core/logging/Icinga_EventLog_Enums.psm1 b/lib/core/logging/Icinga_EventLog_Enums.psm1 index 1ece133a..a0427380 100644 --- a/lib/core/logging/Icinga_EventLog_Enums.psm1 +++ b/lib/core/logging/Icinga_EventLog_Enums.psm1 @@ -110,6 +110,12 @@ if ($null -eq $IcingaEventLogEnums -Or $IcingaEventLogEnums.ContainsKey('Framewo 'Details' = 'The local Icinga Agent certificate seems not to be signed by our Icinga CA yet. Using this certificate for the REST-Api as example might not work yet. Please check the state of the certificate and complete the signing process if required [IWKB000013]'; 'EventId' = 1506; }; + 1507 = @{ + 'EntryType' = 'Error'; + 'Message' = 'An internal threading error occurred. A frozen thread was detected'; + 'Details' = 'One of the internal Icinga for Windows threads was being active but not responding for at least 5 minutes. The frozen thread has been terminated and restarted.'; + 'EventId' = 1507; + }; 1550 = @{ 'EntryType' = 'Error'; 'Message' = 'Unsupported web authentication used'; diff --git a/lib/core/thread/New-IcingaThreadInstance.psm1 b/lib/core/thread/New-IcingaThreadInstance.psm1 index 13c3c3f1..ddaa6adc 100644 --- a/lib/core/thread/New-IcingaThreadInstance.psm1 +++ b/lib/core/thread/New-IcingaThreadInstance.psm1 @@ -1,23 +1,39 @@ function New-IcingaThreadInstance() { param ( - [string]$Name, - $ThreadPool, - [ScriptBlock]$ScriptBlock, - [string]$Command, - [hashtable]$CmdParameters, - [array]$Arguments, - [Switch]$Start + [string]$Name = '', + [string]$ThreadName = $null, + $ThreadPool = $null, + [ScriptBlock]$ScriptBlock = $null, + [string]$Command = '', + [hashtable]$CmdParameters = @{ }, + [array]$Arguments = @(), + [Switch]$Start = $FALSE, + [switch]$CheckAliveState = $FALSE ); - $CallStack = Get-PSCallStack; - $SourceCommand = $CallStack[1].Command; + if ([string]::IsNullOrEmpty($ThreadName)) { + $CallStack = Get-PSCallStack; + $SourceCommand = $CallStack[1].Command; - if ([string]::IsNullOrEmpty($Name)) { - $Name = New-IcingaThreadHash -ShellScript $ScriptBlock -Arguments $Arguments; - } + if ([string]::IsNullOrEmpty($Name)) { + $Name = New-IcingaThreadHash -ShellScript $ScriptBlock -Arguments $Arguments; + } + + $ThreadName = [string]::Format('{0}::{1}::{2}::0', $SourceCommand, $Command, $Name); + + [int]$ThreadIndex = 0; + + while ($TRUE) { + + if ($Global:Icinga.Public.Threads.ContainsKey($ThreadName) -eq $FALSE) { + break; + } - $ThreadName = [string]::Format('{0}::{1}::{2}::0', $SourceCommand, $Command, $Name); + $ThreadIndex += 1; + $ThreadName = [string]::Format('{0}::{1}::{2}::{3}', $SourceCommand, $Command, $Name, $ThreadIndex); + } + } Write-IcingaDebugMessage -Message ( [string]::Format( @@ -51,6 +67,9 @@ function New-IcingaThreadInstance() [void]$Shell.AddParameter('JeaEnabled', $Global:Icinga.Protected.JEAContext); } + [void]$Shell.AddCommand('Set-IcingaEnvironmentThreadName'); + [void]$Shell.AddParameter('ThreadName', $ThreadName); + [void]$Shell.AddCommand($Command); $CodeHash = $Command; @@ -94,16 +113,13 @@ function New-IcingaThreadInstance() Add-Member -InputObject $Thread -MemberType NoteProperty -Name Started -Value $FALSE; } - [int]$ThreadIndex = 0; - - while ($TRUE) { - - if ($Global:Icinga.Public.Threads.ContainsKey($ThreadName) -eq $FALSE) { - $Global:Icinga.Public.Threads.Add($ThreadName, $Thread); - break; - } + $Global:Icinga.Public.Threads.Add($ThreadName, $Thread); - $ThreadIndex += 1; - $ThreadName = [string]::Format('{0}::{1}::{2}::{3}', $SourceCommand, $Command, $Name, $ThreadIndex); + if ($CheckAliveState) { + Set-IcingaForWindowsThreadAlive ` + -ThreadName $ThreadName ` + -ThreadCmd $Command ` + -ThreadArgs $CmdParameters ` + -ThreadPool $ThreadPool; } } diff --git a/lib/core/thread/Remove-IcingaThread.psm1 b/lib/core/thread/Remove-IcingaThread.psm1 index 4dc7e1ad..295954c6 100644 --- a/lib/core/thread/Remove-IcingaThread.psm1 +++ b/lib/core/thread/Remove-IcingaThread.psm1 @@ -1,7 +1,7 @@ function Remove-IcingaThread() { param( - [string]$Thread + [string]$Thread = '' ); if ([string]::IsNullOrEmpty($Thread)) { diff --git a/lib/core/thread/Set-IcingaEnvironmentThreadName.psm1 b/lib/core/thread/Set-IcingaEnvironmentThreadName.psm1 new file mode 100644 index 00000000..cf73e503 --- /dev/null +++ b/lib/core/thread/Set-IcingaEnvironmentThreadName.psm1 @@ -0,0 +1,8 @@ +function Set-IcingaEnvironmentThreadName() +{ + param ( + [string]$ThreadName = '' + ); + + $Global:Icinga.Protected.ThreadName = $ThreadName; +} diff --git a/lib/core/thread/Stop-IcingaThread.psm1 b/lib/core/thread/Stop-IcingaThread.psm1 index e934b74e..3a8f2ff4 100644 --- a/lib/core/thread/Stop-IcingaThread.psm1 +++ b/lib/core/thread/Stop-IcingaThread.psm1 @@ -1,7 +1,7 @@ function Stop-IcingaThread() { - param( - [string]$Thread + param ( + [string]$Thread = '' ); if ([string]::IsNullOrEmpty($Thread)) { diff --git a/lib/daemon/Add-IcingaForWindowsDaemon.psm1 b/lib/daemon/Add-IcingaForWindowsDaemon.psm1 index 1933359b..85d5225f 100644 --- a/lib/daemon/Add-IcingaForWindowsDaemon.psm1 +++ b/lib/daemon/Add-IcingaForWindowsDaemon.psm1 @@ -21,5 +21,8 @@ function Add-IcingaForWindowsDaemon() while ($TRUE) { Start-Sleep -Seconds 1; + + # Handle possible threads being frozen + Suspend-IcingaForWindowsFrozenThreads; } } diff --git a/lib/daemon/Set-IcingaForWindowsThreadAlive.psm1 b/lib/daemon/Set-IcingaForWindowsThreadAlive.psm1 new file mode 100644 index 00000000..dd1781bf --- /dev/null +++ b/lib/daemon/Set-IcingaForWindowsThreadAlive.psm1 @@ -0,0 +1,39 @@ +function Set-IcingaForWindowsThreadAlive() +{ + param ( + [string]$ThreadName = '', + [string]$ThreadCmd = '', + $ThreadPool = $null, + [hashtable]$ThreadArgs = @{ }, + [switch]$Active = $FALSE, + [hashtable]$TerminateAction = @{ } + ); + + if ([string]::IsNullOrEmpty($ThreadName)) { + return; + } + + if ($Global:Icinga.Public.ThreadAliveHousekeeping.ContainsKey($ThreadName) -eq $FALSE) { + if ($null -eq $ThreadPool) { + return; + } + + $Global:Icinga.Public.ThreadAliveHousekeeping.Add( + $ThreadName, + @{ + 'LastSeen' = [DateTime]::Now; + 'Command' = $ThreadCmd; + 'Arguments' = $ThreadArgs; + 'ThreadPool' = $ThreadPool; + 'Active' = [bool]$Active; + 'TerminateAction' = $TerminateAction; + } + ); + + return; + } + + $Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].LastSeen = [DateTime]::Now; + $Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].Active = [bool]$Active; + $Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].TerminateAction = $TerminateAction; +} diff --git a/lib/daemon/Suspend-IcingaForWindowsFrozenThreads.psm1 b/lib/daemon/Suspend-IcingaForWindowsFrozenThreads.psm1 new file mode 100644 index 00000000..fa4e8b4b --- /dev/null +++ b/lib/daemon/Suspend-IcingaForWindowsFrozenThreads.psm1 @@ -0,0 +1,51 @@ +function Suspend-IcingaForWindowsFrozenThreads() +{ + try { + [array]$ConfiguredThreads = $Global:Icinga.Public.ThreadAliveHousekeeping.Keys; + + foreach ($thread in $ConfiguredThreads) { + $ThreadConfig = $Global:Icinga.Public.ThreadAliveHousekeeping[$thread]; + + # Only check active threads + if ($ThreadConfig.Active -eq $FALSE) { + continue; + } + + # Check if the thread is active and not doing something for 5 minutes + if (([DateTime]::Now - $ThreadConfig.LastSeen).TotalSeconds -lt 300) { + continue; + } + + # If it does, kill the thread + Remove-IcingaThread -Thread $thread; + + if ($ThreadConfig.TerminateAction.Count -ne 0) { + $TerminateArguments = @{ }; + if ($ThreadConfig.TerminateAction.ContainsKey('Arguments')) { + $TerminateArguments = $ThreadConfig.TerminateAction.Arguments; + } + + if ($ThreadConfig.TerminateAction.ContainsKey('Command')) { + $TerminateCmd = $ThreadConfig.TerminateAction.Command; + + if ([string]::IsNullOrEmpty($TerminateCmd) -eq $FALSE) { + & $TerminateCmd @TerminateArguments | Out-Null; + } + } + } + + # Now restart it + New-IcingaThreadInstance ` + -ThreadName $thread ` + -ThreadPool $ThreadConfig.ThreadPool ` + -Command $ThreadConfig.Command ` + -CmdParameters $ThreadConfig.Arguments ` + -Start ` + -CheckAliveState; + + Write-IcingaEventMessage -EventId 1507 -Namespace 'Framework' -Objects $thread; + } + } catch { + # Nothing to do here + } +} diff --git a/lib/daemons/RestAPI/threads/Get-IcingaNextRESTApiThreadId.psm1 b/lib/daemons/RestAPI/threads/Get-IcingaNextRESTApiThreadId.psm1 index 08950095..4840933b 100644 --- a/lib/daemons/RestAPI/threads/Get-IcingaNextRESTApiThreadId.psm1 +++ b/lib/daemons/RestAPI/threads/Get-IcingaNextRESTApiThreadId.psm1 @@ -1,5 +1,29 @@ function Get-IcingaNextRESTApiThreadId() { + # Improve our thread management by distributing new REST requests to a non-active thread + [array]$ConfiguredThreads = $Global:Icinga.Public.ThreadAliveHousekeeping.Keys; + + foreach ($thread in $ConfiguredThreads) { + if ($thread.ToLower() -NotLike 'Start-IcingaForWindowsRESTThread::New-IcingaForWindowsRESTThread::CheckThread::*') { + continue; + } + + $ThreadConfig = $Global:Icinga.Public.ThreadAliveHousekeeping[$thread]; + + # If our thread is busy, skip this one and check for another one + if ($ThreadConfig.Active) { + continue; + } + + $ThreadIndex = $thread.Replace('Start-IcingaForWindowsRESTThread::New-IcingaForWindowsRESTThread::CheckThread::', ''); + + if (Test-Numeric $ThreadIndex) { + $Global:Icinga.Public.Daemons.RESTApi.LastThreadId = [int]$ThreadIndex; + return ([int]$ThreadIndex) + } + } + + # In case we are not having any spare thread left, distribute the thread to the next thread in our list [int]$ConcurrentThreads = $Global:Icinga.Public.Daemons.RESTApi.TotalThreads - 1; [int]$LastThreadId = $Global:Icinga.Public.Daemons.RESTApi.LastThreadId + 1; diff --git a/lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1 b/lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1 index 67572a59..146a0e7a 100644 --- a/lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1 +++ b/lib/daemons/RestAPI/threads/New-IcingaForWindowsRESTThread.psm1 @@ -1,6 +1,6 @@ function New-IcingaForWindowsRESTThread() { - param( + param ( $RequireAuth, $ThreadId ); @@ -69,6 +69,9 @@ function New-IcingaForWindowsRESTThread() } } + # Set our thread being active + Set-IcingaForWindowsThreadAlive -ThreadName $Global:Icinga.Protected.ThreadName -Active -TerminateAction @{ 'Command' = 'Close-IcingaTCPConnection'; 'Arguments' = @{ 'Client' = $Connection.Client } }; + # We should remove clients from the blacklist who are sending valid requests Remove-IcingaRESTClientBlacklist -Client $Connection.Client -ClientList $Global:Icinga.Public.Daemons.RESTApi.ClientBlacklist; switch (Get-IcingaRESTPathElement -Request $RESTRequest -Index 0) { @@ -85,6 +88,10 @@ function New-IcingaForWindowsRESTThread() ) -Stream $Connection.Stream; }; } + + # set our thread no longer be active. We do this, because below there is no way we can + # actually get stuck on a endless loop, caused by external modules + Set-IcingaForWindowsThreadAlive -ThreadName $Global:Icinga.Protected.ThreadName; } } catch { $ExMsg = $_.Exception.Message; diff --git a/lib/daemons/RestAPI/threads/Start-IcingaForWindowsRESTThread.psm1 b/lib/daemons/RestAPI/threads/Start-IcingaForWindowsRESTThread.psm1 index 8921c34d..83bb1d33 100644 --- a/lib/daemons/RestAPI/threads/Start-IcingaForWindowsRESTThread.psm1 +++ b/lib/daemons/RestAPI/threads/Start-IcingaForWindowsRESTThread.psm1 @@ -15,5 +15,6 @@ function Start-IcingaForWindowsRESTThread() 'RequireAuth' = $RequireAuth; 'ThreadId' = $ThreadId; } ` - -Start; + -Start ` + -CheckAliveState; }