Skip to content

Commit 36fe4f2

Browse files
LordHepipudLordHepipud
authored
Merge pull request #498 from Icinga:feature/thread_keep_alive_and_housekeeping_on_freeze
Feature: Add thread queuing optimisation and frozen thread detection Adds feature to check for frozen threads on REST-Api, ensuring that non-responding threads are killed after 5 minutes without progress and restartet. Also improves queing of REST-Api tasks into threads, which now prioritizes to check for inactive threads first to enque new calls and falls back to old behaviour, in case all threads are busy.
2 parents 722b8ba + e4ddbea commit 36fe4f2

13 files changed

+186
-28
lines changed

‎doc/100-General/10-Changelog.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
2525
* [#469](https://github.com/Icinga/icinga-powershell-framework/pull/469) Improves plugin doc generator to allow multi-lines in code examples and updates plugin overview as table, adding a short description on what the plugin is for
2626
* [#495](https://github.com/Icinga/icinga-powershell-framework/pull/495) Adds feature to check the sign status for the local Icinga Agent certificate and notifying the user, in case the certificate is not yet signed by the Icinga CA
2727
* [#496](https://github.com/Icinga/icinga-powershell-framework/pull/496) Improves REST-Api default timeout for internal plugin execution calls from 30s to 120s
28+
* [#498](https://github.com/Icinga/icinga-powershell-framework/pull/498) Adds feature for thread queuing optimisation and frozen thread detection for REST calls
2829

2930
## 1.8.0 (2022-02-08)
3031

‎lib/core/framework/New-IcingaEnvironmentVariable.psm1‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ function New-IcingaEnvironmentVariable()
5050
$Global:Icinga.Public.Add('Daemons', @{ });
5151
$Global:Icinga.Public.Add('Threads', @{ });
5252
$Global:Icinga.Public.Add('ThreadPools', @{ });
53+
$Global:Icinga.Public.Add('ThreadAliveHousekeeping', @{ });
5354
}
5455

5556
# Session specific configuration which should never be modified by users!
@@ -60,5 +61,6 @@ function New-IcingaEnvironmentVariable()
6061
$Global:Icinga.Protected.Add('JEAContext', $FALSE);
6162
$Global:Icinga.Protected.Add('RunAsDaemon', $FALSE);
6263
$Global:Icinga.Protected.Add('Minimal', $FALSE);
64+
$Global:Icinga.Protected.Add('ThreadName', '');
6365
}
6466
}

‎lib/core/logging/Icinga_EventLog_Enums.psm1‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,12 @@ if ($null -eq $IcingaEventLogEnums -Or $IcingaEventLogEnums.ContainsKey('Framewo
110110
'Details' = 'The local Icinga Agent certificate seems not to be signed by our Icinga CA yet. Using this certificate for the REST-Api as example might not work yet. Please check the state of the certificate and complete the signing process if required [IWKB000013]';
111111
'EventId' = 1506;
112112
};
113+
1507 = @{
114+
'EntryType' = 'Error';
115+
'Message' = 'An internal threading error occurred. A frozen thread was detected';
116+
'Details' = 'One of the internal Icinga for Windows threads was being active but not responding for at least 5 minutes. The frozen thread has been terminated and restarted.';
117+
'EventId' = 1507;
118+
};
113119
1550 = @{
114120
'EntryType' = 'Error';
115121
'Message' = 'Unsupported web authentication used';

‎lib/core/thread/New-IcingaThreadInstance.psm1‎

Lines changed: 39 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,39 @@
11
function New-IcingaThreadInstance()
22
{
33
param (
4-
[string]$Name,
5-
$ThreadPool,
6-
[ScriptBlock]$ScriptBlock,
7-
[string]$Command,
8-
[hashtable]$CmdParameters,
9-
[array]$Arguments,
10-
[Switch]$Start
4+
[string]$Name = '',
5+
[string]$ThreadName = $null,
6+
$ThreadPool = $null,
7+
[ScriptBlock]$ScriptBlock = $null,
8+
[string]$Command = '',
9+
[hashtable]$CmdParameters = @{ },
10+
[array]$Arguments = @(),
11+
[Switch]$Start = $FALSE,
12+
[switch]$CheckAliveState = $FALSE
1113
);
1214

13-
$CallStack = Get-PSCallStack;
14-
$SourceCommand = $CallStack[1].Command;
15+
if ([string]::IsNullOrEmpty($ThreadName)) {
16+
$CallStack = Get-PSCallStack;
17+
$SourceCommand = $CallStack[1].Command;
1518

16-
if ([string]::IsNullOrEmpty($Name)) {
17-
$Name = New-IcingaThreadHash -ShellScript $ScriptBlock -Arguments $Arguments;
18-
}
19+
if ([string]::IsNullOrEmpty($Name)) {
20+
$Name = New-IcingaThreadHash -ShellScript $ScriptBlock -Arguments $Arguments;
21+
}
22+
23+
$ThreadName = [string]::Format('{0}::{1}::{2}::0', $SourceCommand, $Command, $Name);
24+
25+
[int]$ThreadIndex = 0;
26+
27+
while ($TRUE) {
28+
29+
if ($Global:Icinga.Public.Threads.ContainsKey($ThreadName) -eq $FALSE) {
30+
break;
31+
}
1932

20-
$ThreadName = [string]::Format('{0}::{1}::{2}::0', $SourceCommand, $Command, $Name);
33+
$ThreadIndex += 1;
34+
$ThreadName = [string]::Format('{0}::{1}::{2}::{3}', $SourceCommand, $Command, $Name, $ThreadIndex);
35+
}
36+
}
2137

2238
Write-IcingaDebugMessage -Message (
2339
[string]::Format(
@@ -51,6 +67,9 @@ function New-IcingaThreadInstance()
5167
[void]$Shell.AddParameter('JeaEnabled', $Global:Icinga.Protected.JEAContext);
5268
}
5369

70+
[void]$Shell.AddCommand('Set-IcingaEnvironmentThreadName');
71+
[void]$Shell.AddParameter('ThreadName', $ThreadName);
72+
5473
[void]$Shell.AddCommand($Command);
5574

5675
$CodeHash = $Command;
@@ -94,16 +113,13 @@ function New-IcingaThreadInstance()
94113
Add-Member -InputObject $Thread -MemberType NoteProperty -Name Started -Value $FALSE;
95114
}
96115

97-
[int]$ThreadIndex = 0;
98-
99-
while ($TRUE) {
100-
101-
if ($Global:Icinga.Public.Threads.ContainsKey($ThreadName) -eq $FALSE) {
102-
$Global:Icinga.Public.Threads.Add($ThreadName, $Thread);
103-
break;
104-
}
116+
$Global:Icinga.Public.Threads.Add($ThreadName, $Thread);
105117

106-
$ThreadIndex += 1;
107-
$ThreadName = [string]::Format('{0}::{1}::{2}::{3}', $SourceCommand, $Command, $Name, $ThreadIndex);
118+
if ($CheckAliveState) {
119+
Set-IcingaForWindowsThreadAlive `
120+
-ThreadName $ThreadName `
121+
-ThreadCmd $Command `
122+
-ThreadArgs $CmdParameters `
123+
-ThreadPool $ThreadPool;
108124
}
109125
}

‎lib/core/thread/Remove-IcingaThread.psm1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
function Remove-IcingaThread()
22
{
33
param(
4-
[string]$Thread
4+
[string]$Thread = ''
55
);
66

77
if ([string]::IsNullOrEmpty($Thread)) {

‎lib/core/thread/Set-IcingaEnvironmentThreadName.psm1‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
function Set-IcingaEnvironmentThreadName()
2+
{
3+
param (
4+
[string]$ThreadName = ''
5+
);
6+
7+
$Global:Icinga.Protected.ThreadName = $ThreadName;
8+
}

‎lib/core/thread/Stop-IcingaThread.psm1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
function Stop-IcingaThread()
22
{
3-
param(
4-
[string]$Thread
3+
param (
4+
[string]$Thread = ''
55
);
66

77
if ([string]::IsNullOrEmpty($Thread)) {

‎lib/daemon/Add-IcingaForWindowsDaemon.psm1‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,8 @@ function Add-IcingaForWindowsDaemon()
2121

2222
while ($TRUE) {
2323
Start-Sleep -Seconds 1;
24+
25+
# Handle possible threads being frozen
26+
Suspend-IcingaForWindowsFrozenThreads;
2427
}
2528
}

‎lib/daemon/Set-IcingaForWindowsThreadAlive.psm1‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
function Set-IcingaForWindowsThreadAlive()
2+
{
3+
param (
4+
[string]$ThreadName = '',
5+
[string]$ThreadCmd = '',
6+
$ThreadPool = $null,
7+
[hashtable]$ThreadArgs = @{ },
8+
[switch]$Active = $FALSE,
9+
[hashtable]$TerminateAction = @{ }
10+
);
11+
12+
if ([string]::IsNullOrEmpty($ThreadName)) {
13+
return;
14+
}
15+
16+
if ($Global:Icinga.Public.ThreadAliveHousekeeping.ContainsKey($ThreadName) -eq $FALSE) {
17+
if ($null -eq $ThreadPool) {
18+
return;
19+
}
20+
21+
$Global:Icinga.Public.ThreadAliveHousekeeping.Add(
22+
$ThreadName,
23+
@{
24+
'LastSeen' = [DateTime]::Now;
25+
'Command' = $ThreadCmd;
26+
'Arguments' = $ThreadArgs;
27+
'ThreadPool' = $ThreadPool;
28+
'Active' = [bool]$Active;
29+
'TerminateAction' = $TerminateAction;
30+
}
31+
);
32+
33+
return;
34+
}
35+
36+
$Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].LastSeen = [DateTime]::Now;
37+
$Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].Active = [bool]$Active;
38+
$Global:Icinga.Public.ThreadAliveHousekeeping[$ThreadName].TerminateAction = $TerminateAction;
39+
}

‎lib/daemon/Suspend-IcingaForWindowsFrozenThreads.psm1‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
function Suspend-IcingaForWindowsFrozenThreads()
2+
{
3+
try {
4+
[array]$ConfiguredThreads = $Global:Icinga.Public.ThreadAliveHousekeeping.Keys;
5+
6+
foreach ($thread in $ConfiguredThreads) {
7+
$ThreadConfig = $Global:Icinga.Public.ThreadAliveHousekeeping[$thread];
8+
9+
# Only check active threads
10+
if ($ThreadConfig.Active -eq $FALSE) {
11+
continue;
12+
}
13+
14+
# Check if the thread is active and not doing something for 5 minutes
15+
if (([DateTime]::Now - $ThreadConfig.LastSeen).TotalSeconds -lt 300) {
16+
continue;
17+
}
18+
19+
# If it does, kill the thread
20+
Remove-IcingaThread -Thread $thread;
21+
22+
if ($ThreadConfig.TerminateAction.Count -ne 0) {
23+
$TerminateArguments = @{ };
24+
if ($ThreadConfig.TerminateAction.ContainsKey('Arguments')) {
25+
$TerminateArguments = $ThreadConfig.TerminateAction.Arguments;
26+
}
27+
28+
if ($ThreadConfig.TerminateAction.ContainsKey('Command')) {
29+
$TerminateCmd = $ThreadConfig.TerminateAction.Command;
30+
31+
if ([string]::IsNullOrEmpty($TerminateCmd) -eq $FALSE) {
32+
& $TerminateCmd @TerminateArguments | Out-Null;
33+
}
34+
}
35+
}
36+
37+
# Now restart it
38+
New-IcingaThreadInstance `
39+
-ThreadName $thread `
40+
-ThreadPool $ThreadConfig.ThreadPool `
41+
-Command $ThreadConfig.Command `
42+
-CmdParameters $ThreadConfig.Arguments `
43+
-Start `
44+
-CheckAliveState;
45+
46+
Write-IcingaEventMessage -EventId 1507 -Namespace 'Framework' -Objects $thread;
47+
}
48+
} catch {
49+
# Nothing to do here
50+
}
51+
}

0 commit comments

Comments
 (0)