Skip to content

Commit af89c79

Browse files
RincewindsHatRincewindsHat
authored
Merge pull request #26 from NETWAYS/filesystem_timeout
Add timeout for reading filesystem usage to exit properly with hanging filesystems.
2 parents f2701ce + 1bd01ca commit af89c79

File tree

4 files changed

+74
-32
lines changed

4 files changed

+74
-32
lines changed

‎cmd/filesystem.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package cmd
22

33
import (
4+
"context"
45
"errors"
56
"fmt"
67
"strconv"
8+
"time"
79

810
"github.com/NETWAYS/check_system_basics/internal/common/thresholds"
911
"github.com/NETWAYS/check_system_basics/internal/filesystem"
@@ -137,7 +139,10 @@ var diskCmd = &cobra.Command{
137139
}
138140

139141
// Retrieve stats
140-
err = filesystem.GetDiskUsage(filesystemList, &FsConfig)
142+
internalTimeout := time.Duration(Timeout/2) * time.Second
143+
ctx := context.Background()
144+
145+
err = filesystem.GetDiskUsage(ctx, internalTimeout, filesystemList, &FsConfig)
141146
if err != nil {
142147
check.ExitError(err)
143148
}
@@ -146,8 +151,9 @@ var diskCmd = &cobra.Command{
146151
for index := range filesystemList {
147152
sc := computeFsCheckResult(&filesystemList[index], &FsConfig)
148153

149-
sc.Output = fmt.Sprintf("%s (%.2f%% used space, %.2f%% free inodes)", sc.Output, filesystemList[index].UsageStats.UsedPercent, 100-filesystemList[index].UsageStats.InodesUsedPercent)
150-
154+
if filesystemList[index].Error == nil {
155+
sc.Output = fmt.Sprintf("%s (%.2f%% used space, %.2f%% free inodes)", sc.Output, filesystemList[index].UsageStats.UsedPercent, 100-filesystemList[index].UsageStats.InodesUsedPercent)
156+
}
151157
overall.AddSubcheck(sc)
152158
}
153159

‎internal/common/filter/filter.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
)
66

77
type Filterable interface {
8-
GetFilterableValue(uint) string
8+
GetFilterableValue(val uint) string
99
}
1010

1111
type Options struct {

‎internal/filesystem/filesystem.go‎

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package filesystem
22

33
import (
4+
"context"
45
"errors"
56
"regexp"
6-
"syscall"
7+
"time"
78

89
"github.com/shirou/gopsutil/v3/disk"
910
)
@@ -15,16 +16,43 @@ type FilesystemType struct {
1516
Error error
1617
}
1718

18-
func GetDiskUsage(fsList []FilesystemType, _ *CheckConfig) error {
19-
for index := range fsList {
20-
diskUsage, err := disk.Usage(fsList[index].PartStats.Mountpoint)
19+
type tmpFileSystemWrapper struct {
20+
usage disk.UsageStat
21+
err error
22+
}
23+
24+
func GetDiskUsageSingle(ctx context.Context, timeout time.Duration, fs *FilesystemType) {
25+
myCtx, cancel := context.WithTimeout(ctx, timeout)
26+
defer cancel()
27+
28+
resChan := make(chan tmpFileSystemWrapper, 1)
29+
30+
go func() {
31+
tmp := tmpFileSystemWrapper{}
32+
usageStats, err := disk.Usage(fs.PartStats.Mountpoint)
33+
tmp.usage = *usageStats
34+
tmp.err = err
2135

22-
if errors.Is(err, syscall.Errno(13)) {
23-
fsList[index].Error = err
24-
continue
36+
resChan <- tmp
37+
}()
38+
39+
select {
40+
case tmp := <-resChan:
41+
if tmp.err != nil {
42+
fs.Error = tmp.err
43+
return
2544
}
2645

27-
fsList[index].UsageStats = *diskUsage
46+
fs.UsageStats = tmp.usage
47+
case <-myCtx.Done():
48+
err := errors.New("Timeout exceeded for fs " + fs.PartStats.Mountpoint + ". Maybe hanging network filesystem?")
49+
fs.Error = err
50+
}
51+
}
52+
53+
func GetDiskUsage(ctx context.Context, timeout time.Duration, fsList []FilesystemType, _ *CheckConfig) error {
54+
for index := range fsList {
55+
GetDiskUsageSingle(ctx, timeout/time.Duration(len(fsList)), &fsList[index])
2856
}
2957

3058
return nil

‎internal/sensors/sensors.go‎

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,14 @@ type Device struct {
2929
Sensors []Sensor
3030
}
3131

32+
const (
33+
inputFileSuffix string = "_input"
34+
critThresholdFileSuffix string = "_crit"
35+
lowestValueFileSuffix string = "_lowest"
36+
highestValueFileSuffix string = "_highest"
37+
maxValueFileSuffix string = "_max"
38+
)
39+
3240
func (d *Device) String() string {
3341
result := d.Name
3442
result += ": "
@@ -263,7 +271,7 @@ func readHumiditySensor(devicePath string, index int) (Sensor, error) {
263271
sensor.Perfdata.Label = label
264272

265273
// Look for input (the actual value)
266-
value, err := readIntFromFile(basePath + "_input")
274+
value, err := readIntFromFile(basePath + inputFileSuffix)
267275

268276
if err != nil {
269277
return sensor, err
@@ -285,7 +293,7 @@ func readEnergySensor(devicePath string, index int) (Sensor, error) {
285293
sensor.Perfdata.Label = label
286294

287295
// Look for input (the actual value)
288-
value, err := readIntFromFile(basePath + "_input")
296+
value, err := readIntFromFile(basePath + inputFileSuffix)
289297
if err != nil {
290298
return sensor, err
291299
}
@@ -307,7 +315,7 @@ func readCurrSensor(devicePath string, index int) (Sensor, error) {
307315
sensor.Perfdata.Label = label
308316

309317
// Look for input (the actual value)
310-
value, err := readIntFromFile(basePath + "_input")
318+
value, err := readIntFromFile(basePath + inputFileSuffix)
311319
if err != nil {
312320
return sensor, err
313321
}
@@ -317,14 +325,14 @@ func readCurrSensor(devicePath string, index int) (Sensor, error) {
317325

318326
// == Min
319327
// Is there a currN_lowest file? Use it for max value
320-
value, err = readIntFromFile(basePath + "_lowest")
328+
value, err = readIntFromFile(basePath + lowestValueFileSuffix)
321329
if err == nil {
322330
sensor.Perfdata.Min = float64(value) / 1000
323331
}
324332

325333
// == Max
326334
// Is there a currN_highest file? Use it for max value
327-
value, err = readIntFromFile(basePath + "_highest")
335+
value, err = readIntFromFile(basePath + highestValueFileSuffix)
328336
if err == nil {
329337
sensor.Perfdata.Max = float64(value) / 1000
330338
}
@@ -343,7 +351,7 @@ func readCurrSensor(devicePath string, index int) (Sensor, error) {
343351
critPresent = true
344352
}
345353
// Is there a currN_crit file? If yes, use that as upper critical
346-
value, err = readIntFromFile(basePath + "_crit")
354+
value, err = readIntFromFile(basePath + critThresholdFileSuffix)
347355
if err == nil {
348356
tmp.Upper = float64(value) / 1000
349357
critPresent = true
@@ -391,7 +399,7 @@ func readFanSensor(devicePath string, index int) (Sensor, error) {
391399
sensor.Perfdata.Label = label
392400

393401
// Look for input (the actual value)
394-
value, err := readIntFromFile(basePath + "_input")
402+
value, err := readIntFromFile(basePath + inputFileSuffix)
395403
if err != nil {
396404
return sensor, err
397405
}
@@ -403,7 +411,7 @@ func readFanSensor(devicePath string, index int) (Sensor, error) {
403411

404412
// == Max
405413
// Is there a tempN_highest file? Use it for max value
406-
value, err = readIntFromFile(basePath + "_max")
414+
value, err = readIntFromFile(basePath + maxValueFileSuffix)
407415
if err == nil {
408416
sensor.Perfdata.Max = float64(value)
409417
}
@@ -424,7 +432,7 @@ func readVoltageSensor(_, devicePath string, index int) (Sensor, error) {
424432
sensor.Perfdata.Label = label
425433

426434
// Look for input (the actual value)
427-
value, err := readIntFromFile(basePath + "_input")
435+
value, err := readIntFromFile(basePath + inputFileSuffix)
428436
if err != nil {
429437
return sensor, err
430438
}
@@ -440,7 +448,7 @@ func readVoltageSensor(_, devicePath string, index int) (Sensor, error) {
440448
}
441449
warnPresent := false
442450
// Is there a inN_max file? If yes, use that as warning
443-
value, err = readIntFromFile(basePath + "_max")
451+
value, err = readIntFromFile(basePath + maxValueFileSuffix)
444452
if err == nil {
445453
tmpWarn.Upper = float64(value) / 1000
446454
warnPresent = true
@@ -458,7 +466,7 @@ func readVoltageSensor(_, devicePath string, index int) (Sensor, error) {
458466
}
459467
critPresent := false
460468
// Is there a inN_crit file? If yes, use that as critical
461-
value, err = readIntFromFile(basePath + "_crit")
469+
value, err = readIntFromFile(basePath + critThresholdFileSuffix)
462470
if err == nil {
463471
tmpCrit.Upper = float64(value) / 1000
464472
critPresent = true
@@ -476,14 +484,14 @@ func readVoltageSensor(_, devicePath string, index int) (Sensor, error) {
476484
}
477485

478486
// == Min
479-
value, err = readIntFromFile(basePath + "_lowest")
487+
value, err = readIntFromFile(basePath + lowestValueFileSuffix)
480488
if err == nil {
481489
sensor.Perfdata.Min = float64(value) / 1000
482490
}
483491

484492
// == Max
485493
// Is there a tempN_highest file? Use it for max value
486-
value, err = readIntFromFile(basePath + "_highest")
494+
value, err = readIntFromFile(basePath + highestValueFileSuffix)
487495
if err == nil {
488496
sensor.Perfdata.Max = float64(value) / 1000
489497
}
@@ -503,7 +511,7 @@ func readPowerSensor(_, devicePath string, index int) (Sensor, error) {
503511
sensor.Perfdata.Label = label
504512

505513
// Look for input (the actual value)
506-
value, err := readIntFromFile(basePath + "_input")
514+
value, err := readIntFromFile(basePath + inputFileSuffix)
507515

508516
if err != nil {
509517
return sensor, err
@@ -551,7 +559,7 @@ func readPowerSensor(_, devicePath string, index int) (Sensor, error) {
551559
}
552560
critPresent := false
553561
// Is there a powerN_crit file? If yes, use that as critical
554-
value, err = readIntFromFile(basePath + "_crit")
562+
value, err = readIntFromFile(basePath + critThresholdFileSuffix)
555563
if err == nil {
556564
tmpCrit.Upper = float64(value)
557565
critPresent = true
@@ -574,7 +582,7 @@ func readTempSensor(_, devicePath string, index int) (Sensor, error) {
574582
sensor.Perfdata.Label = label
575583

576584
// Look for input (the actual value)
577-
value, err := readIntFromFile(basePath + "_input")
585+
value, err := readIntFromFile(basePath + inputFileSuffix)
578586

579587
if err != nil {
580588
return sensor, err
@@ -591,7 +599,7 @@ func readTempSensor(_, devicePath string, index int) (Sensor, error) {
591599
}
592600
warnPresent := false
593601
// Is there a tempN_max file? If yes, use that as warning
594-
value, err = readIntFromFile(basePath + "_max")
602+
value, err = readIntFromFile(basePath + maxValueFileSuffix)
595603

596604
if err == nil {
597605
tmpWarn.Upper = float64(value / 1000)
@@ -611,7 +619,7 @@ func readTempSensor(_, devicePath string, index int) (Sensor, error) {
611619

612620
critPresent := false
613621
// Is there a tempN_crit file? If yes, use that as critical
614-
value, err = readIntFromFile(basePath + "_crit")
622+
value, err = readIntFromFile(basePath + critThresholdFileSuffix)
615623

616624
if err == nil {
617625
tmpCrit.Upper = float64(value / 1000)
@@ -640,15 +648,15 @@ func readTempSensor(_, devicePath string, index int) (Sensor, error) {
640648

641649
// == Min
642650
// Is there a tempN_lowest file? Use it for min value
643-
value, err = readIntFromFile(basePath + "_lowest")
651+
value, err = readIntFromFile(basePath + lowestValueFileSuffix)
644652

645653
if err == nil {
646654
sensor.Perfdata.Min = value
647655
}
648656

649657
// == Max
650658
// Is there a tempN_highest file? Use it for max value
651-
value, err = readIntFromFile(basePath + "_highest")
659+
value, err = readIntFromFile(basePath + highestValueFileSuffix)
652660

653661
if err == nil {
654662
sensor.Perfdata.Max = value

0 commit comments

Comments
 (0)