GPUEff: change calculation to properly average over all job steps

Author: rkdarst

README.rst

@@ -285,16 +285,17 @@ them.  For other columns, check ``man sacct``.
 
   * ``ReqGPU``: Number of GPUs requested, from ``ReqTRES[gres/gpu]``.
 
-  * ``GpuMem``: From ``TRESUsageInAve[gres/gpumem]``
+  * ``GpuMem``: From ``TRESUsageInTot[gres/gpumem]``
 
-  * ``GpuUtil``: From ``TRESUsageInAve[gres/gpuutil]`` (normalized to
+  * ``GpuUtil``: From ``TRESUsageInTot[gres/gpuutil]`` (normalized to
     fraction 0.0-ngpus).
 
   * ``NGpus``: Number of GPUs from ``AllocTRES[gres/gpu]``.
     Should be the same as ``ReqGPU``, but who knows.
 
-  * ``GpuUtilTot``, ``GpuMemTot``: like above but using the
-    ``TRESUsageInTot`` sacct field.
+  * ``GpuUtilTot``, ``GpuMemTot``: Same as the columns above (before
+    the above columns were made with ``TRESUsageInAve``, these columns
+    saved for the time being for backwards compatibility).
 
   * ``GpuEff``: From ``TRESUsageInTot[gres/gpuutil]``) / (100 *
     ``AllocTRES[gres/gpu]``).

slurm2sql.py

@@ -668,8 +668,8 @@ def calc(row):
     #'_NGPU': slurmGPUCount,             # Number of GPUs, extracted from comment field
     '_NGpus': ExtractField('NGpus', 'AllocTRES', 'gres/gpu', float_metric),
     '_GpuType': slurmGPUType,            # gres/gpu:TYPE= from AllocTres
-    '_GpuUtil': ExtractField('GpuUtil', 'TRESUsageInAve', 'gres/gpuutil', float_metric, wrap=lambda x: x/100.), # can be >100 for multi-GPU.
-    '_GpuMem': ExtractField('GpuMem2', 'TRESUsageInAve', 'gres/gpumem', float_metric),
+    '_GpuUtil': ExtractField('GpuUtil', 'TRESUsageInTot', 'gres/gpuutil', float_metric, wrap=lambda x: x/100.), # can be >100 for multi-GPU.
+    '_GpuMem': ExtractField('GpuMem2', 'TRESUsageInTot', 'gres/gpumem', float_metric),
     '_GpuUtilTot': ExtractField('GpuUtilTot', 'TRESUsageInTot', 'gres/gpuutil', float_metric),
     '_GpuMemTot': ExtractField('GpuMemTot',   'TRESUsageInTot', 'gres/gpumem', float_metric),
     }
@@ -940,7 +940,7 @@ def infer_type(cd):
                'ReqTRES, '
                'max(Elapsed) AS Elapsed, '
                'max(NCPUS) AS NCPUS, '
-               'sum(totalcpu)/max(cputime) AS CPUeff, '  # highest TotalCPU is for the whole allocation
+               'sum(totalcpu)/max(cputime) AS CPUeff, '
                'max(cputime) AS cpu_s_reserved, '
                'sum(totalcpu) AS cpu_s_used, '
                'max(ReqMemNode) AS MemReq, '
@@ -952,10 +952,9 @@ def infer_type(cd):
                'max(NGpus) AS NGpus, '
                'max(GPUType) AS GPUType, '
                'max(NGpus)*max(Elapsed) AS gpu_s_reserved, '
-               'max(NGpus)*max(Elapsed)*max(GPUutil) AS gpu_s_used, '
-               #'max(GPUutil)/max(NGpus) AS GPUeff, '               # Individual job with highest use (check this)
-               'max(GPUEff) AS GPUeff, '               # Individual job with highest use (check this)
-               'max(GPUMem) AS GPUMem, '
+               'max(NGpus)*max(Elapsed)*max(GpuUtil) AS gpu_s_used, '
+               'sum(GpuUtil*Elapsed)/max(Ngpus*Elapsed) AS GpuEff, '
+               'max(GpuMem) AS GpuMem, '
                'MaxDiskRead, '
                'MaxDiskWrite, '
                'sum(TotDiskRead) as TotDiskRead, '

test.py

@@ -77,7 +77,7 @@ def data3(slurm_version_2011):
 
 def csvdata(data):
     """Convert string CSV to a reader for s2s"""
-    reader = csv.DictReader(StringIO(data.strip()))
+    reader = csv.DictReader(StringIO(data.strip()), skipinitialspace=True)
     return reader
 
 def fetch(db, jobid, field, table='slurm'):
@@ -192,8 +192,8 @@ def test_memeff(db):
 
 def test_gpueff(db):
     data = """
-    JobID,AllocTRES,TRESUsageInTot
-    1,gres/gpu=1,gres/gpuutil=23
+    JobID,Elapsed,AllocTRES,  TRESUsageInTot
+    1,    1:00,   gres/gpu=1, gres/gpuutil=23
     """
     slurm2sql.slurm2sql(db, [], csv_input=csvdata(data))
     print(db.execute('select * from eff;').fetchall())