Fix CPU efficiency calculation (TotalCPU is now empty for job steps)

- In our slurm (25.05), it seems that TotalCPU is now empty anytime
  `srun` is used.  This gave a lot of misleading data, but the
  necessary info to compute this is in TRESUsageInTot[cpu].  So now,
  we calculate:
- TotalCPU: from TRESUsageInTot[cpu]
- CPUEff: TRESUsageInTot[cpu] / (AllocTRES[cpu]*Elapsed)
- Adjustments to the `eff` view to use this
- There are some disadvantages, for example we can't get CPU
  efficiency from the allocation steps anymore.  But this seems to be
  how it works now (sort of like how memory allocation is reported
  now).

Author: rkdarst

README.rst

@@ -201,6 +201,9 @@ percentages, and *unixtime*.
 Columns which are the same in raw ``sacct`` output aren't documented
 specifically here (but note the default units above).
 
+The syntax ``ColumnName[name]`` means the ``name=`` value from the
+given column name.
+
 Below are some notable columns which do not exist in sacct (for the
 rest, check out the `sacct manual page <https://slurm.schedmd.com/sacct.html#lbAF>`_).  It's good
 to verify that any of our custom columns make sense before trusting
@@ -252,13 +255,23 @@ them.  For other columns, check ``man sacct``.
     stripped out and give invalid data.  File an issue and this will
     be added.
 
+* **CPU related**
+
+  * ``TotalCPU`` is the total CPU seconds used (time × number of
+    CPUs).  In our latest slurm, the raw column is zero for job steps,
+    so this is now extracted from ``TRESUsageInTot[cpu]``
+
+  * ``CPUEff``: CPU efficiency (0.0-1.0).  All the same caveats as above
+    apply: test before trusting.  This is calculated as
+    ``TRESUsageInTot[cpu]/(AllocTRES[cpu]*Elapsed)``.
+
 * **Memory related**
 
-  * ``AllocMem``: The ``mem=`` value from ``AllocTRES`` field.  You
-    probably want to use this.
+  * ``AllocMem``: The ``AllocTRES[mem]`` field.  You probably want to
+    use this.
 
-  * ``TotalMem``: The ``mem=`` value from ``TRESUsageInTot`` field.
-    You probably want to use this.
+  * ``TotalMem``: The ``TRESUsageInTot[mem]`` field.  You probably
+    want to use this.
 
   * ``ReqMem``: The raw slurm value from the ReqMem column.
 
@@ -268,25 +281,23 @@ them.  For other columns, check ``man sacct``.
   * ``MemEff``: Computed ``TotalMem / AllocMem``.
 
 * **GPU information.**  These use values from the ``TRESUsageInAve``
-  fields in modern Slurm
+  fields in modern Slurm:
 
-  * ``ReqGPU``: Number of GPUs requested.  Extracted from ``ReqTRES``.
+  * ``ReqGPU``: Number of GPUs requested, from ``ReqTRES[gres/gpu]``.
 
-  * ``GpuMem``: ``gres/gpumem`` from ``TRESUsageInAve``
+  * ``GpuMem``: From ``TRESUsageInAve[gres/gpumem]``
 
-  * ``GpuUtil``: ``gres/gpuutil`` (fraction 0.0-1.0).
+  * ``GpuUtil``: From ``TRESUsageInAve[gres/gpuutil]`` (normalized to
+    fraction 0.0-ngpus).
 
-  * ``NGpus``: Number of GPUs from ``gres/gpu`` in ``AllocTRES``.
+  * ``NGpus``: Number of GPUs from ``AllocTRES[gres/gpu]``.
     Should be the same as ``ReqGPU``, but who knows.
 
   * ``GpuUtilTot``, ``GpuMemTot``: like above but using the
     ``TRESUsageInTot`` sacct field.
 
-  * ``GpuEff``: ``gres/gpuutil`` (from ``TRESUsageInTot``) / (100 *
-    ``gres/gpu`` (from ``AllocTRES``).
-
-* ``CPUEff``: CPU efficiency (0.0-1.0).  All the same caveats as above
-  apply: test before trusting.
+  * ``GpuEff``: From ``TRESUsageInTot[gres/gpuutil]``) / (100 *
+    ``AllocTRES[gres/gpu]``).
 
 * And more, see the code for now.
 
@@ -298,11 +309,9 @@ accounting database that are hardest to remember:
 * ``CPUTime``: Reserved CPU time (Elapsed * number of CPUs).  CPUEff ≈
   TotalCPU/CPUTime = TotalCPU/(NCPUs x Elapsed)
 
-* ``TotalCPU``: SystemCPU + TotalCPU, seconds of productive work.
-
 The ``eff`` table adds the following:
 
-* ``CPUEff``: Highest CPUEff for any job step
+* ``CPUEff``: sum(cpu usage over steps) / allocated (cpu usage over steps)
 
 * ``MemEff``: Highest MemEff for any job step
 

slurm2sql.py

@@ -507,19 +507,26 @@ def calc(row):
             return float_bytes(m_used.group(1)) / alloc
         return None
 
-
+RE_TRES_CPU = re.compile(rf'\bcpu=([^,]*)\b')
 class slurmCPUEff(linefunc):
     # This matches the seff tool currently:
     # https://github.com/SchedMD/slurm/blob/master/contribs/seff/seff
+    # Update 2025-10-20: on our slurm TotalCPU is now empty, so we get the CPU value from TRESUsageInTot
     type = 'real'
     @staticmethod
     def calc(row):
-        if not ('Elapsed' in row and 'TotalCPU' in row and 'NCPUS' in row):
+        if not ('Elapsed' in row and 'TRESUsageInTot' in row):
             return
         walltime = slurmtime(row['Elapsed'])
         if not walltime: return None
+        m_cpu_alloc = RE_TRES_CPU.search(row['AllocTRES'])
+        if not m_cpu_alloc: return None
+        m_cpu_used = RE_TRES_CPU.search(row['TRESUsageInTot'])
+        if not m_cpu_used: return None
+        cpu_alloc = int_metric(m_cpu_alloc.group(1))
+        cpu_used = slurmtime(m_cpu_used.group(1))
         try:
-            cpueff = slurmtime(row['TotalCPU']) / (walltime * int(row['NCPUS']))
+            cpueff = cpu_used / (walltime * cpu_alloc)
         except ZeroDivisionError:
             return float('nan')
         return cpueff
@@ -621,7 +628,7 @@ def calc(row):
     'ReqCPUS': nullint,                 # Requested CPUs
     'AllocCPUS': nullint,               # === NCPUS
     'CPUTime': slurmtime,               # = Elapsed * NCPUS    (= CPUTimeRaw)  (not how much used)
-    'TotalCPU': slurmtime,              # = Elapsed * NCPUS * efficiency
+    'TotalCPU': ExtractField("TotalCPU", "TRESUsageInTot", "cpu", slurmtime),   # = Elapsed * NCPUS * efficiency
     'UserCPU': slurmtime,               #
     'SystemCPU': slurmtime,             #
     '_CPUEff': slurmCPUEff,             # CPU efficiency, should be same as seff
@@ -933,9 +940,9 @@ def infer_type(cd):
                'ReqTRES, '
                'max(Elapsed) AS Elapsed, '
                'max(NCPUS) AS NCPUS, '
-               'max(totalcpu)/max(cputime) AS CPUeff, '  # highest TotalCPU is for the whole allocation
+               'sum(totalcpu)/max(cputime) AS CPUeff, '  # highest TotalCPU is for the whole allocation
                'max(cputime) AS cpu_s_reserved, '
-               'max(totalcpu) AS cpu_s_used, '
+               'sum(totalcpu) AS cpu_s_used, '
                'max(ReqMemNode) AS MemReq, '
                'max(AllocMem) AS AllocMem, '
                'max(TotalMem) AS TotalMem, '

test.py

@@ -153,15 +153,31 @@ def test_queuetime(db, data1):
 #
 def test_cpueff(db):
     data = """
-    JobID,CPUTime,TotalCPU
-    1,50:00,25:00
+    JobID, CPUTime, TotalCPU, TRESUsageInTot
+    1,     50:00,   25:00,    cpu=00:25:00
     """
     slurm2sql.slurm2sql(db, [], csv_input=csvdata(data))
     print(db.execute('select * from eff;').fetchall())
     assert fetch(db, 1, 'CPUTime') == 3000
     assert fetch(db, 1, 'TotalCPU') == 1500
     assert fetch(db, 1, 'CPUeff', table='eff') == 0.5
 
+def test_cpueff_steps(db):
+    data = """
+    JobID,CPUTime,TotalCPU,TRESUsageInTot
+    1,   50:00, 02:00,
+    1.1, 25:00, 24:00, cpu=00:25:00
+    1.2, 25:00, 24:00, cpu=00:25:00
+    """
+    slurm2sql.slurm2sql(db, [], csv_input=csvdata(data))
+    print(db.execute('select * from eff;').fetchall())
+    #assert fetch(db, 1, 'CPUTime') == 3000
+    #assert fetch(db, 1, 'TotalCPU') == 1500
+    assert fetch(db, 1, 'CPUeff', table='eff') == 1.0
+    assert fetch(db, 1, 'cpu_s_reserved', table='eff') == 3000
+    assert fetch(db, 1, 'cpu_s_used', table='eff') == 3000
+
+
 def test_memeff(db):
     data = """
     JobID,AllocTRES,TRESUsageInTot
@@ -231,14 +247,14 @@ def test_sacct(db, capsys):
 #
 def test_seff(db, capsys):
     data = """
-    JobID,Start,End,CPUTime,TotalCPU
-    111,1970-01-01T00:00:00,1970-01-01T00:50:00,50:00,25:00
-    111.2,,,,25:00
+    JobID, Start,               End,                 Elapsed, CPUTime, TotalCPU,TRESUsageInTot
+    111,   1970-01-01T00:00:00, 1970-01-01T00:50:00, 50:00,   50:00,        ,
+    111.2,                    ,                    ,      ,        ,   25:00,   cpu=00:25:00
     """
     slurm2sql.seff_cli(argv=[], csv_input=csvdata(data))
-    captured = capsys.readouterr()
-    assert '111' in captured.out
-    assert '50%' in captured.out
+    captured = capsys.readouterr().out
+    assert '111' in captured
+    assert '50%' in captured
 
 def test_seff_mem(db, capsys):
     data = """