More detailed statistics for matrices

Author: janniklinde

src/main/java/org/apache/sysds/runtime/controlprogram/ProgramBlock.java

@@ -275,6 +275,12 @@ private void executeSingleInstruction(Instruction currInst, ExecutionContext ec)
 							if (data != null) {
 								ext.setDataType(data.getDataType().toString());
 								ext.setValueType(data.getValueType().toString());
+								if (data instanceof MatrixObject) {
+									MatrixObject m = (MatrixObject)data;
+									ext.setMeta("NumRows", (double)m.getNumRows());
+									ext.setMeta("NumCols", (double)m.getNumColumns());
+									ext.setMeta("NNZ", (double)m.getNnz());
+								}
 							}
 							ext.setExecNanos(nanoTime);
 							Statistics.extendLineageItem(li.getValue(), ext);

src/main/java/org/apache/sysds/utils/Statistics.java

@@ -78,6 +78,7 @@ public static class NGramStats {
 		public final long n;
 		public final long cumTimeNanos;
 		public final double m2;
+		public final HashMap<String, Double> meta;
 
 		public static <T> Comparator<NGramBuilder.NGramEntry<T, NGramStats>> getComparator() {
 			return Comparator.comparingLong(entry -> entry.getCumStats().cumTimeNanos);
@@ -95,13 +96,25 @@ public static NGramStats merge(NGramStats stats1, NGramStats stats2) {
 
 			double newM2 = stats1.m2 + stats2.m2 + delta * delta * stats1.n * stats2.n / (double)newN;
 
-			return new NGramStats(newN, cumTimeNanos, newM2);
+			HashMap<String, Double> cpy = null;
+
+			if (stats1.meta != null) {
+				cpy = new HashMap<>(stats1.meta);
+				final HashMap<String, Double> mCpy = cpy;
+				if (stats2.meta != null)
+					stats2.meta.forEach((key, value) -> mCpy.merge(key, value, Double::sum));
+			} else if (stats2.meta != null) {
+				cpy = new HashMap<>(stats2.meta);
+			}
+
+			return new NGramStats(newN, cumTimeNanos, newM2, cpy);
 		}
 
-		public NGramStats(final long n, final long cumTimeNanos, final double m2) {
+		public NGramStats(final long n, final long cumTimeNanos, final double m2, HashMap<String, Double> meta) {
 			this.n = n;
 			this.cumTimeNanos = cumTimeNanos;
 			this.m2 = m2;
+			this.meta = meta;
 		}
 
 		public double getTimeVariance() {
@@ -111,13 +124,19 @@ public double getTimeVariance() {
 		public String toString() {
 			return String.format(Locale.US, "%.5f", (cumTimeNanos / 1000000000d));
 		}
+
+		public HashMap<String, Double> getMeta() {
+			return meta;
+		}
 	}
 
 	public static class LineageNGramExtension {
 		private String _datatype;
 		private String _valuetype;
 		private long _execNanos;
 
+		private HashMap<String, Double> _meta;
+
 		public void setDataType(String dataType) {
 			_datatype = dataType;
 		}
@@ -141,6 +160,18 @@ public void setExecNanos(long nanos) {
 		public long getExecNanos() {
 			return _execNanos;
 		}
+
+		public void setMeta(String key, Double value) {
+			if (_meta == null)
+				_meta = new HashMap<>();
+			_meta.put(key, value);
+		}
+
+		public Object getMeta(String key) {
+			if (_meta == null)
+				return null;
+			return _meta.get(key);
+		}
 	}
 
 	private static long compileStartTime = 0;
@@ -508,10 +539,10 @@ private static void addLineagePaths(LineageItem li, ArrayList<Entry<LineageItem,
 			clearNGramRecording();
 			// We then record a new n-gram with all the LineageItems of the current lineage path
 			Entry<LineageItem, LineageNGramExtension> currentEntry = currentPath.get(currentPath.size()-1);
-			matchingBuilder.append(LineageItemUtils.explainLineageAsInstruction(currentEntry.getKey(), currentEntry.getValue()) + (indexes.size() > 0 ? ("[" + indexes.get(currentPath.size()-2) + "]") : ""), new NGramStats(1, currentEntry.getValue() != null ? currentEntry.getValue().getExecNanos() : 0, 0));
+			matchingBuilder.append(LineageItemUtils.explainLineageAsInstruction(currentEntry.getKey(), currentEntry.getValue()) + (indexes.size() > 0 ? ("[" + indexes.get(currentPath.size()-2) + "]") : ""), new NGramStats(1, currentEntry.getValue() != null ? currentEntry.getValue().getExecNanos() : 0, 0, currentEntry.getValue() != null ? currentEntry.getValue()._meta : null));
 			for (int i = currentPath.size()-2; i >= 0; i--) {
 				currentEntry = currentPath.get(i);
-				matchingBuilder.append(LineageItemUtils.explainLineageAsInstruction(currentEntry.getKey(), currentEntry.getValue()) + (i > 0 ? ("[" + indexes.get(i-1) + "]") : ""), new NGramStats(1, currentEntry.getValue() != null ? currentEntry.getValue().getExecNanos() : 0, 0));
+				matchingBuilder.append(LineageItemUtils.explainLineageAsInstruction(currentEntry.getKey(), currentEntry.getValue()) + (i > 0 ? ("[" + indexes.get(i-1) + "]") : ""), new NGramStats(1, currentEntry.getValue() != null ? currentEntry.getValue().getExecNanos() : 0, 0, currentEntry.getValue() != null ? currentEntry.getValue()._meta : null));
 			}
 		}
 
@@ -538,7 +569,7 @@ public static void maintainNGrams(String instName, long timeNanos) {
 		});
 
 		for (int i = 0; i < tmp.length; i++)
-			tmp[i].append(instName, new NGramStats(1, timeNanos, 0));
+			tmp[i].append(instName, new NGramStats(1, timeNanos, 0, null));
 	}
 
 	@SuppressWarnings("unchecked")
@@ -604,6 +635,8 @@ public static String nGramToCSV(final NGramBuilder<String, NGramStats> mbuilder)
 			colList.add("Col" + (j + 1) + "::Mean(Time[s])");
 		for (int j = 0; j < mbuilder.getSize(); j++)
 			colList.add("Col" + (j + 1) + "::StdDev(Time[s])/Col" + (j + 1) + "::Mean(Time[s])");
+		for (int j = 0; j < mbuilder.getSize(); j++)
+			colList.add("Col" + (j + 1) + "_Meta");
 
 		colList.add("Count");
 
@@ -620,6 +653,18 @@ public static String nGramToCSV(final NGramBuilder<String, NGramStats> mbuilder)
 			} else {
 				builder.append(stdDevs);
 			}
+			builder.append(",");
+			if (e.getCumStats().getMeta() != null) {
+				boolean first = true;
+				for (Entry<String, Double> metaData : e.getCumStats().getMeta().entrySet()) {
+					if (first)
+						first = false;
+					else
+						builder.append("&");
+					if (metaData.getValue() != null)
+						builder.append(metaData.getKey() + ":" + metaData.getValue());
+				}
+			}
 			return builder.toString();
 		});
 	}