Cleanup simdhash failure handling

Author: kg

src/coreclr/interpreter/compiler.h

@@ -550,7 +550,9 @@ class InterpCompiler
     {
         if (!PointerInNameMap(ptr))
         {
-            dn_simdhash_ptr_ptr_try_add(m_pointerToNameMap.GetValue(), ptr, (void*)name);
+            dn_simdhash_add_result addResult = dn_simdhash_ptr_ptr_try_add(m_pointerToNameMap.GetValue(), ptr, (void*)name);
+            if (addResult < 0)
+                NOMEM();
         }
     }
     void PrintNameInPointerMap(void* ptr);
@@ -868,7 +870,19 @@ int32_t InterpDataItemIndexMap::GetDataItemIndexForT(const T& lookup)
     VarSizedDataWithPayload<T>* pLookup = new(hashItemPayload) VarSizedDataWithPayload<T>();
     memcpy(&pLookup->payload, &lookup, sizeof(T));
 
-    dn_simdhash_ght_insert(hash, (void*)pLookup, (void*)(size_t)dataItemIndex);
+    dn_simdhash_add_result addResult = dn_simdhash_ght_try_insert(
+        hash, (void*)pLookup, (void*)(size_t)dataItemIndex, DN_SIMDHASH_INSERT_MODE_ENSURE_UNIQUE
+    );
+    switch (addResult) {
+        case DN_SIMDHASH_OUT_OF_MEMORY:
+            NOMEM();
+            break;
+        case DN_SIMDHASH_ADD_OVERWROTE:
+        case DN_SIMDHASH_INTERNAL_ERROR:
+            assert(!"Internal error in dn_simdhash");
+            break;
+    }
+
     return dataItemIndex;
 }
 

src/coreclr/interpreter/stackmap.cpp

@@ -30,9 +30,10 @@ InterpreterStackMap* GetInterpreterStackMap(ICorJitInfo* jitInfo, CORINFO_CLASS_
     {
         result = new InterpreterStackMap(jitInfo, classHandle);
         dn_simdhash_add_result addResult = dn_simdhash_ptr_ptr_try_add(t_sharedStackMapLookup, classHandle, result);
-        assert(addResult == DN_SIMDHASH_ADD_INSERTED);
         if (addResult == DN_SIMDHASH_OUT_OF_MEMORY)
             NOMEM();
+        else
+            assert(addResult == DN_SIMDHASH_ADD_INSERTED);
     }
     return result;
 }