[ILASM] Add support for deterministic builds and PDB checksums (#109091)

Follow-up to #85344. Fixes #8293. Fixes #62484.

Author: amanasifkhalid

src/coreclr/dlls/mscorpe/ceefilegenwriter.cpp

@@ -482,6 +482,11 @@ HRESULT CeeFileGenWriter::getFileTimeStamp(DWORD *pTimeStamp)
     return getPEWriter().getFileTimeStamp(pTimeStamp);
 } // HRESULT CeeFileGenWriter::getFileTimeStamp()
 
+void CeeFileGenWriter::setFileHeaderTimeStamp(DWORD timeStamp)
+{
+    return getPEWriter().setFileHeaderTimeStamp(timeStamp);
+} // void CeeFileGenWriter::setFileHeaderTimeStamp()
+
 HRESULT CeeFileGenWriter::setAddrReloc(UCHAR *instrAddr, DWORD value)
 {
     *(DWORD *)instrAddr = VAL32(value);

src/coreclr/dlls/mscorpe/iceefilegen.cpp

@@ -446,7 +446,7 @@ HRESULT ICeeFileGen::SetVTableEntry64(HCEEFILE ceeFile, ULONG size, void* ptr)
     return gen->setVTableEntry64(size, ptr);
 }
 
-HRESULT ICeeFileGen::GetFileTimeStamp (HCEEFILE ceeFile, DWORD *pTimeStamp)
+HRESULT ICeeFileGen::GetFileTimeStamp(HCEEFILE ceeFile, DWORD *pTimeStamp)
 {
     TESTANDRETURNPOINTER(ceeFile);
     TESTANDRETURNPOINTER(pTimeStamp);
@@ -455,3 +455,12 @@ HRESULT ICeeFileGen::GetFileTimeStamp (HCEEFILE ceeFile, DWORD *pTimeStamp)
     return(gen->getFileTimeStamp(pTimeStamp));
 }
 
+HRESULT ICeeFileGen::SetFileHeaderTimeStamp(HCEEFILE ceeFile, DWORD timeStamp)
+{
+    TESTANDRETURNPOINTER(ceeFile);
+
+    CeeFileGenWriter *gen = reinterpret_cast<CeeFileGenWriter*>(ceeFile);
+    gen->setFileHeaderTimeStamp(timeStamp);
+    return S_OK;
+}
+

src/coreclr/dlls/mscorpe/pewriter.cpp

@@ -1637,6 +1637,11 @@ HRESULT PEWriter::getFileTimeStamp(DWORD *pTimeStamp)
     return S_OK;
 }
 
+void PEWriter::setFileHeaderTimeStamp(DWORD timeStamp)
+{
+    m_ntHeaders->FileHeader.TimeDateStamp = timeStamp;
+}
+
 DWORD PEWriter::getImageBase32()
 {
     _ASSERTE(isPE32());

src/coreclr/dlls/mscorpe/pewriter.h

@@ -83,6 +83,7 @@ class PEWriter : public PESectionMan
     size_t getImageBase();
 
     HRESULT getFileTimeStamp(DWORD *pTimeStamp);
+    void setFileHeaderTimeStamp(DWORD timeStamp);
 
     IMAGE_NT_HEADERS32* ntHeaders32()    { return (IMAGE_NT_HEADERS32*) m_ntHeaders; }
     IMAGE_NT_HEADERS64* ntHeaders64()    { return (IMAGE_NT_HEADERS64*) m_ntHeaders; }

src/coreclr/ilasm/assem.cpp

@@ -15,6 +15,10 @@
 
 #include "assembler.h"
 
+#if !defined(_WIN32) && !defined(__APPLE__)
+#include "sha256.h"
+#endif
+
 void indexKeywords(Indx* indx); // defined in asmparse.y
 
 unsigned int g_uCodePage = CP_ACP;
@@ -28,6 +32,7 @@ Assembler::Assembler()
 {
     m_pDisp = NULL;
     m_pEmitter = NULL;
+    m_pInternalEmitForDeterministicMvid = NULL;
     m_pImporter = NULL;
 
     char* pszFQN = new char[16];
@@ -107,6 +112,7 @@ Assembler::Assembler()
     m_fGeneratePDB = FALSE;
     m_fIsMscorlib = FALSE;
     m_fOptimize = FALSE;
+    m_fDeterministic = FALSE;
     m_tkSysObject = 0;
     m_tkSysString = 0;
     m_tkSysValue = 0;
@@ -208,6 +214,11 @@ Assembler::~Assembler()
         m_pEmitter->Release();
         m_pEmitter = NULL;
     }
+    if (m_pInternalEmitForDeterministicMvid != NULL)
+    {
+        m_pInternalEmitForDeterministicMvid->Release();
+        m_pInternalEmitForDeterministicMvid = NULL;
+    }
     if (m_pPortablePdbWriter != NULL)
     {
         delete m_pPortablePdbWriter;
@@ -233,13 +244,19 @@ BOOL Assembler::Init(BOOL generatePdb)
     }
 
     if (FAILED(CreateICeeFileGen(&m_pCeeFileGen))) return FALSE;
-
     if (FAILED(m_pCeeFileGen->CreateCeeFileEx(&m_pCeeFile,(ULONG)m_dwCeeFileFlags))) return FALSE;
-
     if (FAILED(m_pCeeFileGen->GetSectionCreate(m_pCeeFile, ".il", sdReadOnly, &m_pILSection))) return FALSE;
     if (FAILED(m_pCeeFileGen->GetSectionCreate (m_pCeeFile, ".sdata", sdReadWrite, &m_pGlobalDataSection))) return FALSE;
     if (FAILED(m_pCeeFileGen->GetSectionCreate (m_pCeeFile, ".tls", sdReadWrite, &m_pTLSSection))) return FALSE;
 
+#if !defined(_WIN32) && !defined(__APPLE__)
+    if (m_fDeterministic && !IsOpenSslAvailable())
+    {
+        fprintf(stderr, "OpenSSL is not available, but required for build determinism\n");
+        return FALSE;
+    }
+#endif
+
     m_fGeneratePDB = generatePdb;
 
     return TRUE;

src/coreclr/ilasm/assembler.cpp

@@ -2434,12 +2434,9 @@ void Assembler::SetPdbFileName(_In_ __nullterminated char* szName)
 HRESULT Assembler::SavePdbFile()
 {
     HRESULT hr = S_OK;
-    mdMethodDef entryPoint;
 
     if (FAILED(hr = (m_pPortablePdbWriter == NULL ? E_FAIL : S_OK))) goto exit;
     if (FAILED(hr = (m_pPortablePdbWriter->GetEmitter() == NULL ? E_FAIL : S_OK))) goto exit;
-    if (FAILED(hr = m_pCeeFileGen->GetEntryPoint(m_pCeeFile, &entryPoint))) goto exit;
-    if (FAILED(hr = m_pPortablePdbWriter->BuildPdbStream(m_pEmitter, entryPoint))) goto exit;
     if (FAILED(hr = m_pPortablePdbWriter->GetEmitter()->Save(m_wzPdbFileName, 0))) goto exit;
 
 exit:

src/coreclr/ilasm/assembler.h

@@ -752,6 +752,7 @@ class Assembler {
     BOOL    m_fIsMscorlib;
     BOOL    m_fTolerateDupMethods;
     BOOL    m_fOptimize;
+    BOOL    m_fDeterministic;
     mdToken m_tkSysObject;
     mdToken m_tkSysString;
     mdToken m_tkSysValue;
@@ -760,6 +761,7 @@ class Assembler {
 
     IMetaDataDispenserEx2 *m_pDisp;
     IMetaDataEmit3      *m_pEmitter;
+    IMDInternalEmit     *m_pInternalEmitForDeterministicMvid;
     ICeeFileGen        *m_pCeeFileGen;
     IMetaDataImport2    *m_pImporter;			// Import interface.
     HCEEFILE m_pCeeFile;
@@ -845,7 +847,7 @@ class Assembler {
     BOOL EmitClass(Class *pClass);
     HRESULT CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename);
     HRESULT CreateTLSDirectory();
-    HRESULT CreateDebugDirectory();
+    HRESULT CreateDebugDirectory(BYTE(&pdbChecksum)[32]);
     HRESULT InitMetaData();
     Class *FindCreateClass(_In_ __nullterminated const char *pszFQN);
     BOOL EmitFieldRef(_In_z_ char *pszArg, int opcode);

src/coreclr/ilasm/main.cpp

@@ -180,6 +180,7 @@ extern "C" int _cdecl wmain(int argc, _In_ WCHAR **argv)
       printf("\n/DEBUG          Disable JIT optimization, create PDB file, use sequence points from PDB");
       printf("\n/DEBUG=IMPL     Disable JIT optimization, create PDB file, use implicit sequence points");
       printf("\n/DEBUG=OPT      Enable JIT optimization, create PDB file, use implicit sequence points");
+      printf("\n/DET            Produce deterministic outputs");
       printf("\n/OPTIMIZE       Optimize long instructions to short");
       printf("\n/FOLD           Fold the identical method bodies into one");
       printf("\n/CLOCK          Measure and report compilation times");
@@ -316,6 +317,10 @@ extern "C" int _cdecl wmain(int argc, _In_ WCHAR **argv)
                     {
                       pAsm->m_fOptimize = TRUE;
                     }
+                    else if (!_stricmp(szOpt, "DET"))
+                    {
+                      pAsm->m_fDeterministic = TRUE;
+                    }
                     else if (!_stricmp(szOpt, "X64"))
                     {
                       pAsm->m_dwCeeFileFlags &= ~ICEE_CREATE_MACHINE_MASK;

src/coreclr/ilasm/portable_pdb.cpp

@@ -4,6 +4,7 @@
 #include "portable_pdb.h"
 #include <time.h>
 #include "assembler.h"
+#include "sha256.h"
 
 //*****************************************************************************
 // Document
@@ -103,6 +104,10 @@ HRESULT PortablePdbWriter::Init(IMetaDataDispenserEx2* mdDispenser)
         0,
         IID_IMetaDataEmit3,
         (IUnknown**)&m_pdbEmitter);
+
+    if (FAILED(hr)) goto exit;
+
+    hr = m_pdbEmitter->QueryInterface(IID_IILAsmPortablePdbWriter, (void**)&m_ilasmPdbWriter);
 exit:
     return hr;
 }
@@ -122,6 +127,16 @@ ULONG PortablePdbWriter::GetTimestamp()
     return m_pdbStream.id.pdbTimeStamp;
 }
 
+void PortablePdbWriter::SetGuid(REFGUID newGuid)
+{
+    m_pdbStream.id.pdbGuid = newGuid;
+}
+
+void PortablePdbWriter::SetTimestamp(const ULONG newTimestamp)
+{
+    m_pdbStream.id.pdbTimeStamp = newTimestamp;
+}
+
 Document* PortablePdbWriter::GetCurrentDocument()
 {
     return m_currentDocument;
@@ -145,6 +160,17 @@ HRESULT PortablePdbWriter::BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef
     return hr;
 }
 
+HRESULT PortablePdbWriter::ComputeSha256PdbStreamChecksum(BYTE(&checksum)[32])
+{
+    return m_ilasmPdbWriter->ComputeSha256PdbStreamChecksum(Sha256Hash, checksum);
+}
+
+HRESULT PortablePdbWriter::ChangePdbStreamGuid(REFGUID newGuid)
+{
+    m_pdbStream.id.pdbGuid = newGuid;
+    return m_ilasmPdbWriter->ChangePdbStreamGuid(newGuid);
+}
+
 HRESULT PortablePdbWriter::DefineDocument(char* name, GUID* language)
 {
     HRESULT hr = S_OK;

src/coreclr/ilasm/portable_pdb.h

@@ -46,8 +46,12 @@ class PortablePdbWriter
     IMetaDataEmit3* GetEmitter();
     GUID*           GetGuid();
     ULONG           GetTimestamp();
+    void            SetGuid(REFGUID newGuid);
+    void            SetTimestamp(const ULONG newTimestamp);
     Document*       GetCurrentDocument();
     HRESULT         BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef entryPoint);
+    HRESULT         ComputeSha256PdbStreamChecksum(BYTE (&checksum)[32]);
+    HRESULT         ChangePdbStreamGuid(REFGUID newGuid);
     HRESULT         DefineDocument(char* name, GUID* language);
     HRESULT         DefineSequencePoints(Method* method);
     HRESULT         DefineLocalScope(Method* method);
@@ -59,10 +63,11 @@ class PortablePdbWriter
     BOOL            _DefineLocalScope(mdMethodDef methodDefToken, Scope* currScope);
 
 private:
-    IMetaDataEmit3* m_pdbEmitter;
-    PORT_PDB_STREAM m_pdbStream;
-    DocumentList    m_documentList;
-    Document*       m_currentDocument;
+    IMetaDataEmit3*          m_pdbEmitter;
+    IILAsmPortablePdbWriter* m_ilasmPdbWriter;
+    PORT_PDB_STREAM          m_pdbStream;
+    DocumentList             m_documentList;
+    Document*                m_currentDocument;
 };
 
 #endif