diff --git a/src/coreclr/inc/crosscomp.h b/src/coreclr/inc/crosscomp.h index d5e4ca3004b8db..aeb061ca5ba479 100644 --- a/src/coreclr/inc/crosscomp.h +++ b/src/coreclr/inc/crosscomp.h @@ -282,7 +282,8 @@ typedef struct DECLSPEC_ALIGN(16) _T_CONTEXT { } T_CONTEXT, *PT_CONTEXT; // _IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY (see ExternalAPIs\Win9CoreSystem\inc\winnt.h) -typedef struct _T_RUNTIME_FUNCTION { +#ifdef HOST_UNIX +typedef struct _IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY { DWORD BeginAddress; union { DWORD UnwindData; @@ -294,12 +295,11 @@ typedef struct _T_RUNTIME_FUNCTION { DWORD H : 1; DWORD CR : 2; DWORD FrameSize : 9; - } PackedUnwindData; + }; }; -} T_RUNTIME_FUNCTION, *PT_RUNTIME_FUNCTION; +} IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY, * PIMAGE_ARM64_RUNTIME_FUNCTION_ENTRY; -#ifdef HOST_UNIX typedef EXCEPTION_DISPOSITION @@ -310,6 +310,8 @@ EXCEPTION_DISPOSITION PVOID DispatcherContext ); #endif + +typedef IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY T_RUNTIME_FUNCTION, * PT_RUNTIME_FUNCTION; // // Define exception dispatch context structure. // diff --git a/src/coreclr/pal/inc/pal.h b/src/coreclr/pal/inc/pal.h index 2bf5b17344d3de..8e9c70d33ec788 100644 --- a/src/coreclr/pal/inc/pal.h +++ b/src/coreclr/pal/inc/pal.h @@ -2085,6 +2085,34 @@ typedef struct _KNONVOLATILE_CONTEXT_POINTERS { } KNONVOLATILE_CONTEXT_POINTERS, *PKNONVOLATILE_CONTEXT_POINTERS; +typedef struct _IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY { + DWORD BeginAddress; + union { + DWORD UnwindData; + struct { + DWORD Flag : 2; + DWORD FunctionLength : 11; + DWORD RegF : 3; + DWORD RegI : 4; + DWORD H : 1; + DWORD CR : 2; + DWORD FrameSize : 9; + }; + }; +} IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY, * PIMAGE_ARM64_RUNTIME_FUNCTION_ENTRY; + +typedef union IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY_XDATA { + ULONG HeaderData; + struct { + ULONG FunctionLength : 18; // in words (2 bytes) + ULONG Version : 2; + ULONG ExceptionDataPresent : 1; + ULONG EpilogInHeader : 1; + ULONG EpilogCount : 5; // number of epilogs or byte index of the first unwind code for the one only epilog + ULONG CodeWords : 5; // number of dwords with unwind codes + }; +} IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY_XDATA; + #elif defined(HOST_LOONGARCH64) // Please refer to src/coreclr/pal/src/arch/loongarch64/asmconstants.h @@ -3139,13 +3167,17 @@ enum { // // A function table entry is generated for each frame function. // +#if defined(HOST_ARM64) +typedef IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY RUNTIME_FUNCTION, *PRUNTIME_FUNCTION; +#else // HOST_ARM64 typedef struct _RUNTIME_FUNCTION { DWORD BeginAddress; -#ifdef TARGET_AMD64 +#ifdef HOST_AMD64 DWORD EndAddress; #endif DWORD UnwindData; } RUNTIME_FUNCTION, *PRUNTIME_FUNCTION; +#endif // HOST_ARM64 #define STANDARD_RIGHTS_REQUIRED (0x000F0000L) #define SYNCHRONIZE (0x00100000L) diff --git a/src/coreclr/unwinder/arm64/unwinder.cpp b/src/coreclr/unwinder/arm64/unwinder.cpp index 6a4c25ef776984..8096204dffa6e7 100644 --- a/src/coreclr/unwinder/arm64/unwinder.cpp +++ b/src/coreclr/unwinder/arm64/unwinder.cpp @@ -9,6 +9,61 @@ #include "unwinder.h" +#define NOTHING + +#define ARM64_CONTEXT T_CONTEXT + +#ifndef HOST_ARM64 +#define CONTEXT T_CONTEXT +#define PCONTEXT PT_CONTEXT +#define KNONVOLATILE_CONTEXT_POINTERS T_KNONVOLATILE_CONTEXT_POINTERS +#define PKNONVOLATILE_CONTEXT_POINTERS PT_KNONVOLATILE_CONTEXT_POINTERS +#define RUNTIME_FUNCTION T_RUNTIME_FUNCTION +#define PRUNTIME_FUNCTION PT_RUNTIME_FUNCTION +#endif + +#ifndef __in +#define __in _In_ +#define __out _Out_ +#endif + +#ifndef FIELD_OFFSET +#define FIELD_OFFSET(type, field) ((LONG)__builtin_offsetof(type, field)) +#endif + +#ifdef HOST_UNIX +#define RtlZeroMemory ZeroMemory + +typedef enum ARM64_FNPDATA_FLAGS { + PdataRefToFullXdata = 0, + PdataPackedUnwindFunction = 1, + PdataPackedUnwindFragment = 2, +} ARM64_FNPDATA_FLAGS; + +typedef enum ARM64_FNPDATA_CR { + PdataCrUnchained = 0, + PdataCrUnchainedSavedLr = 1, + PdataCrChainedWithPac = 2, + PdataCrChained = 3, +} ARM64_FNPDATA_CR; + +#endif // HOST_UNIX + +// +// MessageId: STATUS_BAD_FUNCTION_TABLE +// +// MessageText: +// +// A malformed function table was encountered during an unwind operation. +// +#define STATUS_BAD_FUNCTION_TABLE ((NTSTATUS)0xC00000FFL) + +// +// Flags for RtlVirtualUnwind2. +// + +#define RTL_VIRTUAL_UNWIND2_VALIDATE_PAC 0x00000001UL + typedef struct _ARM64_KTRAP_FRAME { // @@ -90,6 +145,8 @@ typedef struct _ARM64_VFP_STATE NEON128 V[32]; // All V registers (0-31) } ARM64_VFP_STATE, *PARM64_VFP_STATE, KARM64_VFP_STATE, *PKARM64_VFP_STATE; +#define RTL_VIRTUAL_UNWIND_VALID_FLAGS_ARM64 (RTL_VIRTUAL_UNWIND2_VALIDATE_PAC) + // // Parameters describing the unwind codes. // @@ -101,46 +158,109 @@ typedef struct _ARM64_VFP_STATE // // Macros for accessing memory. These can be overridden if other code // (in particular the debugger) needs to use them. +// + +// +// Macros for accessing memory. These can be overridden if other code +// (in particular the debugger) needs to use them. + +#if !defined(DEBUGGER_UNWIND) #define MEMORY_READ_BYTE(params, addr) (*dac_cast(addr)) +#define MEMORY_READ_WORD(params, addr) (*dac_cast(addr)) #define MEMORY_READ_DWORD(params, addr) (*dac_cast(addr)) #define MEMORY_READ_QWORD(params, addr) (*dac_cast(addr)) +#endif + +// +// ARM64_UNWIND_PARAMS definition. This is the kernel-specific definition, +// and contains information on the original PC, the stack bounds, and +// a pointer to the non-volatile context pointer array. Any usage of +// these fields must be wrapped in a macro so that the debugger can take +// a direct drop of this code and use it. +// + +#if !defined(DEBUGGER_UNWIND) + typedef struct _ARM64_UNWIND_PARAMS { - PT_KNONVOLATILE_CONTEXT_POINTERS ContextPointers; + ULONG_PTR ControlPc; + PULONG_PTR LowLimit; + PULONG_PTR HighLimit; + PKNONVOLATILE_CONTEXT_POINTERS ContextPointers; } ARM64_UNWIND_PARAMS, *PARM64_UNWIND_PARAMS; #define UNWIND_PARAMS_SET_TRAP_FRAME(Params, Address, Size) -#define UPDATE_CONTEXT_POINTERS(Params, RegisterNumber, Address) \ -do { \ - if (ARGUMENT_PRESENT(Params)) { \ - PT_KNONVOLATILE_CONTEXT_POINTERS ContextPointers = (Params)->ContextPointers; \ - if (ARGUMENT_PRESENT(ContextPointers)) { \ - if (RegisterNumber >= 19 && RegisterNumber <= 30) { \ - (&ContextPointers->X19)[RegisterNumber - 19] = (PDWORD64)Address; \ - } \ - } \ - } \ +#if !defined(UPDATE_CONTEXT_POINTERS) +#define UPDATE_CONTEXT_POINTERS(Params, RegisterNumber, Address) \ +do { \ + PKNONVOLATILE_CONTEXT_POINTERS ContextPointers = (Params)->ContextPointers; \ + if (ARGUMENT_PRESENT(ContextPointers)) { \ + if (RegisterNumber >= 19 && RegisterNumber <= 28) { \ + (&ContextPointers->X19)[RegisterNumber - 19] = (PULONG64)Address; \ + } else if (RegisterNumber == 29) { \ + ContextPointers->Fp = (PULONG64)Address; \ + } else if (RegisterNumber == 30) { \ + ContextPointers->Lr = (PULONG64)Address; \ + } \ + } \ } while (0) +#endif // !defined(UPDATE_CONTEXT_POINTERS) + +#if !defined(UPDATE_FP_CONTEXT_POINTERS) +#define UPDATE_FP_CONTEXT_POINTERS(Params, RegisterNumber, Address) \ +do { \ + PKNONVOLATILE_CONTEXT_POINTERS ContextPointers = (Params)->ContextPointers; \ + if (ARGUMENT_PRESENT(ContextPointers) && \ + (RegisterNumber >= 8) && \ + (RegisterNumber <= 15)) { \ + \ + (&ContextPointers->D8)[RegisterNumber - 8] = (PULONG64)Address; \ + } \ +} while (0) +#endif // !defined(UPDATE_FP_CONTEXT_POINTERS) + +#if !defined(VALIDATE_STACK_ADDRESS_EX) +#define VALIDATE_STACK_ADDRESS_EX(Params, Context, Address, DataSize, Alignment, OutStatus) +#endif // !defined(VALIDATE_STACK_ADDRESS_EX) +#if !defined(VALIDATE_STACK_ADDRESS) +#define VALIDATE_STACK_ADDRESS(Params, Context, DataSize, Alignment, OutStatus) \ + VALIDATE_STACK_ADDRESS_EX(Params, Context, (Context)->Sp, DataSize, Alignment, OutStatus) +#endif // !defined(VALIDATE_STACK_ADDRESS) -#define UPDATE_FP_CONTEXT_POINTERS(Params, RegisterNumber, Address) \ -do { \ - if (ARGUMENT_PRESENT(Params)) { \ - PT_KNONVOLATILE_CONTEXT_POINTERS ContextPointers = (Params)->ContextPointers; \ - if (ARGUMENT_PRESENT(ContextPointers) && \ - (RegisterNumber >= 8) && \ - (RegisterNumber <= 15)) { \ - \ - (&ContextPointers->D8)[RegisterNumber - 8] = (PDWORD64)Address; \ - } \ - } \ -} while (0) +#else // !defined(DEBUGGER_UNWIND) + +#if !defined(UPDATE_CONTEXT_POINTERS) +#define UPDATE_CONTEXT_POINTERS(Params, RegisterNumber, Address) +#endif // !defined(UPDATE_CONTEXT_POINTERS) + +#if !defined(UPDATE_FP_CONTEXT_POINTERS) +#define UPDATE_FP_CONTEXT_POINTERS(Params, RegisterNumber, Address) +#endif // !defined(UPDATE_FP_CONTEXT_POINTERS) +#if !defined(VALIDATE_STACK_ADDRESS_EX) #define VALIDATE_STACK_ADDRESS_EX(Params, Context, Address, DataSize, Alignment, OutStatus) +#endif // !defined(VALIDATE_STACK_ADDRESS_EX) + +#if !defined(VALIDATE_STACK_ADDRESS) #define VALIDATE_STACK_ADDRESS(Params, Context, DataSize, Alignment, OutStatus) +#endif // !defined(VALIDATE_STACK_ADDRESS) + +#endif // !defined(DEBUGGER_UNWIND) + +// +// Macros for stripping pointer authentication (PAC) bits. +// + +#if !defined(DEBUGGER_STRIP_PAC) + +// NOTE: Pointer authentication is not used by .NET, so the implementation does nothing +#define STRIP_PAC(Params, pointer) + +#endif // // Macros to clarify opcode parsing @@ -149,336 +269,1024 @@ do { #define OPCODE_IS_END(Op) (((Op) & 0xfe) == 0xe4) // -// This table describes the size of each unwind code, in bytes +// This table describes the size of each unwind code, in bytes, for unwind codes +// in the range 0xE0-0xFF. // -static const BYTE UnwindCodeSizeTable[256] = +static const BYTE UnwindCodeSizeTable[32] = { - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, - 4,1,2,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1 + 4,1,2,1,1,1,1,3, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 2,3,4,5,1,1,1,1 }; -NTSTATUS -RtlpUnwindCustom( - __inout PT_CONTEXT ContextRecord, - _In_ BYTE Opcode, - _In_ PARM64_UNWIND_PARAMS UnwindParams - ) +// +// This table describes the number of instructions represented by each unwind +// code in the range 0xE0-0xFF. +// -/*++ +static const BYTE UnwindCodeInstructionCountTable[32] = +{ + 1,1,1,1,1,1,1,1, // 0xE0-0xE7 + 0, // 0xE8 - MSFT_OP_TRAP_FRAME + 0, // 0xE9 - MSFT_OP_MACHINE_FRAME + 0, // 0xEA - MSFT_OP_CONTEXT + 0, // 0xEB - MSFT_OP_EC_CONTEXT / MSFT_OP_RET_TO_GUEST (unused) + 0, // 0xEC - MSFT_OP_CLEAR_UNWOUND_TO_CALL + 0, // 0XED - MSFT_OP_RET_TO_GUEST_LEAF (unused) + 0,0, // 0xEE-0xEF + 0,0,0,0,0,0,0,0, // 0xF0-0xF7 + 1,1,1,1,1,1,1,1 // 0xF8-0xFF +}; -Routine Description: +#if !defined(ALIGN_DOWN_BY) - Handles custom unwinding operations involving machine-specific - frames. +#define ALIGN_DOWN_BY(length, alignment) \ + ((ULONG_PTR)(length) & ~((ULONG_PTR)(alignment) - 1)) -Arguments: +#endif - ContextRecord - Supplies the address of a context record. +#if !defined(ALIGN_UP_BY) - Opcode - The opcode to decode. +#define ALIGN_UP_BY(length, alignment) \ + (ALIGN_DOWN_BY(((ULONG_PTR)(length) + (alignment) - 1), alignment)) - UnwindParams - Additional parameters shared with caller. +#endif -Return Value: +#define OP_BUFFER_PRE_ADJUST(_sav_slot, _slots) {} +#define OP_BUFFER_POST_ADJUST(_sav_slot, _slots) {(_sav_slot) += (_slots);} - An NTSTATUS indicating either STATUS_SUCCESS if everything went ok, or - another status code if there were problems. +#define DBG_OP(...) ---*/ +#pragma warning(push) +#pragma warning(disable:4214) // bit field types other than int +#pragma warning(disable:4201) // nameless struct/union +#pragma warning(disable:4309) // truncation of constant value -{ - ULONG Fpcr; - ULONG Fpsr; - ULONG RegIndex; - ULONG_PTR SourceAddress; - ULONG_PTR StartingSp; - NTSTATUS Status; - ULONG_PTR VfpStateAddress; +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wbitfield-constant-conversion" +#endif - StartingSp = ContextRecord->Sp; - Status = STATUS_SUCCESS; +void emit_save_fplr(char** buff, LONG offset) { + union uop { + char val; + struct { + char z : 6; // pair at[sp + #Z * 8], offset <= 504 + char fixed : 2; + }; + }; - // - // The opcode describes the special-case stack - // + union uop *op; - switch (Opcode) - { + OP_BUFFER_PRE_ADJUST(*buff, 1); - // - // Trap frame case - // + offset = ((offset)/8); + op = (union uop*)(*buff); + op->fixed = 1; + op->z = (char)offset; - case 0xe8: // MSFT_OP_TRAP_FRAME: + OP_BUFFER_POST_ADJUST(*buff, 1); +} - // - // Ensure there is enough valid space for the trap frame - // +void emit_save_fplr_x(char** buff, LONG offset) { + union uop { + char val; + struct { + char z : 6; // pair at [sp-(#Z+1)*8]!, pre-indexed offset >= -512 + char fixed : 2; + }; + }; - VALIDATE_STACK_ADDRESS(UnwindParams, ContextRecord, sizeof(ARM64_KTRAP_FRAME), 16, &Status); - if (!NT_SUCCESS(Status)) { - return Status; - } + union uop* op; - // - // Restore X0-X17, and D0-D7 - // + OP_BUFFER_PRE_ADJUST(*buff, 1); - SourceAddress = StartingSp + offsetof(ARM64_KTRAP_FRAME, X); - for (RegIndex = 0; RegIndex < 18; RegIndex++) { - UPDATE_CONTEXT_POINTERS(UnwindParams, RegIndex, SourceAddress); -#ifdef __GNUC__ - *(&ContextRecord->X0 + RegIndex) = MEMORY_READ_QWORD(UnwindParams, SourceAddress); -#else - ContextRecord->X[RegIndex] = MEMORY_READ_QWORD(UnwindParams, SourceAddress); -#endif - SourceAddress += sizeof(ULONG_PTR); - } + offset = ((-offset)/8)-1; + op = (union uop*)(*buff); + op->fixed = 2; + op->z = (char)offset; - SourceAddress = StartingSp + offsetof(ARM64_KTRAP_FRAME, VfpState); - VfpStateAddress = MEMORY_READ_QWORD(UnwindParams, SourceAddress); - if (VfpStateAddress != 0) { + OP_BUFFER_POST_ADJUST(*buff, 1); +} - SourceAddress = VfpStateAddress + offsetof(KARM64_VFP_STATE, Fpcr); - Fpcr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); - SourceAddress = VfpStateAddress + offsetof(KARM64_VFP_STATE, Fpsr); - Fpsr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); - if (Fpcr != (ULONG)-1 && Fpsr != (ULONG)-1) { +void emit_save_regp(char** buff, LONG reg, LONG offset) { + union uop { + short val; + struct { + short z : 6; + short x : 4; // save r(19 + #X) pair at[sp + #Z * 8], offset <= 504 + short fixed : 6; + }; + }; - ContextRecord->Fpcr = Fpcr; - ContextRecord->Fpsr = Fpsr; + union uop* op; - SourceAddress = VfpStateAddress + offsetof(KARM64_VFP_STATE, V); - for (RegIndex = 0; RegIndex < 32; RegIndex++) { - UPDATE_FP_CONTEXT_POINTERS(UnwindParams, RegIndex, SourceAddress); - ContextRecord->V[RegIndex].Low = MEMORY_READ_QWORD(UnwindParams, SourceAddress); - ContextRecord->V[RegIndex].High = MEMORY_READ_QWORD(UnwindParams, SourceAddress + 8); - SourceAddress += 2 * sizeof(ULONGLONG); - } - } - } + OP_BUFFER_PRE_ADJUST(*buff, 2); - // - // Restore R11, R12, SP, LR, PC, and the status registers - // + offset = ((offset)/8); + op = (union uop*)(*buff); + op->fixed = 0x32; + op->x = (short)reg; + op->z = (short)offset; - SourceAddress = StartingSp + offsetof(ARM64_KTRAP_FRAME, Spsr); - ContextRecord->Cpsr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); + OP_BUFFER_POST_ADJUST(*buff, 2); +} - SourceAddress = StartingSp + offsetof(ARM64_KTRAP_FRAME, Sp); - ContextRecord->Sp = MEMORY_READ_QWORD(UnwindParams, SourceAddress); +void emit_save_regp_x(char** buff, LONG reg, LONG offset) { + union uop { + short val; + struct { + short z : 6; + short x : 4; // save pair r(19+#X) at [sp-(#Z+1)*8]!, pre-indexed offset >= -512 + short fixed : 6; + }; + }; - SourceAddress = StartingSp + offsetof(ARM64_KTRAP_FRAME, Lr); - ContextRecord->Lr = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + union uop* op; - SourceAddress = StartingSp + offsetof(ARM64_KTRAP_FRAME, Fp); - ContextRecord->Fp = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + OP_BUFFER_PRE_ADJUST(*buff, 2); - SourceAddress = StartingSp + offsetof(ARM64_KTRAP_FRAME, Pc); - ContextRecord->Pc = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + offset = ((-offset)/8)-1; + op = (union uop*)(*buff); + op->fixed = 0x33; + op->x = (short)reg; + op->z = (short)offset; - // - // Set the trap frame and clear the unwound-to-call flag - // + OP_BUFFER_POST_ADJUST(*buff, 2); +} - UNWIND_PARAMS_SET_TRAP_FRAME(UnwindParams, StartingSp, sizeof(ARM64_KTRAP_FRAME)); - ContextRecord->ContextFlags &= ~CONTEXT_UNWOUND_TO_CALL; - break; +void emit_save_reg(char** buff, LONG reg, LONG offset) { + union uop { + short val; + struct { + short z : 6; + short x : 4; // save reg r(19+#X) at [sp+#Z*8], offset <= 504 + short fixed : 6; + }; + }; - // - // Context case - // + union uop* op; - case 0xea: // MSFT_OP_CONTEXT: + OP_BUFFER_PRE_ADJUST(*buff, 2); - // - // Ensure there is enough valid space for the full CONTEXT structure - // + offset = ((offset)/8); + op = (union uop*)(*buff); + op->fixed = 0x34; + op->x = (short)reg; + op->z = (short)offset; - VALIDATE_STACK_ADDRESS(UnwindParams, ContextRecord, sizeof(CONTEXT), 16, &Status); - if (!NT_SUCCESS(Status)) { - return Status; - } + OP_BUFFER_POST_ADJUST(*buff, 2); +} - // - // Restore X0-X28, and D0-D31 - // +void emit_save_reg_x(char** buff, LONG reg, LONG offset) { + union uop { + short val; + struct { + short z : 5; + short x : 4; // save reg r(19+#X) at [sp-(#Z+1)*8]!, pre-indexed offset >= -256 + short fixed : 7; + }; + }; - SourceAddress = StartingSp + offsetof(T_CONTEXT, X0); - for (RegIndex = 0; RegIndex < 29; RegIndex++) { - UPDATE_CONTEXT_POINTERS(UnwindParams, RegIndex, SourceAddress); -#ifdef __GNUC__ - *(&ContextRecord->X0 + RegIndex) = MEMORY_READ_QWORD(UnwindParams, SourceAddress); -#else - ContextRecord->X[RegIndex] = MEMORY_READ_QWORD(UnwindParams, SourceAddress); -#endif - SourceAddress += sizeof(ULONG_PTR); - } + union uop* op; - SourceAddress = StartingSp + offsetof(T_CONTEXT, V); - for (RegIndex = 0; RegIndex < 32; RegIndex++) { - UPDATE_FP_CONTEXT_POINTERS(UnwindParams, RegIndex, SourceAddress); - ContextRecord->V[RegIndex].Low = MEMORY_READ_QWORD(UnwindParams, SourceAddress); - ContextRecord->V[RegIndex].High = MEMORY_READ_QWORD(UnwindParams, SourceAddress + 8); - SourceAddress += 2 * sizeof(ULONGLONG); - } + OP_BUFFER_PRE_ADJUST(*buff, 2); - // - // Restore SP, LR, PC, and the status registers - // + offset = ((-offset)/8)-1; + op = (union uop*)(*buff); + op->fixed = 0x6A; + op->x = (short)reg; + op->z = (short)offset; - SourceAddress = StartingSp + offsetof(T_CONTEXT, Cpsr); - ContextRecord->Cpsr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); + OP_BUFFER_POST_ADJUST(*buff, 2); +} - SourceAddress = StartingSp + offsetof(T_CONTEXT, Fp); - ContextRecord->Fp = MEMORY_READ_QWORD(UnwindParams, SourceAddress); +void emit_save_lrpair(char** buff, LONG reg, LONG offset) { + union uop { + short val; + struct { + short z : 6; + short x : 3; // save pair at [sp+#Z*8], offset <= 504 + short fixed : 7; + }; + }; - SourceAddress = StartingSp + offsetof(T_CONTEXT, Lr); - ContextRecord->Lr = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + union uop* op; - SourceAddress = StartingSp + offsetof(T_CONTEXT, Sp); - ContextRecord->Sp = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + OP_BUFFER_PRE_ADJUST(*buff, 2); - SourceAddress = StartingSp + offsetof(T_CONTEXT, Pc); - ContextRecord->Pc = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + offset = ((offset)/8); + op = (union uop*)(*buff); + op->fixed = 0x6B; + op->x = (short)(reg / 2); + op->z = (short)offset; - SourceAddress = StartingSp + offsetof(T_CONTEXT, Fpcr); - ContextRecord->Fpcr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); + OP_BUFFER_POST_ADJUST(*buff, 2); +} - SourceAddress = StartingSp + offsetof(T_CONTEXT, Fpsr); - ContextRecord->Fpsr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); +void emit_save_fregp(char** buff, LONG reg, LONG offset) { + union uop { + short val; + struct { + short z : 6; + short x : 3; // save pair d(8+#X) at [sp+#Z*8], offset <= 504 + short fixed : 7; + }; + }; - // - // Inherit the unwound-to-call flag from this context - // + union uop* op; - SourceAddress = StartingSp + offsetof(T_CONTEXT, ContextFlags); - ContextRecord->ContextFlags &= ~CONTEXT_UNWOUND_TO_CALL; - ContextRecord->ContextFlags |= - MEMORY_READ_DWORD(UnwindParams, SourceAddress) & CONTEXT_UNWOUND_TO_CALL; - break; + OP_BUFFER_PRE_ADJUST(*buff, 2); - default: - return STATUS_UNSUCCESSFUL; - } + offset = ((offset)/8); + op = (union uop*)(*buff); + op->fixed = 0x6C; + op->x = (short)reg; + op->z = (short)offset; - return STATUS_SUCCESS; + OP_BUFFER_POST_ADJUST(*buff, 2); } -ULONG -RtlpComputeScopeSize( - _In_ ULONG_PTR UnwindCodePtr, - _In_ ULONG_PTR UnwindCodesEndPtr, - _In_ BOOLEAN IsEpilog, - _In_ PARM64_UNWIND_PARAMS UnwindParams - ) +void emit_save_fregp_x(char** buff, LONG reg, LONG offset) { + union uop { + short val; + struct { + short z : 6; + short x : 3; // save pair d(8 + #X), at[sp - (#Z + 1) * 8]!, pre - indexed offset >= -512 + short fixed : 7; + }; + }; -/*++ + union uop* op; -Routine Description: + OP_BUFFER_PRE_ADJUST(*buff, 2); - Computes the size of an prolog or epilog, in words. + offset = ((-offset)/8)-1; + op = (union uop*)(*buff); + op->fixed = 0x6D; + op->x = (short)reg; + op->z = (short)offset; -Arguments: + OP_BUFFER_POST_ADJUST(*buff, 2); +} - UnwindCodePtr - Supplies a pointer to the start of the unwind - code sequence. +void emit_save_freg(char** buff, LONG reg, LONG offset) { + union uop { + short val; + struct { + short z : 6; + short x : 3; // save reg d(8+#X) at [sp+#Z*8], offset <= 504 + short fixed : 7; + }; + }; - UnwindCodesEndPtr - Supplies a pointer to the byte immediately - following the unwind code table, as described by the header. + union uop* op; - IsEpilog - Specifies TRUE if the scope describes an epilog, - or FALSE if it describes a prolog. + OP_BUFFER_PRE_ADJUST(*buff, 2); - UnwindParams - Additional parameters shared with caller. + offset = ((offset)/8); + op = (union uop*)(*buff); + op->fixed = 0x6E; + op->x = (short)reg; + op->z = (short)offset; -Return Value: + OP_BUFFER_POST_ADJUST(*buff, 2); +} - The size of the scope described by the unwind codes, in halfword units. +void emit_save_freg_x(char** buff, LONG reg, LONG offset) { + union uop { + short val; + struct { + short z : 5; + short x : 3; // save reg d(8+#X) at [sp-(#Z+1)*8]!, pre-indexed offset >= -256 + short fixed : 8; + }; + }; ---*/ + union uop* op; -{ - ULONG ScopeSize; - BYTE Opcode; + OP_BUFFER_PRE_ADJUST(*buff, 2); - // - // Iterate through the unwind codes until we hit an end marker. - // While iterating, accumulate the total scope size. - // + offset = ((-offset)/8)-1; + op = (union uop*)(*buff); + op->fixed = 0xDE; + op->x = (short)reg; + op->z = (short)offset; - ScopeSize = 0; - Opcode = 0; - while (UnwindCodePtr < UnwindCodesEndPtr) { - Opcode = MEMORY_READ_BYTE(UnwindParams, UnwindCodePtr); - if (OPCODE_IS_END(Opcode)) { - break; - } + OP_BUFFER_POST_ADJUST(*buff, 2); +} - UnwindCodePtr += UnwindCodeSizeTable[Opcode]; - ScopeSize++; +void emit_alloc(char** buff, LONG size) { + + union uop_alloc_l { + long val; + struct { + long x : 24; // allocate large stack with size < 256M (2^24 *16) + long fixed : 8; + }; + }; + + union uop_alloc_m { + short val; + struct { + short x : 11; // allocate large stack with size < 32K (2^11 * 16) + short fixed : 5; + }; + }; + + union uop_alloc_s { + char val; + struct { + char x : 5; // allocate small stack with size < 512 (2^5 * 16) + char fixed : 3; + }; + }; + + if (size >= 16384) { + union uop_alloc_l* op; + + OP_BUFFER_PRE_ADJUST(*buff, 4); + + op = (union uop_alloc_l*)(*buff); + op->fixed = 0xE0; + op->x = size / 16; + + OP_BUFFER_POST_ADJUST(*buff, 4); } + else if (size >= 512) { + union uop_alloc_m* op; - // - // Epilogs have one extra instruction at the end that needs to be - // accounted for. - // + OP_BUFFER_PRE_ADJUST(*buff, 2); - if (IsEpilog) { - ScopeSize++; + op = (union uop_alloc_m*)(*buff); + op->fixed = 0x18; + op->x = (short)(size / 16); + + OP_BUFFER_POST_ADJUST(*buff, 2); } + else { + union uop_alloc_s* op; - return ScopeSize; + OP_BUFFER_PRE_ADJUST(*buff, 1); + + op = (union uop_alloc_s*)(*buff); + op->fixed = 0x0; + op->x = (char)(size / 16); + + OP_BUFFER_POST_ADJUST(*buff, 1); + } } -NTSTATUS -RtlpUnwindRestoreRegisterRange( - __inout PT_CONTEXT ContextRecord, - _In_ LONG SpOffset, - _In_ ULONG FirstRegister, - _In_ ULONG RegisterCount, - _In_ PARM64_UNWIND_PARAMS UnwindParams - ) +void emit_end(char** buff) { + char* op; -/*++ + OP_BUFFER_PRE_ADJUST(*buff, 1); -Routine Description: + op = (char*)(*buff); + *op = 0xE4; - Restores a series of integer registers from the stack. + OP_BUFFER_POST_ADJUST(*buff, 1); +} -Arguments: +void emit_end_c(char** buff) { + char* op; - ContextRecord - Supplies the address of a context record. + OP_BUFFER_PRE_ADJUST(*buff, 1); - SpOffset - Specifies a stack offset. Positive values are simply used - as a base offset. Negative values assume a predecrement behavior: - a 0 offset is used for restoration, but the absolute value of the - offset is added to the final Sp. + op = (char*)(*buff); + *op = 0xE5; - FirstRegister - Specifies the index of the first register to restore. + OP_BUFFER_POST_ADJUST(*buff, 1); +} - RegisterCount - Specifies the number of registers to restore. +void emit_set_fp(char** buff) { + char* op; - UnwindParams - Additional parameters shared with caller. + OP_BUFFER_PRE_ADJUST(*buff, 1); -Return Value: + op = (char*)(*buff); + *op = 0xE1; - None. + OP_BUFFER_POST_ADJUST(*buff, 1); +} ---*/ +void emit_nop(char** buff) { + char* op; -{ - ULONG_PTR CurAddress; + OP_BUFFER_PRE_ADJUST(*buff, 1); + + op = (char*)(*buff); + *op = 0xE3; + + OP_BUFFER_POST_ADJUST(*buff, 1); +} + +void emit_pac(char** buff) { + char* op; + + OP_BUFFER_PRE_ADJUST(*buff, 1); + + op = (char*)(*buff); + *op = 0xFC; + + OP_BUFFER_POST_ADJUST(*buff, 1); +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +#pragma warning(pop) + +#define NO_HOME_NOPS ((size_t)-1) + +VOID +RtlpExpandCompactToFull ( + _In_ IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY* fnent_pdata, + _Inout_ IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY_XDATA* fnent_xdata +) +{ + + LONG intsz; + LONG fpsz; + LONG savsz; + LONG locsz; + LONG famsz; + BOOLEAN sav_predec_done = FALSE; + BOOLEAN fp_set = FALSE; + LONG sav_slot = 0; + char* op_buffer; + char* op_buffer_start; + char* op_buffer_end; + size_t op_buffer_used; + size_t ops_before_nops = NO_HOME_NOPS; + + // + // Calculate sizes. + // + + famsz = fnent_pdata->FrameSize * 2; + intsz = fnent_pdata->RegI; + if (fnent_pdata->CR == PdataCrUnchainedSavedLr) { + intsz += 1; // lr + } + + fpsz = fnent_pdata->RegF; + if (fnent_pdata->RegF != 0) { + fpsz += 1; + } + + savsz = intsz + fpsz; + + // + // Usually Homes are saved as part of the savesz area. + // In other words, they are saved in the space allocated + // by the pre-decrement operation performed by a non-volatile + // register save. If there are no non-volatile register saves, + // then Homes are saved in the localsz area. + // + + if (savsz > 0) { + savsz += (fnent_pdata->H * 8); + } + + savsz = ALIGN_UP_BY(savsz, 2); + locsz = famsz - savsz; + + // + // Initialize xdata main header. + // + + fnent_xdata->FunctionLength = fnent_pdata->FunctionLength; + fnent_xdata->Version = 0; + fnent_xdata->ExceptionDataPresent = 0; + op_buffer_start = (char*)(fnent_xdata + 1); + op_buffer_end = op_buffer_start + ((fnent_xdata->CodeWords) * 4); + op_buffer = op_buffer_start; + + DBG_OP("end\n"); + emit_end(&op_buffer); + + if (fnent_pdata->CR == PdataCrChainedWithPac) { + DBG_OP("pac\n"); + emit_pac(&op_buffer); + } + + // + // Save the integer registers. + // + + if (intsz != 0) { + ULONG intreg; + + // + // Special case for only x19 + LR, for which an _x option is not + // available, so do the SP decrement by itself first. + // + + if ((fnent_pdata->RegI == 1) && (fnent_pdata->CR == PdataCrUnchainedSavedLr)) { + DBG_OP("alloc_s (%i)\n", savsz * 8); + emit_alloc(&op_buffer, savsz * 8); + sav_predec_done = TRUE; + } + + // + // Issue save-pair instructions as long as there are even number + // or registers to lave left. + // + + for (intreg = 0; intreg < ((fnent_pdata->RegI / 2) * 2); intreg += 2) { + if (!sav_predec_done) { + DBG_OP("save_regp_x\t(%s, %s, %i)\n", int_reg_names[intreg], int_reg_names[intreg + 1], -savsz * 8); + emit_save_regp_x(&op_buffer, intreg, -savsz * 8); + sav_slot += 2; + sav_predec_done = TRUE; + } + else { + DBG_OP("save_regp\t(%s, %s, %i)\n", int_reg_names[intreg], int_reg_names[intreg + 1], sav_slot * 8); + emit_save_regp(&op_buffer, intreg, sav_slot * 8); + sav_slot += 2; + } + } + + // + // Address the remaining possible cases: + // - Last remaining odd register + // - LR, when CR=1 (saving LR needed but no FP chain) + // - Both, as a pair + // + + if ((fnent_pdata->RegI % 2) == 1) { + if (fnent_pdata->CR == PdataCrUnchainedSavedLr) { + + // + // special case at the top of the function makes sure + // !sav_predec_done can't even happen. + // + + _ASSERTE(sav_predec_done); + + DBG_OP("save_lrpair\t(%s, %i)\n", int_reg_names[intreg], sav_slot * 8); + emit_save_lrpair(&op_buffer, intreg, sav_slot * 8); + sav_slot += 2; + } + else { + if (!sav_predec_done) { + DBG_OP("save_reg_x\t(%s, %i)\n", int_reg_names[intreg], -savsz * 8); + emit_save_reg_x(&op_buffer, intreg, -savsz * 8); + sav_slot += 1; + sav_predec_done = TRUE; + } + else { + DBG_OP("save_reg\t(%s, %i)\n", int_reg_names[intreg], sav_slot * 8); + emit_save_reg(&op_buffer, intreg, sav_slot * 8); + sav_slot += 1; + } + } + } + else { + if (fnent_pdata->CR == PdataCrUnchainedSavedLr) { + if (!sav_predec_done) { + DBG_OP("save_reg_x\t(%s, %i)\n", int_reg_names[11], -savsz * 8); + emit_save_reg_x(&op_buffer, 11, -savsz * 8); + sav_slot += 1; + sav_predec_done = TRUE; + } + else { + DBG_OP("save_reg\t(%s, %i)\n", int_reg_names[11], sav_slot * 8); + emit_save_reg(&op_buffer, 11, sav_slot * 8); + sav_slot += 1; + } + } + } + } + + // + // Save the floating point registers. + // + + if (fpsz != 0) { + LONG fpreg; + + for (fpreg = 0; fpreg < ((fpsz / 2) * 2); fpreg += 2) { + if (!sav_predec_done) { + DBG_OP("save_fregp_x\t(%s, %s, %i)\n", fp_reg_names[fpreg], fp_reg_names[fpreg + 1], -savsz * 8); + emit_save_fregp_x(&op_buffer, fpreg, -savsz * 8); + sav_slot += 2; + sav_predec_done = TRUE; + } + else { + DBG_OP("save_fregp\t(%s, %s, %i)\n", fp_reg_names[fpreg], fp_reg_names[fpreg + 1], sav_slot * 8); + emit_save_fregp(&op_buffer, fpreg, sav_slot * 8); + sav_slot += 2; + } + } + + if ((fpsz % 2) == 1) { + if (!sav_predec_done) { + DBG_OP("save_freg_x\t(%s, %i)\n", fp_reg_names[fpreg], -savsz * 8); + emit_save_freg_x(&op_buffer, fpreg, -savsz * 8); + sav_slot += 1; + sav_predec_done = TRUE; + } + else { + DBG_OP("save_freg\t(%s, %i)\n", fp_reg_names[fpreg], sav_slot * 8); + emit_save_freg(&op_buffer, fpreg, sav_slot * 8); + sav_slot += 1; + } + } + } + + // + // Save parameter registers. Record the instructions + // that save them, if Homes are being saved into the + // savesz area. If they are being saved into the localsz + // area, then they don't realy need to be indicated since + // they are no-ops and there is nothing following them. + // In that case, the Homes save instructions will just + // be considered part of the body. + // + + if ((fnent_pdata->H != 0) && sav_predec_done) { + ops_before_nops = op_buffer - op_buffer_start; + DBG_OP("nop\nnop\nnop\nnop\n"); + emit_nop(&op_buffer); + emit_nop(&op_buffer); + emit_nop(&op_buffer); + emit_nop(&op_buffer); + } + + // + // Reserve space for locals and fp,lr chain. + // + + if (locsz > 0) { + if ((fnent_pdata->CR == PdataCrChained) || + (fnent_pdata->CR == PdataCrChainedWithPac)) { + + if (locsz <= (512 / 8)) { + DBG_OP("save_fplr_x\t(%i)\n", -locsz * 8); + emit_save_fplr_x(&op_buffer, -locsz * 8); + } + else { + DBG_OP("alloc\t\t(%i)\n", locsz * 8); + emit_alloc(&op_buffer, locsz * 8); + DBG_OP("save_fplr\t(%i)\n", 0); + emit_save_fplr(&op_buffer, 0); + } + + DBG_OP("set_fp\n"); + emit_set_fp(&op_buffer); + fp_set = TRUE; + } + else { + DBG_OP("alloc\t\t(%i)\n", locsz * 8); + emit_alloc(&op_buffer, locsz * 8); + } + } + + if (fnent_pdata->Flag == PdataPackedUnwindFragment) { + DBG_OP("end_c\n"); + emit_end_c(&op_buffer); + } + + // + // Adjust epilog information in the header + // + + if (fnent_pdata->Flag == PdataPackedUnwindFragment) { + + // + // Fragment case: no epilog + // + + fnent_xdata->EpilogInHeader = 0; + fnent_xdata->EpilogCount = 0; + } + else { + + // + // With EpilogInHeader true, EpilogCount represents + // the op index to the start of the epilog. If the + // set_fp is present in the prolog, set this field + // to 1 so that this op is skipped for the epilog. + // + + fnent_xdata->EpilogInHeader = 1; + if (fp_set) { + fnent_xdata->EpilogCount = 1; + } + else { + fnent_xdata->EpilogCount = 0; + } + } + + // + // Flip the buffer around. This will acomplish two + // needed things: + // - Opcodes closer to the body show first; + // - Opcodes become big-endian, as they should. + // + + op_buffer_used = op_buffer - op_buffer_start; + if (op_buffer_used > 1) { + char* lo = op_buffer_start; + char* hi = op_buffer - 1; + char swap; + while (lo < hi) { + swap = *lo; + *lo++ = *hi; + *hi-- = swap; + } + } + + // + // On functions with homed parameters, generate the + // epilog by copying the prolog minus the param + // saving NOPs. + // + + if ((ops_before_nops != NO_HOME_NOPS) && (fnent_xdata->EpilogInHeader != 0)) { + char* src = op_buffer - 1; + char* dst = src + op_buffer_used -4; + char* skip = src - ops_before_nops; + while (src >= op_buffer_start) { + if (src == skip) { + src -= 4; + continue; + } + + *dst-- = *src--; + } + + fnent_xdata->EpilogCount += (ULONG)op_buffer_used; + op_buffer_used = (op_buffer_used * 2) - 4; + } + + // + // Adjust the CodeWords count. + // + + op_buffer_used = ALIGN_UP_BY(op_buffer_used, 4); + op_buffer_used /= 4; + fnent_xdata->CodeWords = (ULONG)op_buffer_used; + + return; +} + + +static +ULONG_PTR +RtlpGetUnwindCodeSize ( + _In_ ULONG UnwindCode, + _In_opt_ PULONG ScopeSize + ) + +/*++ + +Routine Description: + + This function determines the number of bytes in an unwind code based on the + first byte of that unwind code. + +Argument: + + UnwindCode - Supplies the first byte of the unwind code. + + ScopeSize - Supplies a pointer to a variable that is incremented by the + number of instructions represented by the specified unwind code. + +Return Value: + + The number of bytes in the specified unwind code is returned as the + function value. + +--*/ + +{ + _ASSERTE(UnwindCode <= 0xFF); + + if (UnwindCode < 0xC0) { + if (ARGUMENT_PRESENT(ScopeSize)) { + *ScopeSize += 1; + } + + return 1; + + } else if (UnwindCode < 0xE0) { + if (ARGUMENT_PRESENT(ScopeSize)) { + *ScopeSize += 1; + } + + return 2; + + } else { + if (ARGUMENT_PRESENT(ScopeSize)) { + *ScopeSize += UnwindCodeInstructionCountTable[UnwindCode - 0xE0]; + } + + return UnwindCodeSizeTable[UnwindCode - 0xE0]; + } +} + +static +ULONG +RtlpComputeScopeSize ( + __in ULONG_PTR UnwindCodePtr, + __in ULONG_PTR UnwindCodesEndPtr, + __in BOOLEAN IsEpilog, + __in PARM64_UNWIND_PARAMS UnwindParams + ) + +/*++ + +Routine Description: + + Computes the size of an prolog or epilog, in words. + +Arguments: + + UnwindCodePtr - Supplies a pointer to the start of the unwind + code sequence. + + UnwindCodesEndPtr - Supplies a pointer to the byte immediately + following the unwind code table, as described by the header. + + IsEpilog - Specifies TRUE if the scope describes an epilog, + or FALSE if it describes a prolog. + + UnwindParams - Additional parameters shared with caller. + +Return Value: + + The size of the scope described by the unwind codes, in halfword units. + +--*/ + +{ + ULONG ScopeSize; + BYTE Opcode; + + UNREFERENCED_PARAMETER(UnwindParams); + + // + // Iterate through the unwind codes until we hit an end marker. + // While iterating, accumulate the total scope size. + // + + ScopeSize = 0; + Opcode = 0; + while (UnwindCodePtr < UnwindCodesEndPtr) { + Opcode = MEMORY_READ_BYTE(UnwindParams, UnwindCodePtr); + if (OPCODE_IS_END(Opcode)) { + break; + } + + UnwindCodePtr += RtlpGetUnwindCodeSize(Opcode, &ScopeSize); + } + + // + // Epilogs have one extra instruction at the end that needs to be + // accounted for. + // + + if (IsEpilog) { + ScopeSize++; + } + + return ScopeSize; +} + +static +NTSTATUS +RtlpUnwindRestoreRegisterRange ( + _Inout_ PCONTEXT ContextRecord, + _In_ LONG SpOffset, + _In_range_(0, 30) ULONG FirstRegister, + _In_range_(1, 31-FirstRegister) ULONG RegisterCount, + _In_ PARM64_UNWIND_PARAMS UnwindParams + ) + +/*++ + +Routine Description: + + Restores a series of integer registers from the stack. + +Arguments: + + ContextRecord - Supplies the address of a context record. + + SpOffset - Specifies a stack offset. Positive values are simply used + as a base offset. Negative values assume a predecrement behavior: + a 0 offset is used for restoration, but the absolute value of the + offset is added to the final Sp. + + FirstRegister - Specifies the index of the first register to restore. + + RegisterCount - Specifies the number of registers to restore. + + UnwindParams - Additional parameters shared with caller. + +Return Value: + + None. + +--*/ + +{ + ULONG_PTR CurAddress; + ULONG RegIndex; + NTSTATUS Status; + + // + // Validate non-overflowing register count. + // + + if ((FirstRegister + RegisterCount) > 31) { + return STATUS_UNWIND_INVALID_SEQUENCE; + } + + // + // Compute the source address and validate it. + // + + CurAddress = ContextRecord->Sp; + if (SpOffset >= 0) { + CurAddress += SpOffset; + } + + Status = STATUS_SUCCESS; + VALIDATE_STACK_ADDRESS(UnwindParams, ContextRecord, 8 * RegisterCount, 8, &Status); + if (Status != STATUS_SUCCESS) { + return Status; + } + + // + // Restore the registers + // + + for (RegIndex = 0; RegIndex < RegisterCount; RegIndex++) { + UPDATE_CONTEXT_POINTERS(UnwindParams, FirstRegister + RegIndex, CurAddress); + ContextRecord->X[FirstRegister + RegIndex] = MEMORY_READ_QWORD(UnwindParams, CurAddress); + CurAddress += 8; + } + if (SpOffset < 0) { + ContextRecord->Sp -= SpOffset; + } + + return STATUS_SUCCESS; +} + +static +NTSTATUS +RtlpUnwindRestoreFpRegisterRange ( + __inout PCONTEXT ContextRecord, + __in LONG SpOffset, + __in ULONG FirstRegister, + __in ULONG RegisterCount, + __in PARM64_UNWIND_PARAMS UnwindParams + ) + +/*++ + +Routine Description: + + Restores a series of floating-point registers from the stack. + +Arguments: + + ContextRecord - Supplies the address of a context record. + + SpOffset - Specifies a stack offset. Positive values are simply used + as a base offset. Negative values assume a predecrement behavior: + a 0 offset is used for restoration, but the absolute value of the + offset is added to the final Sp. + + FirstRegister - Specifies the index of the first register to restore. + + RegisterCount - Specifies the number of registers to restore. + + UnwindParams - Additional parameters shared with caller. + +Return Value: + + None. + +--*/ + +{ + ULONG_PTR CurAddress; ULONG RegIndex; NTSTATUS Status; + // + // Validate non-overflowing register count. + // + + if ((FirstRegister + RegisterCount) > 32) { + return STATUS_UNWIND_INVALID_SEQUENCE; + } + // // Compute the source address and validate it. // @@ -499,12 +1307,8 @@ Return Value: // for (RegIndex = 0; RegIndex < RegisterCount; RegIndex++) { - UPDATE_CONTEXT_POINTERS(UnwindParams, FirstRegister + RegIndex, CurAddress); -#ifdef __GNUC__ - *(&ContextRecord->X0 + FirstRegister + RegIndex) = MEMORY_READ_QWORD(UnwindParams, CurAddress); -#else - ContextRecord->X[FirstRegister + RegIndex] = MEMORY_READ_QWORD(UnwindParams, CurAddress); -#endif + UPDATE_FP_CONTEXT_POINTERS(UnwindParams, FirstRegister + RegIndex, CurAddress); + ContextRecord->V[FirstRegister + RegIndex].Low = MEMORY_READ_QWORD(UnwindParams, CurAddress); CurAddress += 8; } if (SpOffset < 0) { @@ -514,20 +1318,21 @@ Return Value: return STATUS_SUCCESS; } +static NTSTATUS -RtlpUnwindRestoreFpRegisterRange( - __inout PT_CONTEXT ContextRecord, - _In_ LONG SpOffset, - _In_ ULONG FirstRegister, - _In_ ULONG RegisterCount, - _In_ PARM64_UNWIND_PARAMS UnwindParams +RtlpUnwindRestoreSimdRegisterRange ( + __inout PCONTEXT ContextRecord, + __in LONG SpOffset, + __in ULONG FirstRegister, + __in ULONG RegisterCount, + __in PARM64_UNWIND_PARAMS UnwindParams ) /*++ Routine Description: - Restores a series of floating-point registers from the stack. + Restores a series of full SIMD (Q) registers from the stack. Arguments: @@ -555,6 +1360,14 @@ Return Value: ULONG RegIndex; NTSTATUS Status; + // + // Validate non-overflowing register count. + // + + if ((FirstRegister + RegisterCount) > 32) { + return STATUS_UNWIND_INVALID_SEQUENCE; + } + // // Compute the source address and validate it. // @@ -565,37 +1378,276 @@ Return Value: } Status = STATUS_SUCCESS; - VALIDATE_STACK_ADDRESS(UnwindParams, ContextRecord, 8 * RegisterCount, 8, &Status); + VALIDATE_STACK_ADDRESS(UnwindParams, ContextRecord, 16 * RegisterCount, 16, &Status); if (Status != STATUS_SUCCESS) { return Status; } - // - // Restore the registers - // + // + // Restore the registers + // + + for (RegIndex = 0; RegIndex < RegisterCount; RegIndex++) { + UPDATE_FP_CONTEXT_POINTERS(UnwindParams, FirstRegister + RegIndex, CurAddress); + ContextRecord->V[FirstRegister + RegIndex].Low = MEMORY_READ_QWORD(UnwindParams, CurAddress); + CurAddress += 8; + ContextRecord->V[FirstRegister + RegIndex].High = MEMORY_READ_QWORD(UnwindParams, CurAddress); + CurAddress += 8; + } + if (SpOffset < 0) { + ContextRecord->Sp -= SpOffset; + } + + return STATUS_SUCCESS; +} + +static +NTSTATUS +RtlpUnwindCustom ( + __inout PCONTEXT ContextRecord, + __in BYTE Opcode, + __in PARM64_UNWIND_PARAMS UnwindParams + ) + +/*++ + +Routine Description: + + Handles custom unwinding operations involving machine-specific + frames. + +Arguments: + + ContextRecord - Supplies the address of a context record. + + Opcode - The opcode to decode. + + UnwindParams - Additional parameters shared with caller. + +Return Value: + + An NTSTATUS indicating either STATUS_SUCCESS if everything went ok, or + another status code if there were problems. + +--*/ + +{ + ULONG Fpcr; + ULONG Fpsr; + ULONG RegIndex; + ULONG_PTR SourceAddress; + ULONG_PTR StartingSp; + NTSTATUS Status; + ULONG_PTR VfpStateAddress; + + StartingSp = ContextRecord->Sp; + Status = STATUS_SUCCESS; + + // + // The opcode describes the special-case stack + // + + switch (Opcode) + { + + // + // Trap frame case + // + + case 0xe8: // MSFT_OP_TRAP_FRAME: + + // + // Ensure there is enough valid space for the trap frame + // + + VALIDATE_STACK_ADDRESS(UnwindParams, ContextRecord, sizeof(ARM64_KTRAP_FRAME), 16, &Status); + if (!NT_SUCCESS(Status)) { + return Status; + } + + // + // Restore X0-X18, and D0-D7 + // + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_KTRAP_FRAME, X); + for (RegIndex = 0; RegIndex < 19; RegIndex++) { + UPDATE_CONTEXT_POINTERS(UnwindParams, RegIndex, SourceAddress); + ContextRecord->X[RegIndex] = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + SourceAddress += sizeof(ULONG_PTR); + } + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_KTRAP_FRAME, VfpState); + VfpStateAddress = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + if (VfpStateAddress != 0) { + + SourceAddress = VfpStateAddress + FIELD_OFFSET(KARM64_VFP_STATE, Fpcr); + Fpcr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); + SourceAddress = VfpStateAddress + FIELD_OFFSET(KARM64_VFP_STATE, Fpsr); + Fpsr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); + if (Fpcr != -1 && Fpsr != -1) { + + ContextRecord->Fpcr = Fpcr; + ContextRecord->Fpsr = Fpsr; + + SourceAddress = VfpStateAddress + FIELD_OFFSET(KARM64_VFP_STATE, V); + for (RegIndex = 0; RegIndex < 32; RegIndex++) { + UPDATE_FP_CONTEXT_POINTERS(UnwindParams, RegIndex, SourceAddress); + ContextRecord->V[RegIndex].Low = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + ContextRecord->V[RegIndex].High = MEMORY_READ_QWORD(UnwindParams, SourceAddress + 8); + SourceAddress += 2 * sizeof(ULONGLONG); + } + } + } + + // + // Restore R11, R12, SP, LR, PC, and the status registers + // + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_KTRAP_FRAME, Spsr); + ContextRecord->Cpsr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_KTRAP_FRAME, Sp); + ContextRecord->Sp = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_KTRAP_FRAME, Lr); + ContextRecord->Lr = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_KTRAP_FRAME, Fp); + ContextRecord->Fp = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_KTRAP_FRAME, Pc); + ContextRecord->Pc = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + + // + // Set the trap frame and clear the unwound-to-call flag + // + + UNWIND_PARAMS_SET_TRAP_FRAME(UnwindParams, StartingSp, sizeof(ARM64_KTRAP_FRAME)); + ContextRecord->ContextFlags &= ~CONTEXT_UNWOUND_TO_CALL; + break; + + // + // Machine frame case + // + + case 0xe9: // MSFT_OP_MACHINE_FRAME: + + // + // Ensure there is enough valid space for the machine frame + // + + VALIDATE_STACK_ADDRESS(UnwindParams, ContextRecord, 16, 16, &Status); + if (!NT_SUCCESS(Status)) { + return Status; + } + + // + // Restore the SP and PC, and clear the unwound-to-call flag + // + + ContextRecord->Sp = MEMORY_READ_QWORD(UnwindParams, StartingSp + 0); + ContextRecord->Pc = MEMORY_READ_QWORD(UnwindParams, StartingSp + 8); + ContextRecord->ContextFlags &= ~CONTEXT_UNWOUND_TO_CALL; + break; + + // + // Context case + // + + case 0xea: // MSFT_OP_CONTEXT: + + // + // Ensure there is enough valid space for the full CONTEXT structure + // + + VALIDATE_STACK_ADDRESS(UnwindParams, ContextRecord, sizeof(ARM64_CONTEXT), 16, &Status); + if (!NT_SUCCESS(Status)) { + return Status; + } + + // + // Restore X0-X28, and D0-D31 + // + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, X); + for (RegIndex = 0; RegIndex < 29; RegIndex++) { + UPDATE_CONTEXT_POINTERS(UnwindParams, RegIndex, SourceAddress); + ContextRecord->X[RegIndex] = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + SourceAddress += sizeof(ULONG_PTR); + } + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, V); + for (RegIndex = 0; RegIndex < 32; RegIndex++) { + UPDATE_FP_CONTEXT_POINTERS(UnwindParams, RegIndex, SourceAddress); + ContextRecord->V[RegIndex].Low = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + ContextRecord->V[RegIndex].High = MEMORY_READ_QWORD(UnwindParams, SourceAddress + 8); + SourceAddress += 2 * sizeof(ULONGLONG); + } + + // + // Restore SP, LR, PC, and the status registers + // + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, Cpsr); + ContextRecord->Cpsr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, Fp); + ContextRecord->Fp = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, Lr); + ContextRecord->Lr = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, Sp); + ContextRecord->Sp = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, Pc); + ContextRecord->Pc = MEMORY_READ_QWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, Fpcr); + ContextRecord->Fpcr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, Fpsr); + ContextRecord->Fpsr = MEMORY_READ_DWORD(UnwindParams, SourceAddress); + + // + // Inherit the unwound-to-call flag from this context + // + + SourceAddress = StartingSp + FIELD_OFFSET(ARM64_CONTEXT, ContextFlags); + ContextRecord->ContextFlags &= ~CONTEXT_UNWOUND_TO_CALL; + ContextRecord->ContextFlags |= + MEMORY_READ_DWORD(UnwindParams, SourceAddress) & CONTEXT_UNWOUND_TO_CALL; + break; + + case 0xeb: // MSFT_OP_EC_CONTEXT: + // NOTE: for .NET, the arm64ec context restoring is not implemented + _ASSERTE(FALSE); + return STATUS_UNSUCCESSFUL; + + case 0xec: // MSFT_OP_CLEAR_UNWOUND_TO_CALL + ContextRecord->ContextFlags &= ~CONTEXT_UNWOUND_TO_CALL; + ContextRecord->Pc = ContextRecord->Lr; + break; - for (RegIndex = 0; RegIndex < RegisterCount; RegIndex++) { - UPDATE_FP_CONTEXT_POINTERS(UnwindParams, FirstRegister + RegIndex, CurAddress); - ContextRecord->V[FirstRegister + RegIndex].Low = MEMORY_READ_QWORD(UnwindParams, CurAddress); - CurAddress += 8; - } - if (SpOffset < 0) { - ContextRecord->Sp -= SpOffset; + default: + return STATUS_UNSUCCESSFUL; } return STATUS_SUCCESS; } NTSTATUS -RtlpUnwindFunctionFull( - _In_ DWORD64 ControlPcRva, - _In_ ULONG_PTR ImageBase, - _In_ PT_RUNTIME_FUNCTION FunctionEntry, - __inout T_CONTEXT *ContextRecord, - _Out_ PDWORD64 EstablisherFrame, - _Outptr_opt_result_maybenull_ PEXCEPTION_ROUTINE *HandlerRoutine, - _Out_ PVOID *HandlerData, - _In_ PARM64_UNWIND_PARAMS UnwindParams +RtlpUnwindFunctionFull ( + __in ULONG ControlPcRva, + __in ULONG_PTR ImageBase, + __in PRUNTIME_FUNCTION FunctionEntry, + __in IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY_XDATA *FunctionEntryExtended, + __inout PCONTEXT ContextRecord, + __out PULONG_PTR EstablisherFrame, + __deref_opt_out_opt PEXCEPTION_ROUTINE *HandlerRoutine, + __out PVOID *HandlerData, + __in PARM64_UNWIND_PARAMS UnwindParams, + __in ULONG UnwindFlags ) /*++ @@ -636,10 +1688,12 @@ Routine Description: returned. HandlerData - Supplies a pointer to a variable that receives a pointer - the language handler data. + the the language handler data. UnwindParams - Additional parameters shared with caller. + UnwindFlags - Supplies additional flags for the unwind operation. + Return Value: STATUS_SUCCESS if the unwind could be completed, a failure status otherwise. @@ -657,11 +1711,11 @@ Return Value: ULONG FunctionLength; ULONG HeaderWord; ULONG NextCode; - DWORD64 OffsetInFunction; + ULONG OffsetInFunction; ULONG ScopeNum; ULONG ScopeSize; ULONG ScopeStart; - DWORD64 SkipWords; + ULONG SkipWords; NTSTATUS Status; ULONG_PTR UnwindCodePtr; ULONG_PTR UnwindCodesEndPtr; @@ -669,6 +1723,8 @@ Return Value: ULONG UnwindIndex; ULONG UnwindWords; + UNREFERENCED_PARAMETER(UnwindFlags); + // // Unless a special frame is encountered, assume that any unwinding // will return us to the return address of a call and set the flag @@ -689,7 +1745,10 @@ Return Value: // Fetch the header word from the .xdata blob // - UnwindDataPtr = ImageBase + FunctionEntry->UnwindData; + UnwindDataPtr = (FunctionEntryExtended != NULL) ? + ((ULONG_PTR)FunctionEntryExtended) : + (ImageBase + FunctionEntry->UnwindData); + HeaderWord = MEMORY_READ_DWORD(UnwindParams, UnwindDataPtr); UnwindDataPtr += 4; @@ -717,11 +1776,11 @@ Return Value: UnwindWords = (EpilogScopeCount >> 16) & 0xff; EpilogScopeCount &= 0xffff; } + + UnwindIndex = 0; if ((HeaderWord & (1 << 21)) != 0) { UnwindIndex = EpilogScopeCount; EpilogScopeCount = 0; - } else { - UnwindIndex = 0; } // @@ -791,6 +1850,11 @@ Return Value: ScopeSize = RtlpComputeScopeSize(UnwindCodePtr + UnwindIndex, UnwindCodesEndPtr, TRUE, UnwindParams); ScopeStart = FunctionLength - ScopeSize; + // + // N.B. This code assumes that no handleable exceptions can occur in + // the prolog or in a chained shrink-wrapping prolog region. + // + if (OffsetInFunction >= ScopeStart) { UnwindCodePtr += UnwindIndex; SkipWords = OffsetInFunction - ScopeStart; @@ -850,7 +1914,7 @@ Return Value: if (OPCODE_IS_END(CurCode)) { break; } - UnwindCodePtr += UnwindCodeSizeTable[CurCode]; + UnwindCodePtr += RtlpGetUnwindCodeSize(CurCode, NULL); SkipWords--; } @@ -885,7 +1949,7 @@ Return Value: ContextRecord, -8 * (CurCode & 0x1f), 19, - 2 + 2 * AccumulatedSaveNexts, + 2 + (2 * AccumulatedSaveNexts), UnwindParams); AccumulatedSaveNexts = 0; } @@ -946,7 +2010,7 @@ Return Value: ContextRecord, 8 * (NextCode & 0x3f), 19 + ((CurCode & 3) << 2) + (NextCode >> 6), - 2 + 2 * AccumulatedSaveNexts, + 2 + (2 * AccumulatedSaveNexts), UnwindParams); AccumulatedSaveNexts = 0; } @@ -962,7 +2026,7 @@ Return Value: ContextRecord, -8 * ((NextCode & 0x3f) + 1), 19 + ((CurCode & 3) << 2) + (NextCode >> 6), - 2 + 2 * AccumulatedSaveNexts, + 2 + (2 * AccumulatedSaveNexts), UnwindParams); AccumulatedSaveNexts = 0; } @@ -1040,7 +2104,7 @@ Return Value: ContextRecord, 8 * (NextCode & 0x3f), 8 + ((CurCode & 1) << 2) + (NextCode >> 6), - 2 + 2 * AccumulatedSaveNexts, + 2 + (2 * AccumulatedSaveNexts), UnwindParams); AccumulatedSaveNexts = 0; } @@ -1056,7 +2120,7 @@ Return Value: ContextRecord, -8 * ((NextCode & 0x3f) + 1), 8 + ((CurCode & 1) << 2) + (NextCode >> 6), - 2 + 2 * AccumulatedSaveNexts, + 2 + (2 * AccumulatedSaveNexts), UnwindParams); AccumulatedSaveNexts = 0; } @@ -1158,29 +2222,111 @@ Return Value: } // - // end_c (11100101): end of unwind code in current chained scope + // end_c (11100101): end of unwind code in current chained scope. + // Continue unwinding parent scope. // else if (CurCode == 0xe5) { - if (AccumulatedSaveNexts != 0) { - return STATUS_UNWIND_INVALID_SEQUENCE; - } - goto finished; + NOTHING; } // - // save_next (11100110): save next non-volatile Int or FP register pair. + // save_next_pair (11100110): save next non-volatile Int or FP register pair. // else if (CurCode == 0xe6) { - AccumulatedSaveNexts++; + AccumulatedSaveNexts += 1; + } + + // + // 11100111 ' 0pxrrrrr ' ffoooooo + // p: 0/1 - single/pair + // x: 0/1 - positive offset / negative offset with writeback + // r: register number + // f: 00/01/10 - X / D / Q + // o: offset * 16 for x=1 or p=1 or f=Q / else offset * 8 + // + + else if (CurCode == 0xe7) { + LONG SpOffset; + ULONG RegCount; + union uop { + unsigned short val; + struct { + unsigned char val1; + unsigned char val2; + }; + struct { + unsigned short o : 6; + unsigned short f : 2; + unsigned short r : 5; + unsigned short x : 1; + unsigned short p : 1; + unsigned short fixed : 1; + }; + } op; + + op.val2 = MEMORY_READ_BYTE(UnwindParams, UnwindCodePtr); + UnwindCodePtr += 1; + op.val1 = MEMORY_READ_BYTE(UnwindParams, UnwindCodePtr); + UnwindCodePtr += 1; + + // + // save_next_pair only permited for pairs. + // + + if ((op.p == 0) && (AccumulatedSaveNexts != 0)) { + return STATUS_UNWIND_INVALID_SEQUENCE; + } + + if (op.fixed != 0) { + return STATUS_UNWIND_INVALID_SEQUENCE; + } + + SpOffset = op.o + op.x; + SpOffset *= ((op.x == 1) || (op.f == 2) || (op.p == 1)) ? (16) : (8); + SpOffset *= (op.x == 1) ? (-1) : (1); + RegCount = 1 + op.p + (2 * AccumulatedSaveNexts); + switch (op.f) { + case 0: + Status = RtlpUnwindRestoreRegisterRange( + ContextRecord, + SpOffset, + op.r, + RegCount, + UnwindParams); + break; + + case 1: + Status = RtlpUnwindRestoreFpRegisterRange( + ContextRecord, + SpOffset, + op.r, + RegCount, + UnwindParams); + break; + + case 2: + Status = RtlpUnwindRestoreSimdRegisterRange( + ContextRecord, + SpOffset, + op.r, + RegCount, + UnwindParams); + break; + + default: + return STATUS_UNWIND_INVALID_SEQUENCE; + } + + AccumulatedSaveNexts = 0; } // // custom_0 (111010xx): restore custom structure // - else if (CurCode >= 0xe8 && CurCode <= 0xeb) { + else if (CurCode >= 0xe8 && CurCode <= 0xec) { if (AccumulatedSaveNexts != 0) { return STATUS_UNWIND_INVALID_SEQUENCE; } @@ -1188,6 +2334,44 @@ Return Value: FinalPcFromLr = FALSE; } + // + // pac (11111100): function has pointer authentication + // + + else if (CurCode == 0xfc) { + if (AccumulatedSaveNexts != 0) { + return STATUS_UNWIND_INVALID_SEQUENCE; + } + + STRIP_PAC(UnwindParams, &ContextRecord->Lr); + + // + // TODO: Implement support for UnwindFlags RTL_VIRTUAL_UNWIND2_VALIDATE_PAC. + // + } + + // + // future/nop: the following ranges represent encodings reserved for + // future extension. They are treated as a nop and, therefore, no + // unwind action is taken. + // + // 11111000|yyyyyyyy + // 11111001|yyyyyyyy|yyyyyyyy + // 11111010|yyyyyyyy|yyyyyyyy|yyyyyyyy + // 11111011|yyyyyyyy|yyyyyyyy|yyyyyyyy|yyyyyyyy + // 111111xx + // + + else if (CurCode >= 0xf8) { + if (AccumulatedSaveNexts != 0) { + return STATUS_UNWIND_INVALID_SEQUENCE; + } + + if (CurCode <= 0xfb) { + UnwindCodePtr += 1 + (CurCode & 0x3); + } + } + // // Anything else is invalid // @@ -1224,33 +2408,126 @@ Return Value: } NTSTATUS -RtlpUnwindFunctionCompact( - _In_ DWORD64 ControlPcRva, - _In_ PT_RUNTIME_FUNCTION FunctionEntry, - __inout T_CONTEXT *ContextRecord, - _Out_ PDWORD64 EstablisherFrame, - _Outptr_opt_result_maybenull_ PEXCEPTION_ROUTINE *HandlerRoutine, +RtlpUnwindFunctionCompact ( + __in ULONG ControlPcRva, + __in ULONG_PTR ImageBase, + __in PRUNTIME_FUNCTION FunctionEntry, + __inout PCONTEXT ContextRecord, + __out PULONG_PTR EstablisherFrame, + __deref_opt_out_opt PEXCEPTION_ROUTINE *HandlerRoutine, + __out PVOID *HandlerData, + __in PARM64_UNWIND_PARAMS UnwindParams, + __in ULONG UnwindFlags + ) +{ + + NTSTATUS Status; + + // + // The longest possible array of unwind opcodes that a compressed format can generate is + // 28 + 24 bytes. Rounding it up to a multiple of 4, that results in an array of 52 bytes. + // Note that the following example isn't even fully legal as any allocation above 4KiB would + // require a call to __chkstk and, thus, rule-out compressed encoding. But since it can be + // encoded, it is considered here. + // + // Compressed: + // + // Flag = PdataPackedUnwindFunction + // RegF = 7 + // RegI = 10 + // H = 1 + // CR = PdataCrChainedWithPac + // FrameSize = 8000/16; + // + // Full Prolog: + // e1 40 c1 e7 e3 e3 e3 e3 d9 90 d9 0e d8 8c d8 0a ca 08 c9 86 c9 04 c8 82 cc 19 fc e4 + // + // Full Epilog (same as prolog minus the 4 x NOP for param home spill): + // e1 40 c1 e7 d9 90 d9 0e d8 8c d8 0a ca 08 c9 86 c9 04 c8 82 cc 19 fc e4 + // + // E4 end + // FC pac + // CC 19 save_regp_x (x19, x20, -208) + // C8 82 save_regp (x21, x22, 16) + // C9 04 save_regp (x23, x24, 32) + // C9 86 save_regp (x25, x26, 48) + // CA 08 save_regp (x27, x28, 64) + // D8 0A save_fregp (d8, d9, 80) + // D8 8C save_fregp (d10, d11, 96) + // D9 0E save_fregp (d12, d13, 112) + // D9 90 save_fregp (d14, d15, 128) + // E3 nop + // E3 nop + // E3 nop + // E3 nop + // C1 E7 alloc (7792) + // 40 save_fplr (0) + // E1 set_fp + // + + struct LOCAL_XDATA { + IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY_XDATA xdata; + char ops[60]; + } fnent_xdata = {}; + + fnent_xdata.xdata.CodeWords = sizeof(fnent_xdata.ops) / 4; + RtlpExpandCompactToFull(FunctionEntry, &fnent_xdata.xdata); + Status = RtlpUnwindFunctionFull(ControlPcRva, + ImageBase, + FunctionEntry, + &fnent_xdata.xdata, + ContextRecord, + EstablisherFrame, + HandlerRoutine, + HandlerData, + UnwindParams, + UnwindFlags); + + return Status; +} + +#if !defined(DEBUGGER_UNWIND) + +NTSTATUS +RtlpxVirtualUnwind ( + _In_ ULONG HandlerType, + _In_ ULONG_PTR ImageBase, + _In_ ULONG_PTR ControlPc, + _In_opt_ PRUNTIME_FUNCTION FunctionEntry, + _Inout_ PCONTEXT ContextRecord, _Out_ PVOID *HandlerData, - _In_ PARM64_UNWIND_PARAMS UnwindParams + _Out_ PULONG_PTR EstablisherFrame, + _Inout_opt_ PKNONVOLATILE_CONTEXT_POINTERS ContextPointers, + _In_opt_ PULONG_PTR LowLimit, + _In_opt_ PULONG_PTR HighLimit, + _Outptr_opt_result_maybenull_ PEXCEPTION_ROUTINE *HandlerRoutine, + _In_ ULONG UnwindFlags ) /*++ Routine Description: - This function virtually unwinds the specified function by parsing the - compact .pdata record to determine where in the function the provided - ControlPc is, and then executing a standard, well-defined set of - operations. + This function virtually unwinds the specified function by executing its + prolog code backward or its epilog code forward. - If a context pointers record is specified (in the UnwindParams), then - the address where each nonvolatile register is restored from is recorded - in the appropriate element of the context pointers record. + If a context pointers record is specified, then the address where each + nonvolatile registers is restored from is recorded in the appropriate + element of the context pointers record. Arguments: - ControlPcRva - Supplies the address where control left the specified - function, as an offset relative to the ImageBase. + HandlerType - Supplies the handler type expected for the virtual unwind. + This may be either an exception or an unwind handler. A flag may + optionally be supplied to indicate that the unwind should assume + that the instruction at the PC is the one we are interested in + (versus the PC being a return address). + + ImageBase - Supplies the base address of the image that contains the + function being unwound. + + ControlPc - Supplies the address where control left the specified + function. FunctionEntry - Supplies the address of the function table entry for the specified function. If appropriate, this should have already been @@ -1258,9 +2535,21 @@ Routine Description: ContextRecord - Supplies the address of a context record. + HandlerData - Supplies a pointer to a variable that receives a pointer + the the language handler data. + EstablisherFrame - Supplies a pointer to a variable that receives the the establisher frame pointer value. + ContextPointers - Supplies an optional pointer to a context pointers + record. + + LowLimit - Supplies an optional low limit used to bound the establisher + frame. This must be supplied in conjunction with a high limit. + + HighLimit - Supplies an optional high limit used to bound the establisher + frame. This must be supplied in conjunction with a low limit. + HandlerRoutine - Supplies an optional pointer to a variable that receives the handler routine address. If control did not leave the specified function in either the prolog or an epilog and a handler of the @@ -1268,10 +2557,7 @@ Routine Description: language specific exception handler is returned. Otherwise, NULL is returned. - HandlerData - Supplies a pointer to a variable that receives a pointer - the language handler data. - - UnwindParams - Additional parameters shared with caller. + UnwindFlags - Supplies additional flags for the unwind operation. Return Value: @@ -1281,270 +2567,147 @@ Return Value: --*/ { - ULONG Count; - ULONG Cr; - ULONG CurrentOffset; - ULONG EpilogLength; - ULONG Flag; - ULONG FloatSize; - ULONG FrameSize; - ULONG FRegOpcodes; - ULONG FunctionLength; - ULONG HBit; - ULONG HOpcodes; - ULONG IRegOpcodes; - ULONG IntSize; - ULONG LocalSize; - DWORD64 OffsetInFunction; - DWORD64 OffsetInScope; - ULONG PrologLength; - ULONG RegF; - ULONG RegI; - ULONG RegSize; - ULONG ScopeStart; - ULONG StackAdjustOpcodes; + ULONG ControlPcRva; NTSTATUS Status; - ULONG UnwindData; - - UnwindData = FunctionEntry->UnwindData; - Status = STATUS_SUCCESS; - - // - // Compact records always describe an unwind to a call. - // + ARM64_UNWIND_PARAMS UnwindParams; + ULONG UnwindType; - ContextRecord->ContextFlags |= CONTEXT_UNWOUND_TO_CALL; - - // - // Extract the basic information about how to do a full unwind. - // - - Flag = UnwindData & 3; - FunctionLength = (UnwindData >> 2) & 0x7ff; - RegF = (UnwindData >> 13) & 7; - RegI = (UnwindData >> 16) & 0xf; - HBit = (UnwindData >> 20) & 1; - Cr = (UnwindData >> 21) & 3; - FrameSize = (UnwindData >> 23) & 0x1ff; - - if (Flag == 3) { - return STATUS_UNWIND_INVALID_SEQUENCE; - } - if (Cr == 2) { - return STATUS_UNWIND_INVALID_SEQUENCE; - } - - // - // Determine the size of the locals - // - - IntSize = RegI * 8; - if (Cr == 1) { - IntSize += 8; - } - FloatSize = (RegF == 0) ? 0 : (RegF + 1) * 8; - RegSize = (IntSize + FloatSize + 8*8 * HBit + 0xf) & ~0xf; - if (RegSize > 16 * FrameSize) { - return STATUS_UNWIND_INVALID_SEQUENCE; - } - LocalSize = 16 * FrameSize - RegSize; + UNREFERENCED_PARAMETER(HandlerType); - // - // If we're near the start of the function (within 17 words), - // see if we are within the prolog. - // - // N.B. If the low 2 bits of the UnwindData are 2, then we have - // no prolog. - // + _ASSERTE((UnwindFlags & ~RTL_VIRTUAL_UNWIND_VALID_FLAGS_ARM64) == 0); - OffsetInFunction = (ControlPcRva - FunctionEntry->BeginAddress) / 4; - OffsetInScope = 0; - if (OffsetInFunction < 17 && Flag != 2) { + if (FunctionEntry == NULL) { // - // Compute sizes for each opcode in the prolog. + // If the function does not have a function entry, then it is + // a pure leaf/trivial function. This means the stack pointer + // does not move, and LR is never overwritten, from the time + // it was called to the time it returns. To unwind such function, + // assign the value in LR to PC, simulating a simple ret instruction. // - IRegOpcodes = (IntSize + 8) / 16; - FRegOpcodes = (FloatSize + 8) / 16; - HOpcodes = 4 * HBit; - StackAdjustOpcodes = (Cr == 3) ? 1 : 0; - if (Cr != 3 || LocalSize > 512) { - StackAdjustOpcodes += (LocalSize > 4088) ? 2 : (LocalSize > 0) ? 1 : 0; - } - - // - // Compute the total prolog length and determine if we are within - // its scope. // - // N.B. We must execute prolog operations backwards to unwind, so - // our final scope offset in this case is the distance from the end. + // If the old control PC is the same as the return address, + // then no progress is being made and the stack is most + // likely malformed. // - PrologLength = IRegOpcodes + FRegOpcodes + HOpcodes + StackAdjustOpcodes; - - if (OffsetInFunction < PrologLength) { - OffsetInScope = PrologLength - OffsetInFunction; + if (ControlPc == ContextRecord->Lr) { + return STATUS_BAD_FUNCTION_TABLE; } - } - - // - // If we're near the end of the function (within 15 words), see if - // we are within the epilog. - // - // N.B. If the low 2 bits of the UnwindData are 2, then we have - // no epilog. - // - - if (OffsetInScope == 0 && OffsetInFunction + 15 >= FunctionLength && Flag != 2) { // - // Compute sizes for each opcode in the epilog. + // Set the point where control left the current function by + // obtaining the return address from the current context. + // Also indicate that we unwound from a call so that the + // language-specific handler can differentiate neighboring + // exception scopes. // - IRegOpcodes = (IntSize + 8) / 16; - FRegOpcodes = (FloatSize + 8) / 16; - HOpcodes = HBit; - StackAdjustOpcodes = (Cr == 3) ? 1 : 0; - if (Cr != 3 || LocalSize > 512) { - StackAdjustOpcodes += (LocalSize > 4088) ? 2 : (LocalSize > 0) ? 1 : 0; - } + ContextRecord->Pc = ContextRecord->Lr; + ContextRecord->ContextFlags |= CONTEXT_UNWOUND_TO_CALL; // - // Compute the total epilog length and determine if we are within - // its scope. + // Set remaining output data and return. All work done. // - EpilogLength = IRegOpcodes + FRegOpcodes + HOpcodes + StackAdjustOpcodes + 1; - - ScopeStart = FunctionLength - EpilogLength; - if (OffsetInFunction > ScopeStart) { - OffsetInScope = OffsetInFunction - ScopeStart; - } - } - - // - // Process operations backwards, in the order: stack/frame deallocation, - // VFP register popping, integer register popping, parameter home - // area recovery. - // - // First case is simple: we process everything with no regard for - // the current offset within the scope. - // - - Status = STATUS_SUCCESS; - if (OffsetInScope == 0) { - - if (Cr == 3) { - Status = RtlpUnwindRestoreRegisterRange(ContextRecord, 0, 29, 2, UnwindParams); - } - ContextRecord->Sp += LocalSize; - - if (RegF != 0 && Status == STATUS_SUCCESS) { - Status = RtlpUnwindRestoreFpRegisterRange(ContextRecord, IntSize, 8, RegF + 1, UnwindParams); + *EstablisherFrame = ContextRecord->Sp; + *HandlerData = NULL; + if (ARGUMENT_PRESENT(HandlerRoutine)) { + *HandlerRoutine = NULL; } - if (Cr == 1 && Status == STATUS_SUCCESS) { - Status = RtlpUnwindRestoreRegisterRange(ContextRecord, IntSize - 8, 30, 1, UnwindParams); - } - if (RegI > 0 && Status == STATUS_SUCCESS) { - Status = RtlpUnwindRestoreRegisterRange(ContextRecord, 0, 19, RegI, UnwindParams); - } - ContextRecord->Sp += RegSize; + return STATUS_SUCCESS; } // - // Second case is more complex: we must step along each operation - // to ensure it should be executed. + // Make sure out-of-bound stack accesses don't send us into an infinite + // unwinding loop. // +#if 0 + __try { +#endif + // + // Build an UnwindParams structure containing the starting PC, stack + // limits, and context pointers. + // - else { + UnwindParams.ControlPc = ControlPc; + UnwindParams.LowLimit = LowLimit; + UnwindParams.HighLimit = HighLimit; + UnwindParams.ContextPointers = ContextPointers; + UnwindType = (FunctionEntry->UnwindData & 3); - CurrentOffset = 0; - if (Cr == 3) { - if (LocalSize <= 512) { - if (CurrentOffset++ >= OffsetInScope) { - Status = RtlpUnwindRestoreRegisterRange(ContextRecord, -(LONG)LocalSize, 29, 2, UnwindParams); - } - LocalSize = 0; - } - } - while (LocalSize != 0) { - Count = (LocalSize + 4087) % 4088 + 1; - if (CurrentOffset++ >= OffsetInScope) { - ContextRecord->Sp += Count; - } - LocalSize -= Count; - } + // + // Unwind type 3 refers to a chained record. The top 30 bits of the + // unwind data contains the RVA of the parent pdata record. + // - if (HBit != 0) { - CurrentOffset += 4; - } + if (UnwindType == 3) { + if ((FunctionEntry->UnwindData & 4) == 0) { + FunctionEntry = (PRUNTIME_FUNCTION)(ImageBase + FunctionEntry->UnwindData - 3); + UnwindType = (FunctionEntry->UnwindData & 3); - if (RegF != 0 && Status == STATUS_SUCCESS) { - RegF++; - while (RegF != 0) { - Count = 2 - (RegF & 1); - RegF -= Count; - if (CurrentOffset++ >= OffsetInScope) { - Status = RtlpUnwindRestoreFpRegisterRange( - ContextRecord, - (RegF == 0 && RegI == 0) ? (-(LONG)RegSize) : (IntSize + 8 * RegF), - 8 + RegF, - Count, - UnwindParams); - } - } - } + _ASSERTE(UnwindType != 3); + + ControlPcRva = FunctionEntry->BeginAddress; - if (Cr == 1 && Status == STATUS_SUCCESS) { - if (RegI % 2 == 0) { - if (CurrentOffset++ >= OffsetInScope) { - Status = RtlpUnwindRestoreRegisterRange(ContextRecord, IntSize - 8, 30, 1, UnwindParams); - } } else { - if (CurrentOffset++ >= OffsetInScope) { - RegI--; - Status = RtlpUnwindRestoreRegisterRange(ContextRecord, IntSize - 8, 30, 1, UnwindParams); - if (Status == STATUS_SUCCESS) { - Status = RtlpUnwindRestoreRegisterRange(ContextRecord, IntSize - 16, 19 + RegI, 1, UnwindParams); - } - } + return STATUS_UNWIND_UNSUPPORTED_VERSION; } + + } else { + ControlPcRva = (ULONG)(ControlPc - ImageBase); } - while (RegI != 0 && Status == STATUS_SUCCESS) { - Count = 2 - (RegI & 1); - RegI -= Count; - if (CurrentOffset++ >= OffsetInScope) { - Status = RtlpUnwindRestoreRegisterRange( - ContextRecord, - (RegI == 0) ? (-(LONG)RegSize) : (8 * RegI), - 19 + RegI, - Count, - UnwindParams); - } + // + // Identify the compact .pdata format versus the full .pdata+.xdata format. + // + + if (UnwindType != 0) { + Status = RtlpUnwindFunctionCompact(ControlPcRva, + ImageBase, + FunctionEntry, + ContextRecord, + EstablisherFrame, + HandlerRoutine, + HandlerData, + &UnwindParams, + UnwindFlags); + + } else { + + Status = RtlpUnwindFunctionFull(ControlPcRva, + ImageBase, + FunctionEntry, + NULL, + ContextRecord, + EstablisherFrame, + HandlerRoutine, + HandlerData, + &UnwindParams, + UnwindFlags); } + #if 0 } // - // If we succeeded, post-process the results a bit + // If we do take an exception here, fetch the exception code as the status + // and do not propagate the exception. Since the exception handler also + // uses this function, propagating it will most likely generate the same + // exception at the same point in the unwind, and continuing will typically + // overflow the kernel stack. // - if (Status == STATUS_SUCCESS) { - - ContextRecord->Pc = ContextRecord->Lr; - *EstablisherFrame = ContextRecord->Sp; - - if (ARGUMENT_PRESENT(HandlerRoutine)) { - *HandlerRoutine = NULL; - } - *HandlerData = NULL; + __except (EXCEPTION_EXECUTE_HANDLER) { + Status = GetExceptionCode(); } - +#endif // HOST_WINDOWS return Status; } +#endif // !defined(DEBUGGER_UNWIND) + BOOL OOPStackUnwinderArm64::Unwind(T_CONTEXT * pContext) { DWORD64 ImageBase = 0; @@ -1552,7 +2715,7 @@ BOOL OOPStackUnwinderArm64::Unwind(T_CONTEXT * pContext) if (hr != S_OK) return FALSE; - PEXCEPTION_ROUTINE DummyHandlerRoutine; + PEXCEPTION_ROUTINE DummyHandlerRoutine = NULL; PVOID DummyHandlerData; DWORD64 DummyEstablisherFrame; @@ -1563,49 +2726,29 @@ BOOL OOPStackUnwinderArm64::Unwind(T_CONTEXT * pContext) if (FAILED(GetFunctionEntry(pContext->Pc, &Rfe, sizeof(Rfe)))) return FALSE; - DWORD64 ControlPcRva = pContext->Pc - ImageBase; - - // Long branch pdata - if ((Rfe.UnwindData & 3) == 3) - { - if ((Rfe.UnwindData & 4) == 0) - { - Rfe.BeginAddress = MEMORY_READ_DWORD(NULL, ImageBase + (Rfe.UnwindData - 3)); - Rfe.UnwindData = MEMORY_READ_DWORD(NULL, ImageBase + (Rfe.UnwindData - 3) + sizeof(DWORD)); - - // A long branch should never be described by another long branch - ASSERT_AND_CHECK((Rfe.UnwindData & 3) != 3); - - ControlPcRva = Rfe.BeginAddress; - - } else - { - return FALSE; - } - } + NTSTATUS Status; - if ((Rfe.UnwindData & 3) != 0) - { + Status = RtlpxVirtualUnwind(0 /* HandlerType */, + ImageBase, + pContext->Pc, + &Rfe, + pContext, + &DummyHandlerData, + &DummyEstablisherFrame, + NULL, + NULL, + NULL, + &DummyHandlerRoutine, + 0); - hr = RtlpUnwindFunctionCompact(ControlPcRva, - &Rfe, - pContext, - &DummyEstablisherFrame, - &DummyHandlerRoutine, - &DummyHandlerData, - NULL); + // + // If we fail the unwind, clear the PC to 0. This is recognized by + // many callers as a failure, given that RtlVirtualUnwind does not + // return a status code. + // - } - else - { - hr = RtlpUnwindFunctionFull(ControlPcRva, - ImageBase, - &Rfe, - pContext, - &DummyEstablisherFrame, - &DummyHandlerRoutine, - &DummyHandlerData, - NULL); + if (!NT_SUCCESS(Status)) { + pContext->Pc = 0; } // PC == 0 means unwinding is finished. @@ -1633,78 +2776,48 @@ BOOL DacUnwindStackFrame(T_CONTEXT *pContext, T_KNONVOLATILE_CONTEXT_POINTERS* p } #if defined(HOST_UNIX) + +#undef PRUNTIME_FUNCTION + PEXCEPTION_ROUTINE RtlVirtualUnwind( IN ULONG HandlerType, IN ULONG64 ImageBase, IN ULONG64 ControlPc, - IN PT_RUNTIME_FUNCTION FunctionEntry, + IN PRUNTIME_FUNCTION FunctionEntry, IN OUT PCONTEXT ContextRecord, OUT PVOID *HandlerData, OUT PULONG64 EstablisherFrame, - IN OUT PT_KNONVOLATILE_CONTEXT_POINTERS ContextPointers OPTIONAL + IN OUT PKNONVOLATILE_CONTEXT_POINTERS ContextPointers OPTIONAL ) { - PEXCEPTION_ROUTINE handlerRoutine; - HRESULT hr; - - DWORD64 startingPc = ControlPc; - DWORD64 startingSp = ContextRecord->Sp; - - T_RUNTIME_FUNCTION rfe; - - rfe.BeginAddress = FunctionEntry->BeginAddress; - rfe.UnwindData = FunctionEntry->UnwindData; - - ARM64_UNWIND_PARAMS unwindParams; - unwindParams.ContextPointers = ContextPointers; - - DWORD64 ControlPcRva = ControlPc - ImageBase; - - // Long branch pdata - if ((rfe.UnwindData & 3) == 3) - { - if ((rfe.UnwindData & 4) == 0) - { - rfe.BeginAddress = MEMORY_READ_DWORD(NULL, ImageBase + (rfe.UnwindData - 3)); - rfe.UnwindData = MEMORY_READ_DWORD(NULL, ImageBase + (rfe.UnwindData - 3) + sizeof(DWORD)); - - // A long branch should never be described by another long branch - ASSERT_AND_CHECK((rfe.UnwindData & 3) != 3); - - ControlPcRva = rfe.BeginAddress; + PEXCEPTION_ROUTINE HandlerRoutine; + NTSTATUS Status; - } else - { - return FALSE; - } - } + HandlerRoutine = NULL; + Status = RtlpxVirtualUnwind(HandlerType, + ImageBase, + ControlPc, + (PIMAGE_ARM64_RUNTIME_FUNCTION_ENTRY)FunctionEntry, + ContextRecord, + HandlerData, + EstablisherFrame, + ContextPointers, + NULL, + NULL, + &HandlerRoutine, + 0); - if ((rfe.UnwindData & 3) != 0) - { - hr = RtlpUnwindFunctionCompact(ControlPcRva, - &rfe, - ContextRecord, - EstablisherFrame, - &handlerRoutine, - HandlerData, - &unwindParams); + // + // If we fail the unwind, clear the PC to 0. This is recognized by + // many callers as a failure, given that RtlVirtualUnwind does not + // return a status code. + // + if (!NT_SUCCESS(Status)) { + ContextRecord->Pc = 0; } - else - { - hr = RtlpUnwindFunctionFull(ControlPcRva, - ImageBase, - &rfe, - ContextRecord, - EstablisherFrame, - &handlerRoutine, - HandlerData, - &unwindParams); - } - - _ASSERTE(SUCCEEDED(hr)); - return handlerRoutine; + return HandlerRoutine; } #endif