Merge pull request #2430 from terminal-cs/master

Minor heap changes, Realloc function

Author: quajak

Docs/articles/Kernel/MemoryManagement.md

@@ -33,7 +33,7 @@ The RAT initialisation is triggered using `GCImplementation.Init()` which is cal
 
 The RAT is managed through the `RAT` class. Pages are allocated via `void* RAT.AllocPages(byte aType, uint aPageCount = 1)`. If more than 1 page is allocated at once, the first page will be marked as type `aType`, while all later pages are marked as `Extension`. Pages can be freed (set to type `Empty`) using `void RAT.Free(uint aPageIdx)` which also frees any extension pages which follow the first page. To convert between a pointer and the page index, the method `uint RAT.GetFirstRATIndex(void* aPtr)` can be used. 
 
-The Heap itself is managed by the `Heap` class. It contains the mechanism to allocate (`byte* Heap.Alloc(uint aSize)`) and free (`void Heap.Free(void* aPtr)`) objects of various sizes. Objects are seperated by size in bytes into Small (Smaller than 1/4 Page), Medium (Smaller than 1 Page) and Large (Larger than 1 Page). Currently Medium and Large objects are managed the same way using the methods in `HeapLarge` which do little more than allocating/freeing the necessary number of pages. Small objects are managed differently in `HeapSmall`. 
+The Heap itself is managed by the `Heap` class. It contains the mechanism to allocate (`byte* Heap.Alloc(uint aSize)`), re-allocate ('byte* Heap.Realloc(byte* aPtr, uint newSize)') and free (`void Heap.Free(void* aPtr)`) objects of various sizes. Objects are seperated by size in bytes into Small (Smaller than 1/4 Page), Medium (Smaller than 1 Page) and Large (Larger than 1 Page). Currently Medium and Large objects are managed the same way using the methods in `HeapLarge` which do little more than allocating/freeing the necessary number of pages. Small objects are managed differently in `HeapSmall`. 
 
 Small Objects are managed using the SMT (Size Map Table), which is initalised using `void HeapSmall.InitSMT(uint aMaxItemSize)`. The basic idea of the SMT is to allocate objects of similar sizes on the same page. The SMT grows dynamically as required. The SMT is made up of a series of pages, each of which contains a series of `RootSMTBlock` each of which link to a chain of `SMTBlock`. The `RootSMTBlock` can be thought of as column headers and the `SMTBlock` as the elements stored in the column. The `RootSMTBlock` are a linked list, each containing the maximum object size stored in its pages, the location of the first `SMTBlock` for this size, and the location of the next `RootSMTBlock`. The list is in ascending order of size, so that the smallest large enough `RootSMTBlock` is found first. A `SMTBlock` contains a pointer to the actual page where objects are stored, how much space is left on that page, and a pointer to the next `SMTBlock`. If every `SMTBlock` for a certain size is full, a new `SMTBlock` is allocated. The page linked to by the `SMTBlock` is split into an array of spaces, each large enough to allocate an object of maximum size with header, which can be iterated through via index and fixed size when allocating. Each object allocated on the `HeapSmall` has a header of 2 `ushort`, the first one storing the actual size of the object and the second, the GC status of the object.
 

Tests/Kernels/Cosmos.Compiler.Tests.TypeSystem/Kernel.cs

@@ -103,7 +103,7 @@ private unsafe void TestGarbageCollectorMethods()
             Assert.AreEqual(allocated, afterFree, "Free causes one object to be freed again");
 
             var testString = "asd";
-            Assert.AreEqual(RAT.PageType.Empty, RAT.GetPageType(GCImplementation.GetPointer(testString)), "String is created statically and not managed by GC");
+            Assert.AreEqual((byte)RAT.PageType.Empty, (byte)RAT.GetPageType(GCImplementation.GetPointer(testString)), "String is created statically and not managed by GC");
 
             Assert.IsTrue(Heap.Collect() >= 0, "Running GC Collect first time does not crash and returns non-negative value");
         }

Tests/Kernels/MemoryOperationsTest/Kernel.cs

@@ -5,6 +5,7 @@
 using System.Text;
 using Cosmos.System.ExtendedASCII;
 using Cosmos.System.ScanMaps;
+using Cosmos.Core.Memory;
 using Cosmos.Core;
 using System.Runtime.InteropServices;
 
@@ -156,6 +157,22 @@ static unsafe void TestMemoryBlock(MemoryBlock memoryBlock)
             memoryBlock.Read32(read);
             Assert.AreEqual(values, read, "Using Fill(int, int, int) works");
         }
+        static unsafe void TestRealloc()
+		{
+            // Allocate initial pointer and fill with value 32
+            byte* aPtr = Heap.Alloc(16);
+            MemoryOperations.Fill(aPtr, (byte)32, 16);
+
+            // Resize/realloc to 17 bytes
+            aPtr = Heap.Realloc(aPtr, 17);
+
+            // Test for first 16 being 32 and last being 0
+            for (int i = 0; i < 15; i++)
+			{
+                Assert.AreEqual(aPtr[i], 32, $"Expected value 32 not found in index {i} of aPtr.");
+			}
+            Assert.AreEqual(aPtr[16], 0, "Expected value 0 not found at the end of aPtr.");
+		}
 
         protected override void Run()
         {
@@ -164,6 +181,7 @@ protected override void Run()
                 TestCopy();
                 TestMemoryBlock(new MemoryBlock(0x60000, 128)); //we are testing in SVGA video memory which should not be in use
                 TestManagedMemoryBlock(new ManagedMemoryBlock(128));
+                TestRealloc();
                 SpanTest.Execute();
                 TestController.Completed();
             }

source/Cosmos.Core/GCImplementation.cs

@@ -65,7 +65,7 @@ public static ulong GetAvailableRAM()
         /// <returns>Returns the used PageSize by the MemoryManager in Bytes.</returns>
         public static uint GetUsedRAM()
         {
-            return (RAT.TotalPageCount - RAT.GetPageCount(RAT.PageType.Empty)) * RAT.PageSize;
+            return (RAT.TotalPageCount - RAT.GetPageCount((byte)RAT.PageType.Empty)) * RAT.PageSize;
         }
         /// <summary>
         /// Initialise the Memory Manager, this should not be called anymore since it is done very early during the boot process.

source/Cosmos.Core/Memory/Heap.cs

@@ -32,6 +32,47 @@ public static unsafe void Init()
             HeapLarge.Init();
         }
 
+        /// <summary>
+		/// Re-allocates or "re-sizes" data asigned to a pointer.
+		/// The pointer specified must be the start of an allocated block in the heap.
+		/// This shouldn't be used with objects as a new address is given when realocating memory.
+		/// </summary>
+		/// <param name="aPtr">Existing pointer</param>
+		/// <param name="NewSize">Size to extend to</param>
+		/// <returns>New pointer with specified size while maintaining old data.</returns>
+        public static byte* Realloc(byte* aPtr, uint newSize)
+		{
+            // TODO: don't move memory position if there is enough space in the current one.
+
+            // Get existing size
+            uint Size = (RAT.GetPageType(aPtr) == RAT.PageType.HeapSmall ? ((ushort*)aPtr)[-2] : ((uint*)aPtr)[-4]); 
+
+            if (Size == newSize)
+			{
+                // Return existing pointer as nothing needs to be done.
+                return aPtr;
+			}
+            if (Size > newSize)
+			{
+                Size -= (newSize - Size);
+			}
+
+            // Allocate a new buffer to use
+			byte* ToReturn = Alloc(newSize);
+
+            // Copy the old buffer to the new one
+            MemoryOperations.Copy(ToReturn, aPtr, (int)Size);
+
+            // Comented out to help in the future if we use objects with realloc
+            // Copy the GC state
+            //((ushort*)ToReturn)[-1] = ((ushort*)aPtr)[-1];
+            ((ushort*)ToReturn)[-1] = 0;
+
+            // Free the old data and return
+            Free(aPtr);
+            return ToReturn;
+		}
+
         /// <summary>
         /// Alloc memory block, of a given size.
         /// </summary>
@@ -123,7 +164,7 @@ public static int Collect()
             for (int ratIndex = 0; ratIndex < RAT.TotalPageCount; ratIndex++)
             {
                 var pageType = *(RAT.mRAT + ratIndex);
-                if (pageType == RAT.PageType.HeapMedium || pageType == RAT.PageType.HeapLarge)
+                if (pageType == (byte)RAT.PageType.HeapMedium || pageType == (byte)RAT.PageType.HeapLarge)
                 {
                     var pagePtr = RAT.RamStart + ratIndex * RAT.PageSize;
                     if (*(ushort*)(pagePtr + 3) != 0)
@@ -184,7 +225,7 @@ public static int Collect()
             for (int ratIndex = 0; ratIndex < RAT.TotalPageCount; ratIndex++)
             {
                 var pageType = *(RAT.mRAT + ratIndex);
-                if (pageType == RAT.PageType.HeapMedium || pageType == RAT.PageType.HeapLarge)
+                if (pageType == (byte)RAT.PageType.HeapMedium || pageType == (byte)RAT.PageType.HeapLarge)
                 {
                     var pagePointer = RAT.RamStart + ratIndex * RAT.PageSize;
                     if (*((ushort*)(pagePointer + HeapLarge.PrefixBytes) - 1) == 0)

source/Cosmos.Core/Memory/HeapLarge.cs

@@ -29,7 +29,7 @@ public static void Init()
         /// </summary>
         /// <param name="aSize">A size of block to alloc, in bytes.</param>
         /// <returns>Byte pointer to the start of the block.</returns>
-        public static byte* Alloc(uint aSize, byte aType = RAT.PageType.HeapLarge)
+        public static byte* Alloc(uint aSize, RAT.PageType aType = RAT.PageType.HeapLarge)
         {
             uint xPages = ((aSize + PrefixBytes) / RAT.PageSize) + 1;
             var xPtr = (uint*)RAT.AllocPages(aType, xPages);

source/Cosmos.Core/Memory/RAT.cs

@@ -19,48 +19,47 @@ unsafe static public class RAT
         // than who owns them.
 
         /// <summary>
-        /// PageType class. Used like a enum to define the type of the page.
+        /// PageType enum. Used to define the type of the page.
         /// </summary>
-        /// <remarks>Only used to define page type.</remarks>
-        static public class PageType
+        public enum PageType
         {
             /// <summary>
             /// Empty page.
             /// Can also indicate invalid page.
             /// </summary>
-            public const byte Empty = 0;
+			Empty = 0,
 
             // Data Types from 1, special meanings from 255 down.
             /// <summary>
             /// Indicates that the page contains objects managed by the GC
             /// </summary>
-            public const byte GCManaged = 1;
+            GCManaged = 1,
             /// <summary>
             /// Small heap page.
             /// </summary>
-            public const byte HeapSmall = 3;
+            HeapSmall = 3,
             /// <summary>
             /// Medium heap page.
             /// </summary>
-            public const byte HeapMedium = 5;
+            HeapMedium = 5,
             /// <summary>
             /// Large heap page.
             /// </summary>
-            public const byte HeapLarge = 7;
+            HeapLarge = 7,
 
             /// <summary>
             /// RAT type page.
             /// </summary>
-            public const byte RAT = 32;
+            RAT = 32,
             /// <summary>
             /// Page which is part of the SMT
             /// </summary>
-            public const byte SMT = 64;
+            SMT = 64,
             // Extension of previous page.
             /// <summary>
             /// Extension of pre-existing page.
             /// </summary>
-            public const byte Extension = 128;
+            Extension = 128,
         }
 
         /// <summary>
@@ -144,12 +143,12 @@ public static void Init(byte* aStartPtr, uint aSize)
             // Mark empty pages as such in the RAT Table
             for (byte* p = mRAT; p < mRAT + TotalPageCount - xRatPageCount; p++)
             {
-                *p = PageType.Empty;
+                *p = (byte)PageType.Empty;
             }
             // Mark the rat pages as such
             for (byte* p = mRAT + TotalPageCount - xRatPageCount; p < mRAT + xRatTotalSize; p++)
             {
-                *p = PageType.RAT;
+                *p = (byte)PageType.RAT;
             }
             Heap.Init();
         }
@@ -172,7 +171,7 @@ public static uint GetPageCount(byte aType = 0)
                 }
                 else if (xCounting)
                 {
-                    if (*p == PageType.Extension)
+                    if (*p == (byte)PageType.Extension)
                     {
                         xResult++;
                     }
@@ -191,7 +190,7 @@ public static uint GetPageCount(byte aType = 0)
         /// <param name="aType">A type of pages to alloc.</param>
         /// <param name="aPageCount">Number of pages to alloc. (default = 1)</param>
         /// <returns>A pointer to the first page on success, null on failure.</returns>
-        public static void* AllocPages(byte aType, uint aPageCount = 1)
+        public static void* AllocPages(PageType aType, uint aPageCount = 1)
         {
             byte* xPos = null;
 
@@ -203,7 +202,7 @@ public static uint GetPageCount(byte aType = 0)
             {
                 for (byte* p = mRAT; p < mRAT + TotalPageCount; p++)
                 {
-                    if (*p == PageType.Empty)
+                    if (*p == (byte)PageType.Empty)
                     {
                         xPos = p;
                         break;
@@ -216,7 +215,7 @@ public static uint GetPageCount(byte aType = 0)
                 // so we don't bother to account for such a case. xPos would also have issues.
                 for (byte* p = mRAT + TotalPageCount - 1; p >= mRAT; p--)
                 {
-                    if (*p == PageType.Empty)
+                    if (*p == (byte)PageType.Empty)
                     {
                         if (++xCount == aPageCount)
                         {
@@ -236,10 +235,10 @@ public static uint GetPageCount(byte aType = 0)
             {
                 var diff = xPos - mRAT;
                 byte* xResult = RamStart + diff * PageSize;
-                *xPos = aType;
+                *xPos = (byte)aType;
                 for (byte* p = xPos + 1; p < xPos + xCount; p++)
                 {
-                    *p = PageType.Extension;
+                    *p = (byte)PageType.Extension;
                 }
                 CPU.ZeroFill((uint)xResult, PageSize * aPageCount);
                 return xResult;
@@ -260,7 +259,7 @@ public static uint GetFirstRATIndex(void* aPtr)
             // See note about when mRAT = 0 in Alloc.
             for (byte* p = mRAT + xPos; p >= mRAT; p--)
             {
-                if (*p != PageType.Extension)
+                if (*p != (byte)PageType.Extension)
                 {
                     return (uint)(p - mRAT);
                 }
@@ -279,13 +278,13 @@ public static uint GetFirstRATIndex(void* aPtr)
         /// <param name="aPtr">A pointer to the page to get the type of.</param>
         /// <returns>byte value.</returns>
         /// <exception cref="Exception">Thrown if page type is not found.</exception>
-        public static byte GetPageType(void* aPtr)
+        public static PageType GetPageType(void* aPtr)
         {
             if(aPtr < RamStart || aPtr > HeapEnd)
             {
                 return PageType.Empty;
             }
-            return mRAT[GetFirstRATIndex(aPtr)];
+            return (PageType)mRAT[GetFirstRATIndex(aPtr)];
         }
 
         /// <summary>
@@ -295,14 +294,14 @@ public static byte GetPageType(void* aPtr)
         public static void Free(uint aPageIdx)
         {
             byte* p = mRAT + aPageIdx;
-            *p = PageType.Empty;
+            *p = (byte)PageType.Empty;
             for (; p < mRAT + TotalPageCount; )
             {
-                if (*++p != PageType.Extension)
+                if (*++p != (byte)PageType.Extension)
                 {
                     break;
                 }
-                *p = PageType.Empty;
+                *p = (byte)PageType.Empty;
             }
         }
     }