Unicode 16.0 Support (#111469)

* Unicode 16.0 Support

* Update encoding web files

* Fix failing test

Author: tarekgh

THIRD-PARTY-NOTICES.TXT

@@ -34,37 +34,36 @@ License notice for Unicode data
 
 https://www.unicode.org/license.html
 
-Copyright © 1991-2022 Unicode, Inc. All rights reserved.
+Copyright © 1991-2024 Unicode, Inc. All rights reserved.
 Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of the Unicode data files and any associated documentation
-(the "Data Files") or Unicode software and any associated documentation
-(the "Software") to deal in the Data Files or Software
-without restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, and/or sell copies of
-the Data Files or Software, and to permit persons to whom the Data Files
-or Software are furnished to do so, provided that either
-(a) this copyright and permission notice appear with all copies
-of the Data Files or Software, or
-(b) this copyright and permission notice appear in associated
-Documentation.
-
-THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
-ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT OF THIRD PARTY RIGHTS.
-IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
-NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
-DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
-DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THE DATA FILES OR SOFTWARE.
-
-Except as contained in this notice, the name of a copyright holder
-shall not be used in advertising or otherwise to promote the sale,
-use or other dealings in these Data Files or Software without prior
-written authorization of the copyright holder.
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of data files and any associated documentation (the "Data Files") or
+software and any associated documentation (the "Software") to deal in the
+Data Files or Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell
+copies of the Data Files or Software, and to permit persons to whom the
+Data Files or Software are furnished to do so, provided that either (a)
+this copyright and permission notice appear with all copies of the Data
+Files or Software, or (b) this copyright and permission notice appear in
+associated Documentation.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS.
+
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
+BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
+OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
+FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
 
 License notice for zlib-ng
 -----------------------

src/libraries/Common/tests/TestUtilities.Unicode/System/Text/Unicode/CodePointFlags.cs

@@ -47,5 +47,9 @@ public enum CodePointFlags : ulong
         Unified_Ideograph = 1ul << 31,
         Variation_Selector = 1ul << 32,
         White_Space = 1ul << 33,
+        IDS_Unary_Operator = 1ul << 34,
+        ID_Compat_Math_Continue = 1ul << 35,
+        ID_Compat_Math_Start = 1ul << 36,
+        Modifier_Combining_Mark = 1ul << 37,
     }
 }

src/libraries/Common/tests/TestUtilities.Unicode/System/Text/Unicode/ParsedUnicodeData.cs

@@ -137,10 +137,21 @@ private static Dictionary<int, string> ProcessDerivedNameFile()
                 {
                     if (value.IsSingleCodePoint)
                     {
-                        // Single code point of format "XXXX ; <Name>" (name shouldn't end with '*')
+                        // Single code point of format "XXXX ; <Name>"
+                        // It is possible to get a * at the end of the line, to add one codepoint to previously added range.
+                        // example:
+                        //     18B00..18CD5  ; KHITAN SMALL SCRIPT CHARACTER-*   <<<< IsSingleCodePoint is false
+                        //     18CFF         ; KHITAN SMALL SCRIPT CHARACTER-*   <<<< IsSingleCodePoint is true
 
-                        Assert.False(value.PropName.EndsWith('*'));
-                        dict.Add(value.FirstCodePoint, value.PropName);
+                        if (value.PropName.EndsWith('*'))
+                        {
+                            string baseName = value.PropName[..^1];
+                            dict.Add(value.FirstCodePoint, $"{baseName}{value.FirstCodePoint:X4}");
+                        }
+                        else
+                        {
+                            dict.Add(value.FirstCodePoint, value.PropName);
+                        }
                     }
                     else
                     {

src/libraries/Common/tests/TestUtilities.Unicode/TestUtilities.Unicode.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <UnicodeUcdVersion>15.0</UnicodeUcdVersion>
+    <UnicodeUcdVersion>16.0</UnicodeUcdVersion>
     <TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
   </PropertyGroup>
   <ItemGroup>

src/libraries/System.Private.CoreLib/Tools/GenUnicodeProp/GenUnicodeProp.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
     <OutputType>Exe</OutputType>
-    <UnicodeUcdVersion>15.0</UnicodeUcdVersion>
+    <UnicodeUcdVersion>16.0</UnicodeUcdVersion>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <NoWarn>$(NoWarn);CA1416</NoWarn>
     <EnableDefaultItems>true</EnableDefaultItems>

src/libraries/System.Private.CoreLib/Tools/GenUnicodeProp/Program.cs

@@ -204,14 +204,14 @@ private static void PrintValueArray<T>(string tableName, Dictionary<T, byte> d,
 
         private static void PrintByteArray(string tableName, StreamWriter file, byte[] str)
         {
-            file.Write("\n        private static ReadOnlySpan<byte> " + tableName + " => new byte[" + str.Length + "]\n        {\n");
+            file.Write("\n        private static ReadOnlySpan<byte> " + tableName + " => // " + str.Length + "\n        [\n");
             file.Write("            0x{0:x2}", str[0]);
             for (var i = 1; i < str.Length; i++)
             {
                 file.Write(i % 16 == 0 ? ",\n            " : ", ");
                 file.Write("0x{0:x2}", str[i]);
             }
-            file.Write("\n        };\n");
+            file.Write("\n        ];\n");
         }
 
         private static void PrintAssertTableLevelsBitCountRoutine(string tableName, StreamWriter file, TableLevels expectedLevels)