Merge pull request #10348 from theotherjimmy/fix-memap-m33

Parse M33 + GCC_ARM map files

Author: Cruz Monrreal

tools/memap.py

@@ -24,11 +24,6 @@
 from os import sep
 from os.path import (basename, dirname, join, relpath, abspath, commonprefix,
                      splitext)
-
-# Be sure that the tools directory is in the search path
-ROOT = abspath(join(dirname(__file__), ".."))
-path.insert(0, ROOT)
-
 import re
 import csv
 import json
@@ -39,8 +34,16 @@
 from jinja2 import FileSystemLoader, StrictUndefined
 from jinja2.environment import Environment
 
-from tools.utils import (argparse_filestring_type, argparse_lowercase_hyphen_type,
-                         argparse_uppercase_type)
+
+# Be sure that the tools directory is in the search path
+ROOT = abspath(join(dirname(__file__), ".."))
+path.insert(0, ROOT)
+
+from tools.utils import (
+    argparse_filestring_type,
+    argparse_lowercase_hyphen_type,
+    argparse_uppercase_type
+)  # noqa: E402
 
 
 class _Parser(object):
@@ -105,33 +108,37 @@ def parse_mapfile(self, mapfile):
 
 class _GccParser(_Parser):
     RE_OBJECT_FILE = re.compile(r'^(.+\/.+\.o(bj)?)$')
-    RE_LIBRARY_OBJECT = re.compile(r'^.+' + r''.format(sep) + r'lib((.+\.a)\((.+\.o(bj)?)\))$')
+    RE_LIBRARY_OBJECT = re.compile(
+        r'^.+' + r''.format(sep) + r'lib((.+\.a)\((.+\.o(bj)?)\))$'
+    )
     RE_STD_SECTION = re.compile(r'^\s+.*0x(\w{8,16})\s+0x(\w+)\s(.+)$')
     RE_FILL_SECTION = re.compile(r'^\s*\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$')
     OBJECT_EXTENSIONS = (".o", ".obj")
 
-    ALL_SECTIONS = _Parser.SECTIONS + _Parser.OTHER_SECTIONS + \
-                   _Parser.MISC_FLASH_SECTIONS + ('unknown', 'OUTPUT')
+    ALL_SECTIONS = (
+        _Parser.SECTIONS
+        + _Parser.OTHER_SECTIONS
+        + _Parser.MISC_FLASH_SECTIONS
+        + ('unknown', 'OUTPUT')
+    )
 
     def check_new_section(self, line):
         """ Check whether a new section in a map file has been detected
 
         Positional arguments:
         line - the line to check for a new section
 
-        return value - A section name, if a new section was found, False
+        return value - A section name, if a new section was found, None
                        otherwise
         """
+        line_s = line.strip()
         for i in self.ALL_SECTIONS:
-            if line.startswith(i):
-                # should name of the section (assuming it's a known one)
+            if line_s.startswith(i):
                 return i
-
         if line.startswith('.'):
-            return 'unknown'     # all others are classified are unknown
+            return 'unknown'
         else:
-            return False         # everything else, means no change in section
-
+            return None
 
     def parse_object_name(self, line):
         """ Parse a path to object file
@@ -158,8 +165,10 @@ def parse_object_name(self, line):
                 return join('[lib]', test_re_obj_name.group(2),
                             test_re_obj_name.group(3))
             else:
-                if (not line.startswith("LONG") and
-                    not line.startswith("linker stubs")):
+                if (
+                    not line.startswith("LONG") and
+                    not line.startswith("linker stubs")
+                ):
                     print("Unknown object name found in GCC map file: %s"
                           % line)
                 return '[misc]'
@@ -168,8 +177,8 @@ def parse_section(self, line):
         """ Parse data from a section of gcc map file
 
         examples:
-                        0x00004308       0x7c ./BUILD/K64F/GCC_ARM/mbed-os/hal/targets/hal/TARGET_Freescale/TARGET_KPSDK_MCUS/spi_api.o
-         .text          0x00000608      0x198 ./BUILD/K64F/GCC_ARM/mbed-os/core/mbed-rtos/rtx/TARGET_CORTEX_M/TARGET_RTOS_M4_M7/TOOLCHAIN/HAL_CM4.o
+                        0x00004308       0x7c ./BUILD/K64F/GCC_ARM/spi_api.o
+         .text          0x00000608      0x198 ./BUILD/K64F/HAL_CM4.o
 
         Positional arguments:
         line - the line to parse a section from
@@ -215,7 +224,11 @@ def parse_mapfile(self, file_desc):
                 self.module_add(object_name, object_size, current_section)
 
         common_prefix = dirname(commonprefix([
-            o for o in self.modules.keys() if (o.endswith(self.OBJECT_EXTENSIONS) and not o.startswith("[lib]"))]))
+            o for o in self.modules.keys()
+            if (
+                    o.endswith(self.OBJECT_EXTENSIONS)
+                    and not o.startswith("[lib]")
+            )]))
         new_modules = {}
         for name, stats in self.modules.items():
             if name.startswith("[lib]"):
@@ -245,9 +258,13 @@ def parse_object_name(self, line):
         else:
             is_obj = re.match(self.RE_OBJECT, line)
             if is_obj:
-                return join('[lib]', basename(is_obj.group(1)), is_obj.group(3))
+                return join(
+                    '[lib]', basename(is_obj.group(1)), is_obj.group(3)
+                )
             else:
-                print("Malformed input found when parsing ARMCC map: %s" % line)
+                print(
+                    "Malformed input found when parsing ARMCC map: %s" % line
+                )
                 return '[misc]'
 
     def parse_section(self, line):
@@ -260,7 +277,7 @@ def parse_section(self, line):
 
         Positional arguments:
         line - the line to parse the section data from
-        """
+        """  # noqa: E501
         test_re = re.match(self.RE, line)
 
         if test_re:
@@ -276,8 +293,10 @@ def parse_section(self, line):
                 elif test_re.group(3) == 'Code':
                     section = '.text'
                 else:
-                    print("Malformed input found when parsing armcc map: %s, %r"
-                          % (line, test_re.groups()))
+                    print(
+                        "Malformed input found when parsing armcc map: %s, %r"
+                        % (line, test_re.groups())
+                    )
 
                     return ["", 0, ""]
 
@@ -307,10 +326,20 @@ def parse_mapfile(self, file_desc):
                 self.module_add(*self.parse_section(line))
 
         common_prefix = dirname(commonprefix([
-            o for o in self.modules.keys() if (o.endswith(self.OBJECT_EXTENSIONS) and o != "anon$$obj.o" and o != "anon$$obj.obj" and not o.startswith("[lib]"))]))
+            o for o in self.modules.keys()
+            if (
+                o.endswith(self.OBJECT_EXTENSIONS)
+                and o != "anon$$obj.o"
+                and o != "anon$$obj.obj"
+                and not o.startswith("[lib]")
+            )]))
         new_modules = {}
         for name, stats in self.modules.items():
-            if name == "anon$$obj.o" or name == "anon$$obj.obj" or name.startswith("[lib]"):
+            if (
+                name == "anon$$obj.o"
+                or name == "anon$$obj.obj"
+                or name.startswith("[lib]")
+            ):
                 new_modules[name] = stats
             elif name.endswith(self.OBJECT_EXTENSIONS):
                 new_modules[relpath(name, common_prefix)] = stats
@@ -365,11 +394,13 @@ def parse_section(self, line):
 
         Positional_arguments:
         line - the line to parse section data from
-        """
+        """  # noqa: E501
         test_re = re.match(self.RE, line)
         if test_re:
-            if (test_re.group(2) == 'const' or
-                test_re.group(2) == 'ro code'):
+            if (
+                test_re.group(2) == 'const' or
+                test_re.group(2) == 'ro code'
+            ):
                 section = '.text'
             elif (test_re.group(2) == 'zero' or
                   test_re.group(2) == 'uninit'):
@@ -378,7 +409,7 @@ def parse_section(self, line):
                 elif test_re.group(1)[0:6] == 'CSTACK':
                     section = '.stack'
                 else:
-                    section = '.bss' #  default section
+                    section = '.bss'  # default section
 
             elif test_re.group(2) == 'inited':
                 section = '.data'
@@ -409,7 +440,8 @@ def check_new_library(self, line):
 
     def check_new_object_lib(self, line):
         """
-        Searches for objects within a library section and returns name. Example:
+        Searches for objects within a library section and returns name.
+        Example:
         rt7M_tl.a: [44]
             ABImemclr4.o                 6
             ABImemcpy_unaligned.o      118
@@ -435,7 +467,10 @@ def parse_command_line(self, lines):
                 break
             for arg in line.split(" "):
                 arg = arg.rstrip(" \n")
-                if (not arg.startswith("-")) and arg.endswith(self.OBJECT_EXTENSIONS):
+                if (
+                    not arg.startswith("-")
+                    and arg.endswith(self.OBJECT_EXTENSIONS)
+                ):
                     self.cmd_modules[basename(arg)] = arg
 
         common_prefix = dirname(commonprefix(list(self.cmd_modules.values())))
@@ -458,7 +493,7 @@ def parse_mapfile(self, file_desc):
             for line in infile:
                 self.module_add(*self.parse_section(line))
 
-                if line.startswith('*** MODULE SUMMARY'): # finish section
+                if line.startswith('*** MODULE SUMMARY'):  # finish section
                     break
 
             current_library = ""
@@ -484,7 +519,6 @@ class MemapParser(object):
     print_sections = ('.text', '.data', '.bss')
     delta_sections = ('.text-delta', '.data-delta', '.bss-delta')
 
-
     # sections to print info (generic for all toolchains)
     sections = _Parser.SECTIONS
     misc_flash_sections = _Parser.MISC_FLASH_SECTIONS
@@ -498,7 +532,6 @@ def __init__(self):
         # short version with specific depth
         self.short_modules = dict()
 
-
         # Memory report (sections + summary)
         self.mem_report = []
 
@@ -528,7 +561,7 @@ def reduce_depth(self, depth):
         mbed-os/drivers
 
         """
-        if depth == 0 or depth == None:
+        if depth == 0 or depth is None:
             self.short_modules = deepcopy(self.modules)
         else:
             self.short_modules = dict()
@@ -539,8 +572,9 @@ def reduce_depth(self, depth):
                 new_name = join(*split_name[:depth])
                 self.short_modules.setdefault(new_name, defaultdict(int))
                 for section_idx, value in v.items():
-                    self.short_modules[new_name][section_idx] += self.modules[module_name][section_idx]
-                    self.short_modules[new_name][section_idx + '-delta'] += self.modules[module_name][section_idx]
+                    self.short_modules[new_name][section_idx] += value
+                    delta_name = section_idx + '-delta'
+                    self.short_modules[new_name][delta_name] += value
             if self.old_modules:
                 for module_name, v in self.old_modules.items():
                     split_name = module_name.split(sep)
@@ -549,7 +583,8 @@ def reduce_depth(self, depth):
                     new_name = join(*split_name[:depth])
                     self.short_modules.setdefault(new_name, defaultdict(int))
                     for section_idx, value in v.items():
-                        self.short_modules[new_name][section_idx + '-delta'] -= self.old_modules[module_name][section_idx]
+                        delta_name = section_idx + '-delta'
+                        self.short_modules[new_name][delta_name] -= value
 
     export_formats = ["json", "csv-ci", "html", "table"]
 
@@ -657,7 +692,10 @@ def generate_html(self, file_desc):
                     if not modules:
                         break
                     next_module = modules.pop(0)
-                    if not any(cld['name'] == next_module for cld in cur_text['children']):
+                    if not any(
+                        cld['name'] == next_module
+                        for cld in cur_text['children']
+                    ):
                         break
                     cur_text = self._move_up_tree(cur_text, next_module)
                     cur_data = self._move_up_tree(cur_data, next_module)
@@ -759,8 +797,10 @@ def generate_table(self, file_desc):
             row = [i]
 
             for k in self.print_sections:
-                row.append("{}({:+})".format(self.short_modules[i][k],
-                                             self.short_modules[i][k + "-delta"]))
+                row.append("{}({:+})".format(
+                    self.short_modules[i][k],
+                    self.short_modules[i][k + "-delta"]
+                ))
 
             table.add_row(row)
 
@@ -815,7 +855,7 @@ def compute_report(self):
             for name, sizes in sorted(self.short_modules.items()):
                 self.mem_report.append({
                     "module": name,
-                    "size":{
+                    "size": {
                         k: sizes.get(k, 0) for k in (self.print_sections +
                                                      self.delta_sections)
                     }
@@ -855,6 +895,7 @@ def parse(self, mapfile, toolchain):
             print("I/O error({0}): {1}".format(error.errno, error.strerror))
             return False
 
+
 def main():
     """Entry Point"""
     version = '0.4.0'
@@ -912,16 +953,20 @@ def main():
 
     returned_string = None
     # Write output in file
-    if args.output != None:
-        returned_string = memap.generate_output(args.export, \
-            depth, args.output)
-    else: # Write output in screen
+    if args.output is not None:
+        returned_string = memap.generate_output(
+            args.export,
+            depth,
+            args.output
+        )
+    else:  # Write output in screen
         returned_string = memap.generate_output(args.export, depth)
 
     if args.export == 'table' and returned_string:
         print(returned_string)
 
     exit(0)
 
+
 if __name__ == "__main__":
     main()