How to symbolicate Hermes crash stack symbol?

[iCodeWoods]

Description

As the document states, there are Hermes symbols in the assets, but only certain versions have them (e.g., 0.70.7), while others don't (e.g., 0.71.7).

How can I obtain the Hermes dSYM for version 0.71.7?

Below is the crash report.

Exception Type:  SIGABRT
Exception Codes: #0 at 0x1e9f0e558
Crashed Thread:  2

Thread 2 Crashed:
0   libsystem_kernel.dylib               0x00000001e9f0e558 __pthread_kill + 8
1   libsystem_c.dylib                    0x00000001b24f3178 abort + 176
2   libsystem_malloc.dylib               0x00000001b945b0e4 malloc_vreport + 904
3   libsystem_malloc.dylib               0x00000001b945b38c malloc_zone_error + 100
4   libsystem_malloc.dylib               0x00000001b9455658 nanov2_guard_corruption_detected + 40
5   libsystem_malloc.dylib               0x00000001b9453810 nanov2_allocate_outlined + 400
6   hermes                               0x00000001068d1ab0 0x1066a8000 + 2267824
7   hermes                               0x00000001066e1ef0 0x1066a8000 + 237296
8   hermes                               0x00000001066c8250 0x1066a8000 + 131664
9   hermes                               0x00000001067054dc 0x1066a8000 + 382172
10  hermes                               0x000000010670c8a4 0x1066a8000 + 411812
11  hermes                               0x00000001066ccc7c 0x1066a8000 + 150652
12  hermes                               0x0000000106707c00 0x1066a8000 + 392192
13  hermes                               0x0000000106708164 0x1066a8000 + 393572
14  hermes                               0x00000001066f0290 0x1066a8000 + 295568
15  hermes                               0x00000001066ed71c 0x1066a8000 + 284444
16  hermes                               0x00000001066cf2d8 0x1066a8000 + 160472
17  hermes                               0x00000001066ceb40 0x1066a8000 + 158528
18  hermes                               0x00000001066ee284 0x1066a8000 + 287364
19  hermes                               0x00000001066ed71c 0x1066a8000 + 284444
20  hermes                               0x00000001066cf2d8 0x1066a8000 + 160472
21  hermes                               0x00000001066ceb40 0x1066a8000 + 158528
22  hermes                               0x00000001066b3ce4 0x1066a8000 + 48356
23  Skype4Life                           0x0000000102b617fc facebook::jsi::WithRuntimeDecorator<facebook::react::(anonymous namespace)::ReentrancyCheck, facebook::jsi::Runtime, facebook::jsi::Runtime>::call(facebook::jsi::Function const&, facebook::jsi::Value const&, facebook::jsi::Value const*, unsigned long) (decorator.h:337)
24  Skype4Life                           0x0000000102b9118c facebook::jsi::Value facebook::jsi::Function::call<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, facebook::jsi::Value>(facebook::jsi::Runtime&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&&&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&&&, facebook::jsi::Value&&) const (jsi-inl.h:257)
25  Skype4Life                           0x0000000102b90ff0 std::__1::__function::__func<facebook::react::JSIExecutor::callFunction(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, folly::dynamic const&)::$_5, std::__1::allocator<facebook::react::JSIExecutor::callFunction(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, folly::dynamic const&)::$_5>, void ()>::operator()() (JSIExecutor.cpp:253)
26  Skype4Life                           0x0000000102a96724 void std::__1::__invoke_void_return_wrapper<void, true>::__call<void (*&)(std::__1::function<void ()> const&, std::__1::function<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > ()>), std::__1::function<void ()> const&, std::__1::function<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > ()> >(void (*&&&)(std::__1::function<void ()> const&, std::__1::function<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > ()>), std::__1::function<void ()> const&&&, std::__1::function<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > ()>&&) (type_traits:3918)
27  Skype4Life                           0x0000000102b8e1e8 facebook::react::JSIExecutor::callFunction(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, folly::dynamic const&) (function.h:505)
28  Skype4Life                           0x0000000102b3d684 std::__1::__function::__func<facebook::react::NativeToJsBridge::runOnExecutorQueue(std::__1::function<void (facebook::react::JSExecutor*)>)::$_8, std::__1::allocator<facebook::react::NativeToJsBridge::runOnExecutorQueue(std::__1::function<void (facebook::react::JSExecutor*)>)::$_8>, void ()>::operator()() (function.h:505)
29  Skype4Life                           0x0000000102a99304 facebook::react::tryAndReturnError(std::__1::function<void ()> const&) (function.h:505)
30  Skype4Life                           0x0000000102aa53dc facebook::react::RCTMessageThread::tryFunc(std::__1::function<void ()> const&) (RCTMessageThread.mm:69)
31  Skype4Life                           0x0000000102aa5190 ___ZN8facebook5react16RCTMessageThread8runAsyncENSt3__18functionIFvvEEE_block_invoke (function.h:505)
32  CoreFoundation                       0x00000001ab0066e0 __CFRUNLOOP_IS_CALLING_OUT_TO_A_BLOCK__ + 24
33  CoreFoundation                       0x00000001ab06d210 __CFRunLoopDoBlocks + 360
34  CoreFoundation                       0x00000001ab03d0e8 __CFRunLoopRun + 844
35  CoreFoundation                       0x00000001ab0423ec CFRunLoopRunSpecific + 608
36  Skype4Life                           0x0000000102a8edc4 +[RCTCxxBridge runRunLoop] (RCTCxxBridge.mm:336)
37  Foundation                           0x00000001a52dc544 __NSThread__start__ + 712
38  libsystem_pthread.dylib              0x000000020ad516b8 _pthread_start + 144
39  libsystem_pthread.dylib              0x000000020ad50b88 thread_start + 4

React Native Version

0.71.7

Output of npx react-native info

info Fetching system and libraries information...
System:
    OS: macOS 13.4
    CPU: (10) x64 Apple M1 Pro
    Memory: 26.28 MB / 16.00 GB
    Shell: 5.9 - /bin/zsh
  Binaries:
    Node: 14.21.3 - ~/.nvm/versions/node/v14.21.3/bin/node
    Yarn: 1.22.19 - /opt/homebrew/bin/yarn
    npm: 6.14.18 - ~/.nvm/versions/node/v14.21.3/bin/npm
    Watchman: 2023.04.10.00 - /opt/homebrew/bin/watchman
  Managers:
    CocoaPods: 1.11.3 - /usr/local/bin/pod
  SDKs:
    iOS SDK:
      Platforms: DriverKit 22.4, iOS 16.4, macOS 13.3, tvOS 16.4, watchOS 9.4
    Android SDK: Not Found
  IDEs:
    Android Studio: Not Found
    Xcode: 14.3.1/14E300c - /usr/bin/xcodebuild
  Languages:
    Java: Not Found
  npmPackages:
    @react-native-community/cli: Not Found
    react: Not Found
    react-native: Not Found
    react-native-macos: Not Found
  npmGlobalPackages:
    *react-native*: Not Found
info React Native v0.72.1 is now available (your project is running on v0.71.7).
info Changelog: https://github.com/facebook/react-native/releases/tag/v0.72.1
info Diff: https://react-native-community.github.io/upgrade-helper/?from=0.71.7
info For more info, check out "https://reactnative.dev/docs/upgrading".

Steps to reproduce

Check https://github.com/facebook/react-native/releases?page=2 and there is no hermes artifacts in v0.71.7

Snack, code example, screenshot, or link to a repository

[github-actions]

⚠️

Newer Version of React Native is Available!

ℹ️

You are on a supported minor version, but it looks like there's a newer patch available - 0.71.11. Please upgrade to the highest patch for your minor or latest and verify if the issue persists (alternatively, create a new project and repro the issue in it). If it does not repro, please let us know so we can close out this issue. This helps us ensure we are looking at issues that still exist in the most recent releases.

[iCodeWoods]

Even if we use the newer patch - 0.71.11, it's still don't have hermes artifacts in the assets.

[cortinico]

For context, those artifacts are only for Android debug symbols. They've been removed from 0.71 as they're distributed on Maven Central alongside the React Native artifacts.

As for iOS, I'm unsure where the dSYM are stored (cc @cipolleschi).

[cipolleschi]

hum... I don't think we are generating dSYM anymore, unfortunately. Probably due to some bug or unwanted changes.
I'll have to look into that more deeply.
@dmytrorykun, do you have more information about this?

[dmytrorykun]

do you have more information about this?

@cipolleschi nope, looks like a bug to me.

[iCodeWoods]

Hi @cortinico @cipolleschi @dmytrorykun Thanks for your reply.
The ultimate goal of wanting the dSYM for Hermes is to resolve crashes. Do you have any thoughts on the crash stack trace i provided earlier? It contains several Hermes stack frames, but symbolizing them may not be very helpful. I have encountered similar issues before where crashes triggered by bridge messages end up with stack traces within React Native, but it's difficult to determine which bridge specifically caused the issue. Do you have any suggestions or recommendations for addressing this?

[iCodeWoods]

I understand that it's difficult to debug the issue without being able to reproduce it.

However, at least I would like to ask if there are any ideas regarding the crash triggered by +[RCTCxxBridge runRunLoop].

For example, should I fork a copy of the code and add some logs in this area? (But I think this may impact performance, so it may not be a good idea.)

[cipolleschi]

Can you reproduce the crash locally in a Debug version of the app? Or are these crashes happening only in production?
If you manage to reproduce the crash locally, does it also happen if you disable Hermes? (to rule out that the problem is in hermes).

Some logs would help: for example, the app is calling something in Javascript that is then forwarded to native, as shown by lines 25 and 27 (JSIExecutor::callFunction) but we can't have the function invoked in the stacktrace...

[iCodeWoods]

Hi @cipolleschi Thanks for your reply.

It's an issue that occurs in the production environment and cannot be reproduced locally. Before enabling Hermes, there was another issue in the production environment, but the quantity was only half of this one. After enabling Hermes, the previous issue disappeared, but this new issue appeared, with a quantity twice as much as before.

I would like to know if React Native has any commonly used or general solutions for this type of crash. Although the crash appears in the React Native stack trace, it may not indicate the specific cause, and it is possible that the issue is not related to React Native itself, but rather to the caller's code.

[cipolleschi]

Yeah, I understand. If a crash is happening with both engines, it makes me think that it is either something in the app or in React Native.
Looking at these lines, it seems that the symbols for Hermes are actually created.

But then, here those symbols are stripped.
I think that the idea was to strip them from there and to create a separate .tar with them.

So, one thing you can do while we work to restore them is to:

1. clone react-native repo
2. checkout the branch 0.71-stable
3. run yarn in the main folder
4. cd packages/rn-tester
5. bundle install
6. bundle exec pod install --> this will download the right version of hermes locally
7. cd ../react-native/sdks
8. BUILD_TYPE="Release" ./utils/build-ios-framework.sh --> This should build hermes for iPhone, iPhoneSimulator, Catalyst.

This should build the framework in the /tmp/hermes/hermes-runtime-darwin/destroot/ folder and you can look whether there is a dSYM folder that you can use.
We never update Hermes after 71.4, so the latest you can build from 0.71 is the same you should have in prod.

I really hope this helps.

[iCodeWoods]

Hi @cipolleschi Thanks for your help. I encountered some issues when using the command.

➜  hermes-engine git:(0.71-stable) BUILD_TYPE="Release" ./utils/build-ios-framework.sh
Building hermesc
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
make: Makefile: No such file or directory
make: *** No rule to make target `Makefile'.  Stop.
Host hermesc is required to build apple frameworks!
Building Release framework for iphoneos with architectures: arm64
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
Error: could not load cache
Building hermesc
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
make: Makefile: No such file or directory
make: *** No rule to make target `Makefile'.  Stop.
Host hermesc is required to build apple frameworks!
Building Release framework for iphonesimulator with architectures: x86_64;arm64
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
Error: could not load cache
Building hermesc
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
make: Makefile: No such file or directory
make: *** No rule to make target `Makefile'.  Stop.
Host hermesc is required to build apple frameworks!
Building Release framework for catalyst with architectures: x86_64;arm64
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
Error: could not load cache
/Users/linzhennan/react-native/sdks/hermes-engine/utils/build-apple-framework.sh: line 124: pushd: /Users/linzhennan/react-native/sdks/hermes-engine/utils/../destroot/Library/Frameworks: No such file or directory```