From 72965b6f3cf33d5da326ffd7d291cf7e0ccd9092 Mon Sep 17 00:00:00 2001 From: ekyooo Date: Thu, 9 Jan 2025 08:19:00 +0900 Subject: [PATCH] tools/profile: Add additional information to backtrace (#5109) Add additional information and change the format of the backtrace: - Include the symbol base offset and DSO name. - Symbol and DSO information is included if it's available in the target binary. - Format: [SYMBOL+OFFSET] (MODULE) - Add a -A(or --addr) option for a detailed format that includes the address: ADDR [SYMBOL+OFFSET] (MODULE) before: # ./profile.py [unknown] _PyDict_GetItem_KnownHash [unknown] [unknown] [unknown] [unknown] _PyObject_MakeTpCall _PyObject_CallFunction_SizeT [unknown] _PyEval_EvalFrameDefault PyEval_EvalCode [unknown] [unknown] _PyRun_SimpleFileObject _PyRun_AnyFileObject Py_RunMain Py_BytesMain __libc_start_call_main __libc_start_main_impl _start - python3 (384002) 1 [unknown] [unknown] [unknown] [unknown] [unknown] [unknown] - gnome-shell (2302) 1 After: # ./profile.py runtime.typePointers.next+0x4b [grafana] runtime.gcDrain+0x1f4 [grafana] runtime.gcBgMarkWorker.func2+0xa5 [grafana] runtime.systemstack.abi0+0x4a [grafana] runtime.gcBgMarkWorker+0x1f2 [grafana] runtime.goexit.abi0+0x1 [grafana] - grafana (25858) 1 runtime.nanotime1.abi0+0x89 [containerd] runtime.sysmon+0x24b [containerd] runtime.mstart1+0x95 [containerd] runtime.mstart0+0x76 [containerd] runtime.mstart.abi0+0x5 [containerd] [unknown] - containerd (454924) 1 __get_user_8+0x10 [kernel] __get_user_8+0x10 [kernel] rseq_ip_fixup+0x69 [kernel] __rseq_handle_notify_resume+0x2b [kernel] syscall_exit_to_user_mode+0x1f9 [kernel] do_syscall_64+0x8c [kernel] entry_SYSCALL_64_after_hwframe+0x78 [kernel] runtime.futex.abi0+0x23 [dockerd] runtime.notesleep+0x87 [dockerd] runtime.stopm+0x8c [dockerd] runtime.findRunnable+0xd5f [dockerd] runtime.schedule+0xb1 [dockerd] runtime.park_m+0x1ec [dockerd] runtime.mcall+0x50 [dockerd] runtime.gopark+0xce [dockerd] runtime.(*scavengerState).park+0x49 [dockerd] runtime.bgscavenge+0x59 [dockerd] runtime.gcenable.gowrap2+0x25 [dockerd] runtime.goexit.abi0+0x1 [dockerd] - dockerd (144352) 1 # ./profile.py -A 0xffffffff945afbb8 __get_user_nocheck_4+0x8 [kernel] 0xffffffff945afbb8 __get_user_nocheck_4+0x8 [kernel] 0xffffffff936267be futex_wait_setup+0x7e [kernel] 0xffffffff9362697b __futex_wait+0x13b [kernel] 0xffffffff93626a94 futex_wait+0x74 [kernel] 0xffffffff93622765 do_futex+0x105 [kernel] 0xffffffff9362302a __x64_sys_futex+0x12a [kernel] 0xffffffff934071a7 x64_sys_call+0x1ce7 [kernel] 0xffffffff9462601f do_syscall_64+0x7f [kernel] 0xffffffff94800130 entry_SYSCALL_64_after_hwframe+0x78 [kernel] 0x00007df292898d61 __GI___futex_abstimed_wait_cancelable64+0xe1 [libc.so.6] 0x00007df29289bc7e pthread_cond_timedwait@@GLIBC_2.3.2+0x23e [libc.so.6] 0x0000645da1a160fa os_event::timed_wait(timespec const*)+0x3a [mysqld] 0x0000645da1a2f551 os_event::wait_time_low(std::chrono::duration >, long)+0x81 [mysqld] 0x0000645da19e44a9 log_files_governor(log_t*)+0x159 [mysqld] 0x0000645da19efb19 [unknown] [mysqld] 0x00007df292cecdb4 [unknown] [libstdc++.so.6.0.33] 0x00007df29289ca94 start_thread+0x384 [libc.so.6] 0x00007df292929c3c __GI___clone3+0x2c [libc.so.6] - ib_log_files_g (28830) 1 --- tools/profile.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/profile.py b/tools/profile.py index c3881f7d9561..49b007bfa860 100755 --- a/tools/profile.py +++ b/tools/profile.py @@ -130,6 +130,8 @@ def stack_id_err(stack_id): help="trace cgroups in this BPF map only") parser.add_argument("--mntnsmap", help="trace mount namespaces in this BPF map only") +parser.add_argument("-A", "--address", action="store_true", + help="show raw addresses") # option logic args = parser.parse_args() @@ -393,7 +395,12 @@ def aksym(addr): print(" [Missed Kernel Stack]") else: for addr in kernel_stack: - print(" %s" % aksym(addr).decode('utf-8', 'replace')) + sym_info = b.ksym(addr, True, True).decode('utf-8', 'replace') + if args.address: + print(" 0x%-16x %s" % (addr, sym_info)) + else: + print(" %s" % sym_info) + if not args.kernel_stacks_only: if need_delimiter and k.user_stack_id >= 0 and k.kernel_stack_id >= 0: print(" --") @@ -401,7 +408,11 @@ def aksym(addr): print(" [Missed User Stack]") else: for addr in user_stack: - print(" %s" % b.sym(addr, k.pid).decode('utf-8', 'replace')) + sym_info = b.sym(addr, k.pid, True, True).decode('utf-8', 'replace') + if args.address: + print(" 0x%016x %s" % (addr, sym_info)) + else: + print(" %s" % sym_info) print(" %-16s %s (%d)" % ("-", k.name.decode('utf-8', 'replace'), k.pid)) print(" %d\n" % v.value)