how to get goroutine id with ebpf

huangapple
go评论118阅读模式
英文:

how to get goroutine id with ebpf

问题

我使用cilium ebpf包来编写一个用于获取goroutine id的ebpf程序，但是失败了。我的uprobe.c文件如下所示：

  1. SEC("uprobe/runtime.newproc1")
  2. int uprobe_runtime_newproc1(struct pt_regs *ctx) {
  3.     u32 key     = 2;
  4.     u64 initval = 1, *valp;
  6.     valp = bpf_map_lookup_elem(&uprobe_map, &key);
  7.     if (!valp) {
  8.         bpf_map_update_elem(&uprobe_map, &key, &initval, BPF_ANY);
  9.         return 0;
  10.     }
  11.     __sync_fetch_and_add(valp, 1);
  13.     struct g* goroutine_struct = (void *)PT_REGS_PARM4(ctx);
  15.     // retrieve output parameter
  16.     s64 goid = 0;
  17.     bpf_probe_read(&goid, sizeof(goid), &goroutine_struct->goid);
  19.     bpf_printk("bpf_printk bpf_probe_read goroutine_struct->goid: %lld", goid);
  21.     struct g gs;
  22.     bpf_probe_read(&gs, sizeof(gs), (void *)PT_REGS_PARM4(ctx));
  23.     bpf_printk("bpf_printk bpf_probe_read goroutine_struct.goid: %lld", gs.goid);
  25.     // test
  26.     void* ptr = (void *)PT_REGS_PARM4(ctx);
  27.     s64 goid2 = 0;
  28.     bpf_probe_read(&goid2, sizeof(goid2), (void *)(ptr+152));
  29.     bpf_printk("bpf_printk bpf_probe_read goid2: %lld", goid2);
  31.     return 0;
  32. }

goroutine.h文件如下所示：

  1. #include "common.h"
  3. struct stack  {
  4.     u64 lo;
  5.     u64 hi;
  6. };
  8. struct gobuf  {
  9.     u64 sp;
  10.     u64 pc;
  11.     u64 g;
  12.     u64 ctxt;
  13.     u64 ret;
  14.     u64 lr;
  15.     u64 bp;
  16. };
  18. struct g {
  19.     struct stack stack;
  20.     u64 stackguard0;
  21.     u64 stackguard1;
  23.     u64 _panic;
  24.     u64 _defer;
  25.     u64 m;
  26.     struct gobuf sched ;
  27.     u64 syscallsp;
  28.     u64 syscallpc;
  29.     u64 stktopsp;
  30.     u64 param;
  31.     u32 atomicstatus;
  32.     u32 stackLock;
  33.     s64 goid;          // 这里就是goroutine id！
  34. };

当我运行程序时，cat /sys/kernel/debug/tracing/trace_pipe输出如下，得到了错误的goroutine id：

  1. <...>-1336127 [000] d... 20113210.986990: bpf_trace_printk: bpf_printk bpf_probe_read goroutine_struct->goid: 4938558469562467144
  2. <...>-1336127 [000] d... 20113210.986998: bpf_trace_printk: bpf_printk bpf_probe_read goroutine_struct.goid: 4938558469562467144
  3. <...>-1336127 [000] d... 20113210.986998: bpf_trace_printk: bpf_printk bpf_probe_read goid2: 4938558469562467144

根据你的描述，你认为问题的关键在于将golang结构体g转换为goroutine.h时出错。你是否需要帮助？

英文:

I use cilium ebpf pakage to write a ebpf program for getting the goroutine id.
but failed. my uprobe.c like this :
I think the key problem is that golang struct g trans to goroutine.h is wrong. can anyone help?

uprobe.c

  1. SEC(&quot;uprobe/runtime.newproc1&quot;)
  2. int uprobe_runtime_newproc1(struct pt_regs *ctx) {
  3. 	u32 key     = 2;
  4. 	u64 initval = 1, *valp;
  6. 	valp = bpf_map_lookup_elem(&amp;uprobe_map, &amp;key);
  7. 	if (!valp) {
  8. 		bpf_map_update_elem(&amp;uprobe_map, &amp;key, &amp;initval, BPF_ANY);
  9. 		return 0;
  10. 	}
  11. 	__sync_fetch_and_add(valp, 1);
  13. 	struct g* goroutine_struct = (void *)PT_REGS_PARM4(ctx);
  15.     // retrieve output parameter
  16.     s64 goid = 0;
  17.     bpf_probe_read(&amp;goid, sizeof(goid), &amp;goroutine_struct-&gt;goid);
  19.     bpf_printk(&quot;bpf_printk bpf_probe_read goroutine_struct-&gt;goid: %lld&quot;, goid);
  22.     struct g gs;
  23.     bpf_probe_read(&amp;gs, sizeof(gs), (void *)PT_REGS_PARM4(ctx));
  24.     bpf_printk(&quot;bpf_printk bpf_probe_read goroutine_struct.goid: %lld&quot;, gs.goid);
  26.     // test
  27.     void* ptr = (void *)PT_REGS_PARM4(ctx);
  28.     s64 goid2 = 0;
  29.     bpf_probe_read(&amp;goid2, sizeof(goid2), (void *)(ptr+152));
  30.     bpf_printk(&quot;bpf_printk bpf_probe_read goid2: %lld&quot;, goid2);
  32. 	return 0;
  33. }

goroutine.h

  1. #include &quot;common.h&quot;
  5. struct stack  {
  6. 	u64 lo;
  7. 	u64 hi;
  8. };
  10. struct gobuf  {
  11. 	u64 sp;
  12. 	u64 pc;
  13. 	u64 g;
  14. 	u64 ctxt;
  15. 	u64 ret;
  16. 	u64 lr;
  17. 	u64 bp;
  18. };
  21. /*
  22. go version go1.17.2 linux/amd64
  24. type stack struct {
  25. 	lo uintptr
  26. 	hi uintptr
  27. }
  29. type gobuf struct {
  30. 	sp   uintptr
  31. 	pc   uintptr
  32. 	g    uintptr
  33. 	ctxt uintptr
  34. 	ret  uintptr
  35. 	lr   uintptr
  36. 	bp   uintptr
  37. }
  39. type g struct {	
  40. 	stack       stack   // offset known to runtime/cgo
  41. 	stackguard0 uintptr // offset known to liblink
  42. 	stackguard1 uintptr // offset known to liblink
  44. 	_panic    *_panic // innermost panic - offset known to liblink
  45. 	_defer    *_defer // innermost defer
  46. 	m         *m      // current m; offset known to arm liblink
  47. 	sched     gobuf
  48. 	syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc
  49. 	syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc
  50. 	stktopsp  uintptr // expected sp at top of stack, to check in traceback
  51. 	
  52. 	param        unsafe.Pointer
  53. 	atomicstatus uint32
  54. 	stackLock    uint32 // sigprof/scang lock; TODO: fold in to atomicstatus
  55. 	goid         int64
  56. }
  58. */
  59. struct g {
  60. 	struct stack stack;
  61. 	u64 stackguard0;
  62. 	u64 stackguard1;
  64. 	u64 _panic;
  65. 	u64 _defer;
  66. 	u64 m;
  67. 	struct gobuf sched ;
  68. 	u64 syscallsp;
  69. 	u64 syscallpc;
  70. 	u64 stktopsp;
  71. 	u64 param;
  72. 	u32 atomicstatus;
  73. 	u32 stackLock;
  74. 	s64 goid;          // Here it is!
  75. };

When I run my program , cat /sys/kernel/debug/tracing/trace_pipe output like this , get the wrong go id:

  1. &lt;...&gt;-1336127 [000] d... 20113210.986990: bpf_trace_printk: bpf_printk bpf_probe_read goroutine_struct-&gt;goid: 4938558469562467144
  2. &lt;...&gt;-1336127 [000] d... 20113210.986998: bpf_trace_printk: bpf_printk bpf_probe_read goroutine_struct.goid: 4938558469562467144
  3. &lt;...&gt;-1336127 [000] d... 20113210.986998: bpf_trace_printk: bpf_printk bpf_probe_read goid2: 4938558469562467144
  4. Blockquote

答案1

得分: 1

我找到了一个解决方案：

  1. 我的 Golang 版本是 1.17.2，amd64 架构。amd64 架构使用以下 9 个寄存器来传递整数参数和结果：RAX、RBX、RCX、RDI、RSI、R8、R9、R10、R11。

  2. 在 Go 1.17.2 中，runtime.newproc1 函数有 5 个参数。callergp *g 是第 4 个参数。当我使用 gdb 调试我的用户空间程序时，它使用 rdi 寄存器来保存 callergp *g 的指针地址。所以使用 PT_REGS_PARM1 是正确的方式，因为 (#define PT_REGS_PARM1(x) ((x)->rdi))。

  3. 最后，代码如下：

  1.     SEC("uprobe/runtime.newproc1")
  2.     int uprobe_runtime_newproc1(struct pt_regs *ctx) {
  3.         u32 key     = 2;
  4.         u64 initval = 1, *valp;
  6.         valp = bpf_map_lookup_elem(&uprobe_map, &key);
  7.         if (!valp) {
  8.             bpf_map_update_elem(&uprobe_map, &key, &initval, BPF_ANY);
  9.             return 0;
  10.         }
  11.         __sync_fetch_and_add(valp, 1);
  13.         // retrieve output parameter
  14.         struct g gs;
  15.         bpf_probe_read(&gs, sizeof(gs), (void *)PT_REGS_PARM1(ctx));
  16.         bpf_printk("uprobe_runtime_newproc1 bpf_printk bpf_probe_read goroutine_struct.goid: %lld", gs.goid);
  18.         return 0;
  19.     }
英文:

I found a solution:

  1. my golang version is 1.17.2, amd64. and the amd64 architecture uses the following sequence of 9 registers for integer arguments and results:
    RAX, RBX, RCX, RDI, RSI, R8, R9, R10, R11

  2. runtime.newproc1 which in go 1.17.2 has 5 args. callergp *g is the 4th. when I gdb my userspace program, it use rdi register to save ptr addr of callergp *g.
    so use PT_REGS_PARM1 is the right way. because (#define PT_REGS_PARM1(x) ((x)->rdi))

  3. after all, the code like this :

  1.     SEC(&quot;uprobe/runtime.newproc1&quot;)
  2.     int uprobe_runtime_newproc1(struct pt_regs *ctx) {
  3.     	u32 key     = 2;
  4.     	u64 initval = 1, *valp;
  5.     
  6.     	valp = bpf_map_lookup_elem(&amp;uprobe_map, &amp;key);
  7.     	if (!valp) {
  8.     		bpf_map_update_elem(&amp;uprobe_map, &amp;key, &amp;initval, BPF_ANY);
  9.     		return 0;
  10.     	}
  11.     	__sync_fetch_and_add(valp, 1);
  12.     
  13.         // retrieve output parameter
  14.         struct g gs;
  15.         bpf_probe_read(&amp;gs, sizeof(gs), (void *)PT_REGS_PARM1(ctx));
  16.         bpf_printk(&quot;uprobe_runtime_newproc1 bpf_printk bpf_probe_read goroutine_struct.goid: %lld&quot;, gs.goid);
  17.     
  18.     	return 0;
  19.     }

huangapple
  • 本文由 发表于 2022年11月10日 11:27:02
  • 转载请务必保留本文链接：https://go.coder-hub.com/74383816.html
  • bpf
  • cilium
  • ebpf
  • go
  • goroutine
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定