1、Install

安装systemtap

yum install systemtap systemtap-runtime

查看额外所需要安装的内核调试包：

stap-prep

一般需要安装的三个包是：

kernel-debuginfo
kernel-debuginfo-common
kernel-devel

安装完毕后简单测试一下：

stap -ve 'probe begin { log("hello world") exit() }'

2、Tricks

2.1、定位函数位置

查看内核中函数ip_rcv是在哪里定义的：

stap -l 'kernel.function("ip_rcv")'

可以使用*号来模糊查找：

stap -l 'kernel.function("*recv")'

同理，也可以用来定位用户进程的函数位置：

# stap -l 'process("/root/work/systemtap/nginx-installed/sbin/nginx").function("ngx_shm_alloc")'
process("/root/work/systemtap/nginx-installed/sbin/nginx").function("ngx_shm_alloc@src/os/unix/ngx_shmem.c:15")

2.2、查看可用探测点以及该探测点上可用的变量

在一些探测点上能获取的变量比较有限，这是因为这些变量可能已经被编译器优化掉了，优化掉的变量就获取不到了。一般先用-L参数来看看有哪些变量可以直接使用：

# stap -L 'process("/root/work/systemtap/nginx-installed/sbin/nginx").function("ngx_shm_alloc")'
process("/root/work/systemtap/nginx-installed/sbin/nginx").function("ngx_shm_alloc@src/os/unix/ngx_shmem.c:15") $shm:ngx_shm_t*

可见在该探测点上可以直接使用$shm这个变量，其类型是ngx_shm_t*。

statement探测点也类似：

# stap -L 'process("/root/work/systemtap/nginx-installed/sbin/nginx").statement("ngx_pcalloc@src/core/ngx_palloc.c:*")'
process("/root/work/systemtap/nginx-installed/sbin/nginx").statement("ngx_pcalloc@src/core/ngx_palloc.c:299") $pool:ngx_pool_t* $size:size_t
process("/root/work/systemtap/nginx-installed/sbin/nginx").statement("ngx_pcalloc@src/core/ngx_palloc.c:300") $pool:ngx_pool_t* $size:size_t
process("/root/work/systemtap/nginx-installed/sbin/nginx").statement("ngx_pcalloc@src/core/ngx_palloc.c:303") $pool:ngx_pool_t* $size:size_t $p:void*
process("/root/work/systemtap/nginx-installed/sbin/nginx").statement("ngx_pcalloc@src/core/ngx_palloc.c:304") $pool:ngx_pool_t* $size:size_t $p:void*
process("/root/work/systemtap/nginx-installed/sbin/nginx").statement("ngx_pcalloc@src/core/ngx_palloc.c:307") $pool:ngx_pool_t* $size:size_t $p:void*

2.3、输出调用堆栈

用户态探测点堆栈：print_ubacktrace()、sprint_ubacktrace() 内核态探测点堆栈：print_backtrace()、sprint_backtrace() 不带s和带s的区别是前者直接输出，后者是返回堆栈字符串。

比如看看内核是怎么收包的：

# cat ip_rcv.stp
probe kernel.function("ip_rcv")
{
    printf("--------------------------------------------------------\n");
    print_backtrace();
    printf("--------------------------------------------------------\n");
}
# stap ip_rcv.stp
--------------------------------------------------------
 0xffffffffa8d89700 : ip_rcv+0x0/0x362 [kernel]
 0xffffffffa8d14b10 : __netif_receive_skb_core+0x560/0xc30 [kernel]
 0xffffffffa8d1527d : netif_receive_skb_internal+0x3d/0xb0 [kernel]
 0xffffffffa8d15f9a : napi_gro_receive+0xba/0xe0 [kernel]
 0xffffffffc00dc80c
 0xffffffffc00dc80c (inexact)
 0xffffffffc00dd260 (inexact)
 0xffffffffa8d156a9 : net_rx_action+0x149/0x3b0 [kernel] (inexact)
 0xffffffffa92000e3 : __do_softirq+0xe3/0x30a [kernel] (inexact)
 0xffffffffa86b7850 : irq_exit+0x100/0x110 [kernel] (inexact)
 0xffffffffa9001f1f : do_IRQ+0x7f/0xe0 [kernel] (inexact)
 0xffffffffa9000a8f : ret_from_intr+0x0/0x1d [kernel] (inexact)
 0xffffffffa8e95bfe : native_safe_halt+0xe/0x10 [kernel] (inexact)
 0xffffffffa8e9586c : __cpuidle_text_start+0x1c/0x130 [kernel] (inexact)
 0xffffffffa86e6831 : do_idle+0x1f1/0x280 [kernel] (inexact)
 0xffffffffa86e6a9f : cpu_startup_entry+0x6f/0x80 [kernel] (inexact)
 0xffffffffa9f79206 : start_kernel+0x53b/0x55b [kernel] (inexact)
 0xffffffffa86000e7 : secondary_startup_64+0xb7/0xc0 [kernel] (inexact)
--------------------------------------------------------

2.4、获取全局变量

# cat get_ngx_cycle.stp
probe process("/root/work/systemtap/nginx-installed/sbin/nginx").function("ngx_process_events_and_timers").call {
    ngx_cycle = @var("ngx_cycle@src/core/ngx_cycle.c")
    printf("ngx_cycle->connections: %d\n", ngx_cycle->connections)
    exit()
}
# stap get_ngx_cycle.stp
ngx_cycle->connections: 140436834037776

2.5、获取数据结构

在SystemTap里面，不管是指针还是数据结构，都是用->访问其成员：

# cat get_http_uri.stp
probe process("/root/work/systemtap/nginx-installed/sbin/nginx").function("ngx_http_process_request").call {
    printf("r->uri.len: %d, r->uri.data: %p\n", $r->uri->len, $r->uri->data)
}
# stap get_http_uri.stp
r->uri.len: 1, r->uri.data: 0x1af5434
r->uri.len: 1, r->uri.data: 0x1af5434

输出整个数据结构：

# cat get_r_pool.stp
probe process("/root/work/systemtap/nginx-installed/sbin/nginx").function("ngx_http_process_request").call {
    printf("$r->pool$: %s\n$r->pool$$: %s\n", $r->pool$, $r->pool$$)
}
# stap get_r_pool.stp
$r->pool$: {.d={...}, .max=4016, .current=0x1b54290, .chain=0x0, .large=0x0, .cleanup=0x0, .log=0x1b10410}
$r->pool$$: {.d={.last="/root/work/systemtap/nginx-installed/html/index.html", .end="nt_root", .next=0x1b17ea0, .failed=0}, .max=4016, .current=0x1b54290, .chain=0x0, .large=0x0, .cleanup=0x0, .log=0x1b10410}

2.6、遍历数组

# cat debug_http_header.stp
probe process("/root/work/systemtap/nginx-installed/sbin/nginx").function("ngx_http_finalize_request").call {
    i = 0
    headers_in_part = &$r->headers_in->headers->part
    headers = &@cast(headers_in_part->elts, "ngx_table_elt_t")[0]
    while (headers) {
        if (i >= headers_in_part->nelts) {
            if (!headers_in_part->next) {
                break
            }
            headers_in_part = headers_in_part->next;
            headers = &@cast(headers_in_part->elts, "ngx_table_elt_t")[0]
            i = 0
        }
        h = &@cast(headers, "ngx_table_elt_t")[i]
        printf("%s: %s\n", user_string_n(h->key->data, h->key->len), user_string_n(h->value->data, h->value->len))
        i += 1
    }
}
# stap debug_http_header.stp
User-Agent: curl/7.29.0
Host: 119.45.102.20:8070
Accept: */*
User-Agent: curl/7.29.0
Host: 119.45.102.20:8070
Accept: */*

2.7、跟踪进程执行流程

# cat trace_nginx.stp
probe process("/root/work/systemtap/nginx-installed/sbin/nginx").function("*@src/http/ngx_http_*").call
{
    printf("%s -> %s\n", thread_indent(4), ppfunc());
}

probe process("/root/work/systemtap/nginx-installed/sbin/nginx").function("*@src/http/ngx_http_*").return
{
    printf("%s <- %s\n", thread_indent(-4), ppfunc());
}
# stap trace_nginx.stp
     0 nginx(1675800):    -> ngx_http_init_connection
    33 nginx(1675800):    <- ngx_http_init_connection
     0 nginx(1675800):    -> ngx_http_wait_request_handler
     9 nginx(1675800):        -> ngx_http_create_request
    12 nginx(1675800):            -> ngx_http_alloc_request
    16 nginx(1675800):            <- ngx_http_alloc_request
    17 nginx(1675800):        <- ngx_http_create_request
    20 nginx(1675800):        -> ngx_http_process_request_line

Keep studying

systemtap