ksar、sar及相关内核知识点解析【转】
转自：
关键词：sar、sadc、ksar、/proc/stat、/proc/cpuinfo、/proc/meminfo、/proc/diskstats。
在之前有简单介绍过sar/ksar，最近在使用中感觉需要再深入了解一下。
ksar/sar从内核采集数据，并输出可读性数据。分析相关源码，有助于知道数据来龙去脉。--------------------------------------1. sar源码概览
ksar显示sar/sadc获取的数据，并图形化显示。数据从内核节点，到sadc/sar转换，再到ksar显示。----------------------------2. ksar处理流程
对照ksar每张图表，然后sar/sadc对应的采集转换，再到内核每个数据项含义解析。--------------------------------------------3. ksar解读
期望是从ksar上的图表能对应到内核的代码，明白这些图表数据根源。
rw_sa_stat_loop(count, stdfd, ofd, ofile, sa_dir); #ifdef HAVE_SENSORS /* Cleanup sensors */ sensors_cleanup(); #endif /* HAVE_SENSORS */ /* Free structures */ sa_sys_free(); return 0; }
rw_sa_stat_loop()是整个sadc的核心循环，这里从个sysfs读取信息，并提取关键信息，然后保存。
act[]。void read_stats(void) { int i; __nr_t cpu_nr = act[get_activity_position(act, A_CPU, EXIT_IF_NOT_FOUND)]->nr; record_hdr.uptime0 = 0; if (cpu_nr > 2) { read_uptime(&(record_hdr.uptime0)); } for (i = 0; i < NR_ACT; i++) { if (IS_COLLECTED(act[i]->options)) {---------------------------------------遍历所有act[]，如果act[]中对应的options包含AO_COLLECTED则进行f_read()。 /* Read statistics for current activity */ (*act[i]->f_read)(act[i]); } } if (cpu_nr == 1) { record_hdr.uptime0 = record_hdr.uptime; } }act[]存放了所有统计事件的struct activity。
cpu_act, &pcsw_act, &irq_act, &swap_act, &paging_act, &io_act, &memory_act, &huge_act, &ktables_act, &queue_act, &serial_act, &disk_act, /**/ &net_dev_act, &net_edev_act, &net_nfs_act, &net_nfsd_act, &net_sock_act, &net_ip_act, &net_eip_act, &net_icmp_act, &net_eicmp_act, &net_tcp_act, &net_etcp_act, &net_udp_act, &net_sock6_act, &net_ip6_act, &net_eip6_act, &net_icmp6_act, &net_eicmp6_act, &net_udp6_act, &fchost_act, &softnet_act, /* AO_CLOSE_MARKUP */ /* */ /**/ &pwr_cpufreq_act, &pwr_fan_act, &pwr_temp_act, &pwr_in_act, &pwr_wghfreq_act, &pwr_usb_act, /* AO_CLOSE_MARKUP */ /* */ &filesystem_act };1.2 sar显示统计信息
read_sadc_stat_bunch()读取统计信息，write_stats()调用每个struct activity的f_print()函数，write_stats_avg()调用每个struct activity的f_print_avg()函数。
f_print()和f_print_avg()或从文件中解析字符串，或启动sadc采样，然后再解析。
ksar安装文件。
ksar中看到的图标是结果，这些数据是通过sadc采集，sar解析出来的。
sadc是通过读取sysfs/procfs节点来获取信息，这些节点都是内核提供的统计信息。
sar -o temp.bin 1 600--------------------------------------------------sar将采样数据保存在temp.bin中。
LC_ALL=C sar -A -f temp.bin > sar.txt----------------------------将保存的采样数据temp.bin，转换成更可读性强的文本文件。
所以ksar的每一张图标，都对应了内核统计信息。
sar是数据搬运整理工具，ksar是图形化工具。
下面对每张图标从ksar，到sar/sadc，最终到内核中每个数据。
2.2 ksar操作
2.2.1 数据导入
通过Data->Append from a file...从txt中加载数据，还有其他两种数据来源方式。
2.2.2 数据导出
如果要将图表导出，可以在每张图标下面选择Export PNG之类。
或者通过Export->Export to PDF...，选择指定选项。
3. ksar解析
下面结合ksar图表来逐项解析。
/proc/stat节点，解析其中cpu信息，包括合计cpu信息以及单个cpu信息。
/proc/stat中的信息包括一个合计以及多个cpu单独统计信息。
从/proc/stat中读取的信息都是从启动以来的累计时间，在图标中显示的是一个时间段的差值。
然后计算不同模块耗时占比。
void read_stat_cpu(struct stats_cpu *st_cpu, int nbr, unsigned long long *uptime, unsigned long long *uptime0) { FILE *fp; struct stats_cpu *st_cpu_i; struct stats_cpu sc; char line[8192]; int proc_nb; if ((fp = fopen(STAT, "r")) == NULL) {----------------------------------------------------/proc/stat fprintf(stderr, _("Cannot open %s: %s\n"), STAT, strerror(errno)); exit(2); } while (fgets(line, sizeof(line), fp) != NULL) { if (!strncmp(line, "cpu ", 4)) {------------------------------------------------------统计总cpu不同类别耗时，详细信息参考/proc/stat。这里的信息和内核中一一对应。 memset(st_cpu, 0, STATS_CPU_SIZE); sscanf(line + 5, "%llu %llu %llu %llu %llu %llu %llu %llu %llu %llu", &st_cpu->cpu_user, &st_cpu->cpu_nice, &st_cpu->cpu_sys, &st_cpu->cpu_idle, &st_cpu->cpu_iowait, &st_cpu->cpu_hardirq, &st_cpu->cpu_softirq, &st_cpu->cpu_steal, &st_cpu->cpu_guest, &st_cpu->cpu_guest_nice); *uptime = st_cpu->cpu_user + st_cpu->cpu_nice + st_cpu->cpu_sys + st_cpu->cpu_idle + st_cpu->cpu_iowait + st_cpu->cpu_hardirq + st_cpu->cpu_steal + st_cpu->cpu_softirq; } else if (!strncmp(line, "cpu", 3)) { ... } fclose(fp); }/proc/stat节点，解析其中的ctxt和processes信息。
同样的/proc/stat中，统计信息是启动以来的累计值，图标中现实的不同时间段的差值。
proc/s表示每秒创建的进程数目，cswch/s表示每秒进程切换次数。
void read_stat_pcsw(struct stats_pcsw *st_pcsw) { FILE *fp; char line[8192]; if ((fp = fopen(STAT, "r")) == NULL) return; while (fgets(line, sizeof(line), fp) != NULL) { if (!strncmp(line, "ctxt ", 5)) {--------------------------------------------ctxt是所有CPU的进程切换次数。 /* Read number of context switches */ sscanf(line + 5, "%llu", &st_pcsw->context_switch); } else if (!strncmp(line, "processes ", 10)) {---------------------------------是整个系统创建进程的次数，total_forks， /* Read number of processes created since system boot */ sscanf(line + 10, "%lu", &st_pcsw->processes); } } fclose(fp); }/proc/vmstat节点，解析其中的pswpin和pswpout两个信息。
pswpin/s表示系统每秒从swap分区读入的页面数量，即移除掉swap；pswpout/s表示系统每秒写到swap分区的页面数量，即产生swap。
void read_vmstat_swap(struct stats_swap *st_swap) { FILE *fp; char line[128]; if ((fp = fopen(VMSTAT, "r")) == NULL) return; while (fgets(line, sizeof(line), fp) != NULL) { if (!strncmp(line, "pswpin ", 7)) {-------------------------------对应PSWPIN，表示从swap分区读入页面。 /* Read number of swap pages brought in */ sscanf(line + 7, "%lu", &st_swap->pswpin); } else if (!strncmp(line, "pswpout ", 8)) {-------------------------对应PSWPOUT，表示将page写入到swap分区。 /* Read number of swap pages brought out */ sscanf(line + 8, "%lu", &st_swap->pswpout); } } fclose(fp); }diskstats_show()函数。
void read_diskstats_io(struct stats_io *st_io) { FILE *fp; char line[256]; char dev_name[MAX_NAME_LEN]; unsigned int major, minor; unsigned long rd_ios, wr_ios, rd_sec, wr_sec; if ((fp = fopen(DISKSTATS, "r")) == NULL) return; while (fgets(line, sizeof(line), fp) != NULL) { if (sscanf(line, "%u %u %s %lu %*u %lu %*u %lu %*u %lu",-----------scanf的%u之间的星号，表示跳过此输入。 &major, &minor, dev_name, &rd_ios, &rd_sec, &wr_ios, &wr_sec) == 7) { if (is_device(dev_name, IGNORE_VIRTUAL_DEVICES)) {--------------虚拟设备没有/sys/block//device，此特性用以判断dev_name对应的设备是否是真实的块设备。 st_io->dk_drive += (unsigned long long) rd_ios + (unsigned long long) wr_ios; st_io->dk_drive_rio += rd_ios; st_io->dk_drive_rblk += rd_sec; st_io->dk_drive_wio += wr_ios; st_io->dk_drive_wblk += wr_sec; } } } fclose(fp); } /proc/diskstats来源于内核的diskstats_show()
/proc/meminfo的内存和swap信息。kbmemfree是所有free内存大小，kbavail是扣除保留内存，加上部分pagecache和可回收内存；kbmemused是目前系统内存使用量，不包括内核使用量。
%memused是内存使用占总内存百分比。
kbcommit是当前场景系统需要使用到的内存量，实际上可能并没有申请这么多内存。这是因为分配的内存只有在使用到时，才会产生缺页异常。
这里的kbcommit和页面统计信息相呼应，此处kbcommit突然增加，产生了很多page fault。
从图表中的名称大概就能找到对应的/proc/meminfo中的字符项。
其中%memused和%commit查看stub_print_memory_stats()，%memused为(MemTotal-MemFree)/MemTotal。
%commit为Commited_AS/(MemTotal+SwapTotal)。
在/proc/meminfo中看到的CommitLimit可能是proc/sys/vm/overcommit_kbytes；或者按照当前可以用内存乘以overcommit_ratio这个比例得到。
在实际使用中，还需要结合overcommit_memory类型来看，参考overcommit_memory和overcommit_ratio。
unsigned long vm_commit_limit(void) { unsigned long allowed; if (sysctl_overcommit_kbytes) allowed = sysctl_overcommit_kbytes >> (PAGE_SHIFT - 10); else allowed = ((totalram_pages - hugetlb_total_pages()) * sysctl_overcommit_ratio / 100); allowed += total_swap_pages; return allowed; }read_meminfo()从/proc/meminfo中解析数据。
void read_meminfo(struct stats_memory *st_memory) { FILE *fp; char line[128]; if ((fp = fopen(MEMINFO, "r")) == NULL) return; while (fgets(line, sizeof(line), fp) != NULL) { if (!strncmp(line, "MemTotal:", 9)) { /* Read the total amount of memory in kB */ sscanf(line + 9, "%lu", &st_memory->tlmkb); } else if (!strncmp(line, "MemFree:", 8)) { /* Read the amount of free memory in kB */ sscanf(line + 8, "%lu", &st_memory->frmkb); } else if (!strncmp(line, "MemAvailable:", 13)) { /* Read the amount of available memory in kB */ sscanf(line + 13, "%lu", &st_memory->availablekb); } else if (!strncmp(line, "Buffers:", 8)) { /* Read the amount of buffered memory in kB */ sscanf(line + 8, "%lu", &st_memory->bufkb); } else if (!strncmp(line, "Cached:", 7)) { /* Read the amount of cached memory in kB */ sscanf(line + 7, "%lu", &st_memory->camkb); } else if (!strncmp(line, "SwapCached:", 11)) { /* Read the amount of cached swap in kB */ sscanf(line + 11, "%lu", &st_memory->caskb); } else if (!strncmp(line, "Active:", 7)) { /* Read the amount of active memory in kB */ sscanf(line + 7, "%lu", &st_memory->activekb); } else if (!strncmp(line, "Inactive:", 9)) { /* Read the amount of inactive memory in kB */ sscanf(line + 9, "%lu", &st_memory->inactkb); } else if (!strncmp(line, "SwapTotal:", 10)) { /* Read the total amount of swap memory in kB */ sscanf(line + 10, "%lu", &st_memory->tlskb); } else if (!strncmp(line, "SwapFree:", 9)) { /* Read the amount of free swap memory in kB */ sscanf(line + 9, "%lu", &st_memory->frskb); } else if (!strncmp(line, "Dirty:", 6)) { /* Read the amount of dirty memory in kB */ sscanf(line + 6, "%lu", &st_memory->dirtykb); } else if (!strncmp(line, "Committed_AS:", 13)) { /* Read the amount of commited memory in kB */ sscanf(line + 13, "%lu", &st_memory->comkb); } else if (!strncmp(line, "AnonPages:", 10)) { /* Read the amount of pages mapped into userspace page tables in kB */ sscanf(line + 10, "%lu", &st_memory->anonpgkb); } else if (!strncmp(line, "Slab:", 5)) { /* Read the amount of in-kernel data structures cache in kB */ sscanf(line + 5, "%lu", &st_memory->slabkb); } else if (!strncmp(line, "KernelStack:", 12)) { /* Read the kernel stack utilization in kB */ sscanf(line + 12, "%lu", &st_memory->kstackkb); } else if (!strncmp(line, "PageTables:", 11)) { /* Read the amount of memory dedicated to the lowest level of page tables in kB */ sscanf(line + 11, "%lu", &st_memory->pgtblkb); } else if (!strncmp(line, "VmallocUsed:", 12)) { /* Read the amount of vmalloc area which is used in kB */ sscanf(line + 12, "%lu", &st_memory->vmusedkb); } } fclose(fp); }下图的kbswpfree、kbswpused、kbswpcad通过字面意思即可知其对应的meminfo项为SwapFree、SwapTotal-SwapFree、SwapCached。
rtnl_link_stats64 *stats = dev_get_stats(dev, &temp); seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu " "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n", dev->name, stats->rx_bytes, stats->rx_packets, stats->rx_errors, stats->rx_dropped + stats->rx_missed_errors, stats->rx_fifo_errors, stats->rx_length_errors + stats->rx_over_errors + stats->rx_crc_errors + stats->rx_frame_errors, stats->rx_compressed, stats->multicast, stats->tx_bytes, stats->tx_packets, stats->tx_errors, stats->tx_dropped, stats->tx_fifo_errors, stats->collisions, stats->tx_carrier_errors + stats->tx_aborted_errors + stats->tx_window_errors + stats->tx_heartbeat_errors, stats->tx_compressed); }
rtnl_link_stats64。
void read_net_edev(struct stats_net_edev *st_net_edev, int nbr) { FILE *fp; struct stats_net_edev *st_net_edev_i; static char line[256]; char iface[MAX_IFACE_LEN]; int dev = 0; int pos; if ((fp = fopen(NET_DEV, "r")) == NULL) return; while ((fgets(line, sizeof(line), fp) != NULL) && (dev < nbr)) { pos = strcspn(line, ":"); if (pos < strlen(line)) { st_net_edev_i = st_net_edev + dev; strncpy(iface, line, MINIMUM(pos, MAX_IFACE_LEN - 1)); iface[MINIMUM(pos, MAX_IFACE_LEN - 1)] = '\0'; sscanf(iface, "%s", st_net_edev_i->interface); /* Skip heading spaces */ sscanf(line + pos + 1, "%*u %*u %llu %llu %llu %llu %*u %*u %*u %*u " "%llu %llu %llu %llu %llu", &st_net_edev_i->rx_errors,------------------对应rx_erros，表示接受到的bad packet。 &st_net_edev_i->rx_dropped,-----------------对应rx_dropped+rx_missed_errors。 &st_net_edev_i->rx_fifo_errors,-------------对应rx_fifo_errors。 &st_net_edev_i->rx_frame_errors,------------对应rx_length_errors+rx_over_errros。 &st_net_edev_i->tx_errors, &st_net_edev_i->tx_dropped, &st_net_edev_i->tx_fifo_errors, &st_net_edev_i->collisions,-----------------对应collisions &st_net_edev_i->tx_carrier_errors);---------包括tx_carrier_errors+tx_aborted_errors+tx_window_errors+tx_heartbeat_errors。dev++; } } fclose(fp); }获取网络设备统计信息的核心数据结构式struct rtnl_link_stat64。
sockstat_seq_show()，这里面统计了当前系统socket的使用情况。read_net_sock()函数解析。可见totsck表示系统所有Socket数目，其他子类表示不同类型socket数目。
softnet_seq_show()。static int softnet_seq_show(struct seq_file *seq, void *v) { struct softnet_data *sd = v; unsigned int flow_limit_count = 0; #ifdef CONFIG_NET_FLOW_LIMIT struct sd_flow_limit *fl; rcu_read_lock(); fl = rcu_dereference(sd->flow_limit); if (fl) flow_limit_count = fl->count; rcu_read_unlock(); #endif seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", sd->processed, sd->dropped, sd->time_squeeze, 0, 0, 0, 0, 0, /* was fastroute */ 0, /* was cpu_collision */ sd->received_rps, flow_limit_count); return 0; }联系方式:arnoldlu@qq.com