1. Please install utility to run crash-utility  

$ sudo apt-get install git build-essential bison zlib1g-dev libncurses5-dev libncursesw5-dev  pkg-config flex swig -y

2. Pull crash-utility source code with below command.

$ git clone https://github.com/crash-utility/crash.git

3. Build crash-utility Source code 

$ cd crash
$ make target=ARM64 -j3

(Warninig)

You may see error messages compiling crash-utility due to 'j' option.
In this case, please build crash-utility without 'j' option

$ cd crash
$ make target=ARM64 

After crash-utility is sucessfully compiled, 'crash' is generated.

4. Place the vmcore and vmlinux in the same directory. And copy 'crash' to the same directory where vmcore and vmlinux are placed.

$ ls
$ crash vmcore vmlinux
$ chmod 777 crash 
(Warning)

Please make sure that vmcore is matched to vmlinux. And then you need to use the below
command to start crash utility

5. Run crash-utility using below command

$ ./crash vmcore vmlinux

When compiling crash utility in the Raspbian with Aarch32,
do not use 'j' option like (make -j5).

apt update
apt-get install git build-essential bison zlib1g-dev libncurses5-dev libncursesw5-dev  pkg-config flex swig -y
apt-get install -y texinfo
cd crash
make target=ARM

 

# good command to start 'crash utility'

 

1. 1st verified command 


./crash dump1@0x80000000,dump2@0x880000000 -p 4096 -m vabits_actual=39 -m kimage_voffset=0xffffffbf70000000 --no_panic --smp $1

 

2. Under progress

 

./crash1 -d -6 SYS_COREDUMP -m --kaslr=0x1d43c00000 -m vabits_actual=39 -m kimage_voffset=0xffffffa611c00000 -m --smp vmlinux

diff --git a/arm64.c b/arm64.c
index bdb2a0a..7c3a8ea 100644
--- a/arm64.c
+++ b/arm64.c
@@ -361,6 +361,19 @@ arm64_init(int when)
        /* use machdep parameters */
        arm64_calc_phys_offset();
     
+       error(INFO, "[+][%s][%d] at %s\n", __func__, __LINE__, __FILE__);
+       error(INFO, "kimage_voffset: %lx phys_offset: %lx \n",
+                               machdep->machspec->kimage_voffset, machdep->machspec->phys_offset);
+
+        error(INFO, "CONFIG_ARM64_VA_BITS: %ld\n", ms->CONFIG_ARM64_VA_BITS);
+        error(INFO,  "     VA_BITS_ACTUAL: %ld\n", ms->VA_BITS_ACTUAL);
+        error(INFO, "(calculated) VA_BITS: %ld\n", ms->VA_BITS);
+        error(INFO, " PAGE_OFFSET: %lx\n", ARM64_PAGE_OFFSET_ACTUAL);
+        error(INFO, "    VA_START: %lx\n", ms->VA_START);
+        error(INFO, "     modules: %lx - %lx\n", ms->modules_vaddr, ms->modules_end);
+        error(INFO, "     vmalloc: %lx - %lx\n", ms->vmalloc_start_addr, ms->vmalloc_end);
+        error(INFO, "kernel image: %lx - %lx\n", ms->kimage_text, ms->kimage_end);
+        error(INFO, "     vmemmap: %lx - %lx\n\n", ms->vmemmap_vaddr, ms->vmemmap_end);
        if (CRASHDEBUG(1)) {
            if (machdep->flags & NEW_VMEMMAP)
                fprintf(fp, "kimage_voffset: %lx\n",
@@ -816,6 +829,11 @@ arm64_parse_cmdline_args(void)
                    "setting max_physmem_bits to: %ld\n\n",
                    machdep->max_physmem_bits);
                continue;
+           } else if (arm64_parse_machdep_arg_l(arglist[i], "kaddr_offset",
+               &machdep->machspec->kimage_addr_offset)) {
+               error(WARNING, "setting kimage_addr_offset to: 0x%lx\n\n",
+                                   machdep->machspec->kimage_addr_offset);
+               continue;
            }
  
            error(WARNING, "ignoring --machdep option: %s\n",
@@ -1011,6 +1029,12 @@ arm64_kdump_phys_base(ulong *phys_offset)
    if ((machdep->flags & NEW_VMEMMAP) &&
        machdep->machspec->kimage_voffset &&
        (sp = kernel_symbol_search("memstart_addr"))) {
+
+       physaddr_t kimage_offset_addr = machdep->machspec->kimage_addr_offset;
+
+       if (kimage_offset_addr) {
+           machdep->machspec->kimage_voffset -= kimage_offset_addr;
+       }
        paddr = sp->value - machdep->machspec->kimage_voffset;
        if (READMEM(-1, phys_offset, sizeof(*phys_offset),
            sp->value, paddr) > 0)
diff --git a/defs.h b/defs.h
index 42e381e..a377611 100644
--- a/defs.h
+++ b/defs.h
@@ -3259,6 +3259,7 @@ struct machine_specific {
    ulong VA_BITS_ACTUAL;
    ulong CONFIG_ARM64_VA_BITS;
    ulong VA_START;
+   ulong kimage_addr_offset;
 };
  
 struct arm64_stackframe {
diff --git a/main.c b/main.c
index 83ccd31..971486e 100644
--- a/main.c
+++ b/main.c
@@ -21,6 +21,13 @@
 #include <getopt.h>
 #include <sys/prctl.h>
  
+#define OCIMEM_OFFSET_ADDR             0x14680000
+#define RPI5_KASLR_ADDR_LOCATION     0x146bf6d0
+#define ADDR_READ_OFFSET           0x4
+
+#define RPI5_ADDR_FORMAT_SIZE    19
+#define RPI5_KASLR_ADDR_OFFSET  ( RPI5_KASLR_ADDR_LOCATION - OCIMEM_OFFSET_ADDR + ADDR_READ_OFFSET )
+
 static void setup_environment(int, char **);
 static int is_external_command(void);
 static int is_builtin_command(void);
@@ -80,6 +87,9 @@ main(int argc, char **argv)
 {
    int i, c, option_index;
    char *tmpname;
+   FILE *ocimem_file = NULL;
+   uint64_t c_64, kaslr_offset = 0;
+   char *optarg_backup;
  
    setup_environment(argc, argv);
  
@@ -226,6 +236,29 @@ main(int argc, char **argv)
                        optarg);
                }
            } else if (STREQ(long_options[option_index].name, "kaslr")) {
+               ocimem_file = fopen("./OCIMEM.BIN", "r");
+
+               if(!ocimem_file) {
+                   error(INFO, "Fail to open OCIMEM.BIN\n");
+                   goto OCIMEM_READ_EXIT;
+               }
+
+               fseek(ocimem_file, RPI5_KASLR_ADDR_OFFSET, SEEK_SET);
+
+               optarg_backup = optarg;
+               optarg = malloc(RPI5_ADDR_FORMAT_SIZE);
+
+               for(i=0;i<sizeof(uint64_t);i++) {
+                   c_64 = getc(ocimem_file);
+                   kaslr_offset |= (c_64 << (i*8));
+               }
+              
+               snprintf(optarg, RPI5_ADDR_FORMAT_SIZE, "0x%lx", kaslr_offset);
+               error(INFO, "kaslr_offset=%s\n", optarg);
+OCIMEM_READ_EXIT:
+               if(ocimem_file)
+                   fclose(ocimem_file);
+
                if (!machine_type("X86_64") &&
                    !machine_type("ARM64") && !machine_type("X86") &&
                    !machine_type("S390X"))

 

>>>

* bad dump

SECTIONS_PER_ROOT = 128
SECTION_ROOT_MASK = 0x7f
PAGES_PER_SECTION = 262144
<readmem: ffffffa653a30058, KVADDR, "mem_section", 8, (FOE), 7ffd4f76eb20>
<read_kdump: addr: ffffffa653a30058 paddr: 41e30058 cnt: 8>
read_netdump: addr: ffffffa653a30058 paddr: 41e30058 cnt: 8 offset: 1e34058
<readmem: 0, KVADDR, "memory section root table", 64, (FOE), 5625b4edac70>
crash1: invalid kernel virtual address: 0  type: "memory section root table"


        if (!(vt->mem_sec = (void *)malloc(mem_section_size)))
                error(FATAL, "cannot malloc mem_sec cache\n");
        if (!(vt->mem_section = (char *)malloc(SIZE(mem_section))))
                error(FATAL, "cannot malloc mem_section cache\n");

        if (mem_section_is_ptr)
                get_symbol_data("mem_section", sizeof(void *), &addr);
        else
                addr = symbol_value("mem_section");

        readmem(addr, KVADDR, vt->mem_sec, mem_section_size,
                "memory section root table", FAULT_ON_ERROR);

 * good dump 

crash64>  p &mem_section
p: per_cpu_symbol_search(&mem_section): NULL
GETBUF(344 -> 0)
$1 = (struct mem_section ***) 0xffffff9d4e1a0010 <mem_section>
FREEBUF(0)


 7135 <readmem: ffffffd3bf5eb880, KVADDR, "memory section root table", 32768, (FOE), 556a792d83e0>
 7136 <read_ramdump: addr: ffffffd3bf5eb880 paddr: 17f5eb880 cnt: 1920>
 7137 read_ramdump: addr: ffffffd3bf5eb880 paddr: 17f5eb880 cnt: 1920 offset: 3f5eb880
 7138 <read_ramdump: addr: ffffffd3bf5ec000 paddr: 17f5ec000 cnt: 4096>
 7139 read_ramdump: addr: ffffffd3bf5ec000 paddr: 17f5ec000 cnt: 4096 offset: 3f5ec000
 7140 <read_ramdump: addr: ffffffd3bf5ed000 paddr: 17f5ed000 cnt: 4096>

crash64> rd 0xffffff9d4e1a0010 100
<addr: ffffff9d4e1a0010 count: 100 flag: 490 (KVADDR)>
<readmem: ffffff9d4e1a0010, KVADDR, "64-bit KVADDR", 8, (FOE), 7ffea347dc28>
<read_ramdump: addr: ffffff9d4e1a0010 paddr: a25a0010 cnt: 8>
read_ramdump: addr: ffffff9d4e1a0010 paddr: a25a0010 cnt: 8 offset: 225a0010
ffffff9d4e1a0010:  ffffffd3bf5eb880 <readmem: ffffff9d4e1a0018, KVADDR, "64-bit KVADDR", 8, (FOE), 7ffea347dc28>


./crash64 --minimal SYS_COREDUMP vmlinux

./crash64 --minimal SYS_COREDUMP --kaslr=0x1d43c00000 vmlinux

< 환경: 라즈베리 파이(aarch32) >
 
'crash-utility' 소스를 빌드하면 아래와 같은 에러를 만난다.
 
gcc -c -g -DARM -D_FILE_OFFSET_BITS=64 -DGDB_10_2 -g -O2 build_data.c
gcc -c -g -DARM -D_FILE_OFFSET_BITS=64 -DGDB_10_2 -g -O2 main.c
main.c:20:10: fatal error: curses.h: No such file or directory
 #include <curses.h>
          ^~~~~~~~~~
compilation terminated.
make[5]: *** [Makefile:336: main.o] Error 1
 
curses 라이브러리가 설치되지 않았을 때 이런 에러가 발생한다.
다음 커맨드를 사용해 curses 라이브러리를 설치하면 컴파일 에러가 발생하지 않는다.
 
root@raspberrypi:/home/pi/osc_work/src_crash/crash# apt-get install libncurses5-dev libncursesw5-dev
Reading package lists... Done
Building dependency tree
Reading state information... Done
The following packages were automatically installed and are no longer required:
  gconf-service gconf2-common libexiv2-14 libgconf-2-4 libgfortran3 libgmime-2.6-0 libncurses5 libssl1.0.2 uuid-dev vlc-l10n vlc-plugin-notify
  vlc-plugin-samba vlc-plugin-video-splitter vlc-plugin-visualization
Use 'sudo apt autoremove' to remove them.
The following additional packages will be installed:
  libncurses-dev
Suggested packages:
  ncurses-doc
The following NEW packages will be installed:
  libncurses-dev libncurses5-dev libncursesw5-dev
0 upgraded, 3 newly installed, 0 to remove and 23 not upgraded.
Need to get 280 kB of archives.
After this operation, 1,730 kB of additional disk space will be used.
Do you want to continue? [Y/n] Y
리눅스 커널 동네에서 전 세계적으로 쓰고 있는 디버깅 툴을 소개하고자 합니다.
 
레드헷의 앤더슨이란 형님이 만든 오픈 소스로 배포한 리눅스 크래시(커널 패닉)을 분석할 수 있는 툴인데요, 
리눅스 커널 커뮤니티에서도 이 툴을 활발히 쓰고 있죠.
 
소스를 좀 받아 볼까요? 리눅스 머신에서 아래 커맨드로 소스를 땡겨옵니다.
 
소스를 받고 나서 아래 명령어로 빌드를 해요.
ARM machine인 경우
make target=ARM
make target=ARM extensions
 
X86인 경우
make target=X86_64
make target=X86_64 extensions
 
이제 trace command란 오픈소스 툴을 가져올 차례입니다. 이 툴로 ftrace 각 event를 파싱할 수 있어요.
 
빌드 명령어는 초간단, 그냥 make 만 때리면 되요.
 
자자, 이제 좀 정리해볼까요?
 
crash tool을 빌드한 결과물은 crash, trace command을 빌드한 이미지는 trace-cmd 거거든요.
 
/bin/CrashTool 폴더에 각각 결과물을 복사하구요. 
"make target=ARM extensions" 빌드하면 생성되는 extensions 폴더도 잊지 마세요.
austin.kim@Linux:~/bin/CrashTool$ ls -l
total 52400
-rwxrwxrwx 1 austin.kim home001  8536852 Jul 13 11:30 crash
drwxrwxrwx 4 austin.kim home001     4096 May  4 19:20 extensions
-rwxrwxrwx 1 austin.kim home001   868663 Feb 15 09:23 trace-cmd
 
crash_load.sh란 파일을 아래와 같이 만들고, "./crash_load.sh vmlinux" 명령어를 입력하면 실행 끝이죠.
#!/bin/sh
export CRASH_EXTENSIONS=/home001/austin.kim/bin/CrashTool/extensions
export TRACE_CMD=/home001/austin.kim/bin/CrashTool/trace-cmd 
/home001/austin.kim/bin/CrashTool/crash coredump0.BIN@0x0,coredump1.BIN@0xc0000000 -p 4096 -m phys_base=0x80000000 --no_panic --smp $1
 
실행 결과 화면입니다. 참 쉽죠.
      KERNEL: vmlinux
   DUMPFILES: /var/tmp/ramdump_elf_1X1HKt [temporary ELF header]
              DDRCS0.BIN
              DDRCS1.BIN
        CPUS: 4
        DATE: Wed Nov 29 15:24:00 2017
      UPTIME: 00:35:30
LOAD AVERAGE: 6.66, 7.02, 5.86
       TASKS: 363
    NODENAME: localhost
     RELEASE: 3.18.66-g715eee2
     VERSION: #2 SMP PREEMPT Wed Nov 29 12:37:14 KST 2017
     MACHINE: armv7l  (unknown Mhz)
      MEMORY: 2 GB
       PANIC: "kernel BUG at /src/3.18_kernel_ver/kernel/lib/list_debug.c:40!"
         PID: 0
     COMMAND: "swapper/0"
        TASK: c1917578  (1 of 4)  [THREAD_INFO: c1900000]
         CPU: 0
       STATE: TASK_RUNNING
     WARNING: reported panic task e6357840 not found
 
crash>
 
#Reference
 
 
# Reference: For more information on 'Linux Kernel';
 
디버깅을 통해 배우는 리눅스 커널의 구조와 원리. 1
 
디버깅을 통해 배우는 리눅스 커널의 구조와 원리. 2
 
 
 
 
 
 
디버깅 옵션(-d 6)을 적용해 crash-utilty를 실행하면 vmcore를 로딩하면서 다양한 디버깅 정보를 출력합니다.
 
$ ./crash vmcore vmlinux -d 6
 
crash 7.3.0++
Copyright (C) 2002-2021  Red Hat, Inc.
Copyright (C) 2004, 2005, 2006, 2010  IBM Corporation
Copyright (C) 1999-2006  Hewlett-Packard Co
Copyright (C) 2005, 2006, 2011, 2012  Fujitsu Limited
Copyright (C) 2006, 2007  VA Linux Systems Japan K.K.
Copyright (C) 2005, 2011, 2020-2021  NEC Corporation
Copyright (C) 1999, 2002, 2007  Silicon Graphics, Inc.
Copyright (C) 1999, 2000, 2001, 2002  Mission Critical Linux, Inc.
This program is free software, covered by the GNU General Public License,
and you are welcome to change it and/or distribute copies of it under
certain conditions.  Enter "help copying" to see the conditions.
This program has absolutely no warranty.  Enter "help warranty" for details.
 
NOTE: setting kimage_voffset to: 0xffffffbf70000000
 
NOTE: setting vabits_actual to: 39
 
GNU gdb (GDB) 7.6
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "--host=x86_64-unknown-linux-gnu --target=aarch64-elf-linux"...
 
WARNING: could not find MAGIC_START!
raw_crash64: cannot determine base kernel version
raw_crash64: vmlinux and /var/tmp/ramdump_elf_74SLyJ do not match!
 
Usage:
 
  crash [OPTION]... NAMELIST MEMORY-IMAGE[@ADDRESS]     (dumpfile form)
  crash [OPTION]... [NAMELIST]                          (live system form)
 
...
 
creating temporary ELF header: /var/tmp/ramdump_elf_BGK8hx
 
vmcore_data:
                  flags: c0 (KDUMP_LOCAL|KDUMP_ELF64)
                   ndfd: 16
                    ofp: 7f2e25cad620
            header_size: 848
   num_pt_load_segments: 13
     pt_load_segment[0]:
            file_offset: 350
             phys_start: 80000000
               phys_end: c0000000
              zero_fill: 0
     pt_load_segment[1]:
            file_offset: 40000350
             phys_start: 880000000
               phys_end: 8c0000000
              zero_fill: 0

 

< '시스템 소프트웨어 개발을 위한 Arm 아키텍처의 구조와 원리' 저자>

 
 
크래시 유틸리티를 디버깅할 수 있는 유용한 디버깅 패치다.
100% 내가 만든 것이다.
 
diff --git a/arm64.c b/arm64.c
--- a/arm64.c
+++ b/arm64.c
@@ -361,6 +361,19 @@ arm64_init(int when)
        /* use machdep parameters */
        arm64_calc_phys_offset();
     
+       error(INFO, "[+][%s][%d] at %s\n", __func__, __LINE__, __FILE__);
+       error(INFO, "kimage_voffset: %lx phys_offset: %lx \n",
+                               machdep->machspec->kimage_voffset, machdep->machspec->phys_offset);
+
+        error(INFO, "CONFIG_ARM64_VA_BITS: %ld\n", ms->CONFIG_ARM64_VA_BITS);
+        error(INFO,  "     VA_BITS_ACTUAL: %ld\n", ms->VA_BITS_ACTUAL);
+        error(INFO, "(calculated) VA_BITS: %ld\n", ms->VA_BITS);
+        error(INFO, " PAGE_OFFSET: %lx\n", ARM64_PAGE_OFFSET_ACTUAL);
+        error(INFO, "    VA_START: %lx\n", ms->VA_START);
+        error(INFO, "     modules: %lx - %lx\n", ms->modules_vaddr, ms->modules_end);
+        error(INFO, "     vmalloc: %lx - %lx\n", ms->vmalloc_start_addr, ms->vmalloc_end);
+        error(INFO, "kernel image: %lx - %lx\n", ms->kimage_text, ms->kimage_end);
+        error(INFO, "     vmemmap: %lx - %lx\n\n", ms->vmemmap_vaddr, ms->vmemmap_end);
        if (CRASHDEBUG(1)) {
            if (machdep->flags & NEW_VMEMMAP)
                fprintf(fp, "kimage_voffset: %lx\n",
@@ -816,6 +829,11 @@ arm64_parse_cmdline_args(void)
                    "setting max_physmem_bits to: %ld\n\n",
                    machdep->max_physmem_bits);
                continue;
+           } else if (arm64_parse_machdep_arg_l(arglist[i], "kaddr_offset",
+               &machdep->machspec->kimage_addr_offset)) {
+               error(WARNING, "setting kimage_addr_offset to: 0x%lx\n\n",
+                                   machdep->machspec->kimage_addr_offset);
+               continue;
            }
  
             error(WARNING, "ignoring --machdep option: %s\n",
@@ -1011,6 +1029,12 @@ arm64_kdump_phys_base(ulong *phys_offset)
    if ((machdep->flags & NEW_VMEMMAP) &&
        machdep->machspec->kimage_voffset &&
        (sp = kernel_symbol_search("memstart_addr"))) {
+
+       physaddr_t kimage_offset_addr = machdep->machspec->kimage_addr_offset;
+
+       if (kimage_offset_addr) {
+           machdep->machspec->kimage_voffset -= kimage_offset_addr;
+       }
        paddr = sp->value - machdep->machspec->kimage_voffset;
        if (READMEM(-1, phys_offset, sizeof(*phys_offset),
            sp->value, paddr) > 0) 
 

< '시스템 소프트웨어 개발을 위한 Arm 아키텍처의 구조와 원리' 저자>

 
 

간단하다. 로그 버퍼를 볼 수 있다.

log -m

crash> log -m
<6>[194643.799885 02-09 21:16:26.949] bt_hc_worker (32565): undefined instruction: pc=c032ae1c
....

<0>[194643.799977 02-09 21:16:26.949] Internal error: Oops - BUG: 0 [#1] PREEMPT SMP ARM
<4>[194643.800007 02-09 21:16:26.949] Modules linked in:
<4>[194643.800038 02-09 21:16:26.949] CPU: 0    Tainted: G        W     (3.4.0-perf-g3d4c0e7-00001-g46f9c0a #1)
<4>[194643.800068 02-09 21:16:26.949] PC is at hs_start_rx_locked+0x2c/0x218
<4>[194643.800099 02-09 21:16:26.949] LR is at hs_start_rx_locked+0x2c/0x218
<4>[194643.800129 02-09 21:16:26.949] pc : c032ae1c    lr : c032ae1c    psr : 20000093

 

프로세서의 상태를 parent tree 포맷으로 나타내줌.

crash> ps -p

crash> ps -p
PID: 0      TASK: c0d4cbd8  CPU: 0   COMMAND: "swapper/0"
PID: 2      TASK: dc4403c0  CPU: 0   COMMAND: "kthreadd"
  PID: 75     TASK: eb66da40  CPU: 1   COMMAND: "smux_tx_wq"
 
PID: 0      TASK: c0d4cbd8  CPU: 0   COMMAND: "swapper/0"
PID: 2      TASK: dc4403c0  CPU: 0   COMMAND: "kthreadd"
  PID: 76     TASK: eb66de00  CPU: 1   COMMAND: "smux_rx_wq"

 

프로세서의상태를 다른 parent tree 포맷으로 나타내줌.

crash> ps -c

PID: 0      TASK: c0d4cbd8  CPU: 0   COMMAND: "swapper/0"
  PID: 0      TASK: c0d4cbd8  CPU: 0   COMMAND: "swapper/0"
  PID: 0      TASK: dc4421c0  CPU: 1   COMMAND: "swapper/1"
  PID: 1      TASK: dc440000  CPU: 0   COMMAND: "init"
  PID: 2      TASK: dc4403c0  CPU: 0   COMMAND: "kthreadd"
 
PID: 0      TASK: dc4421c0  CPU: 1   COMMAND: "swapper/1"
  (no children)
 
PID: 1      TASK: dc440000  CPU: 0   COMMAND: "init"
  PID: 142    TASK: eb5fe940  CPU: 0   COMMAND: "ueventd"
  PID: 321    TASK: eadd7c00  CPU: 0   COMMAND: "healthd"
  PID: 322    TASK: eadd5680  CPU: 1   COMMAND: "servicemanager"
  PID: 323    TASK: eadd52c0  CPU: 0   COMMAND: "vold"

 

process user space 상의 좀더 구체적인 정보를 보여줌

crash> ps -a

crash> ps -a
PID: 389    TASK: eb066940  CPU: 0   COMMAND: "sdcard"
ARG: /system/bin/sdcard -u 1023 -g 1023 -l /data/media /mnt/shell/emulated
ENV: VIBE_PIPE_PATH=/dev/pipes
     PATH=/sbin:/vendor/bin:/system/sbin:/system/bin:/system/xbin
     LD_LIBRARY_PATH=/vendor/lib:/system/lib
     ANDROID_BOOTLOGO=1
     ANDROID_ROOT=/system

 

process group 별로 process를 보여줌

crash> ps -g

crash> ps -g
PID: 330    TASK: eadd4b40  CPU: 0   COMMAND: "surfaceflinger"
  PID: 474    TASK: ea95c3c0  CPU: 1   COMMAND: "Binder_1"
  PID: 475    TASK: ea95fc00  CPU: 1   COMMAND: "DispSync"
  PID: 479    TASK: ea95e940  CPU: 0   COMMAND: "Binder_2"

 

아래명령어를 입력하면 process 당 동작 시간을 표시한다. 프로세스들의 stall 현상을 디버깅하기 유용하다.

crash> ps -t

PID: 32569  TASK: e04703c0  CPU: 0   COMMAND: "ContactsProvide"
    RUN TIME: 00:00:00
  START TIME: 194643
       UTIME: 1
       STIME: 0
 

 

커널쓰레드의 정보를 보여줌

crash> ps -k

crash> ps -k
   PID    PPID  CPU   TASK    ST  %MEM     VSZ    RSS  COMM
      0      0   0  c0d4cbd8  RU   0.0       0      0  [swapper/0]
      0      0   1  dc4421c0  RU   0.0       0      0  [swapper/1]
      2      0   0  dc4403c0  IN   0.0       0      0  [kthreadd]
      3      2   0  dc440780  IN   0.0       0      0  [ksoftirqd/0]
      6      2   0  dc4412c0  IN   0.0       0      0  [migration/0]

 

posix regular expression으로 process 정보를 보여줌

crash> ps 'msm*'

crash> ps 'msm*'
   PID    PPID  CPU   TASK    ST  %MEM     VSZ    RSS  COMM
     14      2   0  dc443480  IN   0.0       0      0  [irq/203-msmdata]
     18      2   0  dc6a83c0  IN   0.0       0      0  [msm_slim_ctrl_r]

 

특정프로세스(bt_hc_worker)에서 open하고 있는 파일의목록을 확인해보자.      

crash> files 32565

crash> files 32565
PID: 32565  TASK: c3ffb840  CPU: 0   COMMAND: "bt_hc_worker"
  3  eaef8780  eba79d80  ead3c8c0  CHR   /dev/log/main
  4  eae3de40  eba70780  ead3a190  CHR   /dev/log/radio
  5  ea7029c0  eba73780  ead3ad10  CHR   /dev/log/events
  6  ea74fc00  eba76d80  ead3bce0  CHR   /dev/log/system

 

프로세스의 backtrace(call stack)을 파악할 때 사용함.

crash> bt -tlf 107

crash> bt -tlf 107
PID: 107    TASK: eb66fc00  CPU: 0   COMMAND: "mmcqd/0"
      START: __schedule at c07f66d4
  [eb1a3d9c] handle_irq_event_percpu at c00dcba0
  [eb1a3da8] schedule_timeout at c07f4e20
  [eb1a3dcc] gic_eoi_irq at c001d500

 

디바이스에서사용하고 있는 character device의 목록을 확인할 수 있다.

커널패닉이 발생할 때 이외에 디바이스에 대해 첫 분석을 진행할 때 유용하게 사용할 수 있다.

crash> dev

crash> dev
CHRDEV    NAME              CDEV    OPERATIONS
   1      mem             eb33e2c0  memory_fops
   4      /dev/vc/0       c0f583a4  console_fops
   4      tty             dc668d88  tty_fops
   5      /dev/tty        c0f501ec  tty_fops
   5      /dev/console    c0f50228  console_fops
   5      /dev/ptmx       c0f50350  ptmx_fops
   7      vcs             eb341dc0  vcs_fops
  10      misc            dc67e340  misc_fops
  13      input           dc689c40  input_fops
  21      sg               (none)
  29      fb              dc67de00  fb_fops
  81      video4linux     eb0b0c80  v4l2_fops

 

struct나 주소의 아웃풋을 16진수로 표현하고 싶을 경우에 사용한다.

hex

crash> hex
output radix: 16 (hex)

 

sys 명령어로 현재 시스템의 상태를 파악할 수 있음.

아래와같이 빌드 시간/구동된 시간/메모리 사용량/패닉 reason을 알 수 있음.

crash> sys

crash> sys
      KERNEL: vmlinux
    DUMPFILE: rawdump
        CPUS: 2
        DATE: Sun Feb  9 21:16:26 2014
      UPTIME: 2 days, 06:04:03
LOAD AVERAGE: 8.65, 8.42, 7.68
       TASKS: 770
    NODENAME: localhost
     RELEASE: 3.4.0-perf-g3d4c0e7-00001-g46f9c0a
     VERSION: #1 SMP PREEMPT Tue Feb 4 11:21:16 KST 2014

 

아래명령어로 시스템 콜의 위치를 파악할 수 있음

crash> sys -c

 

설명할필요가 없다. Config 파일의 목록이다.

crash> sys config

crash> sys config
#
# Automatically generated file; DO NOT EDIT.
# Linux/arm 3.4.0 Kernel Configuration
#
CONFIG_ARM=y
CONFIG_ARM_HAS_SG_CHAIN=y

 

crash> struct -o task_struct | grep files

[0x2b8] struct files_struct *files;

 

심볼정보를 보고 싶은 경우 아래와 같은 명령어를 사용함

crash> whatis

crash> whatis linux_banner
const char linux_banner[];
crash> whatis init_task
struct task_struct init_task;

 

커널에서사용하고 있는 모든 타이머를 확인하고 싶은 경우

crash> timer

crash> timer
TVEC_BASES[0]: c0efa0c0
JIFFIES
19434365
EXPIRES  TIMER_LIST  FUNCTION
19434366   dc5838ac   c0403f34  <dhd_watchdog>
19434369   eb0a0c80   c009adc0  <delayed_work_timer_fn>
19434375   c0d4f8a8   c009adc0  <delayed_work_timer_fn>
19434376   eabe1434   c009adc0  <delayed_work_timer_fn>
19434377   d4295e88   c008eb44  <process_timeout>
19434380   c2d4a44c   c009adc0  <delayed_work_timer_fn>
19434385   eb6bf1f0   c009adc0  <delayed_work_timer_fn>
19434400   c0d77dc4   c03818bc  <pm_suspend_timer_fn>
19434400   c0d7ce94   c03818bc  <pm_suspend_timer_fn>
19434400   c2d4a204   c009adc0  <delayed_work_timer_fn>
19434400   eabe15d4   c009adc0  <delayed_work_timer_fn>

 

아래명령어로 심볼의 주소를 알아낼 수 있다.

crash> sym -l | grep start_kern

c07e53a4 (T) secondary_start_kernel
c0c004b0 (T) start_kernel

 

스케줄러의실행 queue를 출력함

crash> runq

crash> runq
CPU 0 RUNQUEUE: c2d4b700
  CURRENT: PID: 32565  TASK: c3ffb840  COMMAND: "bt_hc_worker"
  RT PRIO_ARRAY: c2d4b7d0
     [no tasks queued]
  CFS RB_ROOT: c2d4b770
     [120] PID: 32570  TASK: c3ffbc00  COMMAND: "BluetoothAdapte"
     [120] PID: 9961   TASK: e7aec000  COMMAND: "kworker/u:1"
     [120] PID: 107    TASK: eb66fc00  COMMAND: "mmcqd/0"
 
CPU 1 RUNQUEUE: c2d54700
  CURRENT: PID: 32315  TASK: c98c5a40  COMMAND: "Binder_2"
  RT PRIO_ARRAY: c2d547d0
     [no tasks queued]
  CFS RB_ROOT: c2d54770
     [120] PID: 32302  TASK: e30eb840  COMMAND: "ndroid.settings"
     [130] PID: 32280  TASK: cc63cf00  COMMAND: "Binder_4"

 

네트워크디바이스의 목록을 출력

crash> net

NET_DEVICE  NAME   IP ADDRESS(ES)
dc40b800   lo     127.0.0.1
dc40e000   dummy0

< '시스템 소프트웨어 개발을 위한 Arm 아키텍처의 구조와 원리' 저자>

 
 

+ Recent posts