crash-stack: unwinding by frame pointer on aarch64

To unwind call stack on aarch64 we need to use external
method, as libelf 0.153 does not support unwinding yet.
Possible methods are:
- using libunwind;
- manual walk with frame pointers;
- heuristic unwind by inspecting data stack.

This patch adds support for unwinding on aarch64 with frame pointers,
along with changes needed to modularize unwinding.

Change-Id: Ib2cee21277f6bc500046bf6e9d70cf19a733dca8

8 files changed, 299 insertions, 122 deletions

diff --git a/src/crash-stack/CMakeLists.txt b/src/crash-stack/CMakeLists.txt
index b11c4d0..cd30bc3 100644
--- a/src/crash-stack/CMakeLists.txt
+++ b/src/crash-stack/CMakeLists.txt
@@ -1,11 +1,16 @@
 option(WITH_CORE_DUMP "builds with support for core dump files (with GPL2 license)")
 set(CRASH_STACK_BIN "crash-stack")
-set(CRASH_STACK_SRCS crash-stack.c)
+set(CRASH_STACK_SRCS crash-stack.c crash-stack-libelf-helpers.c)
 
 if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "armv7l")
   set(CRASH_STACK_SRCS ${CRASH_STACK_SRCS} crash-stack-arm.c wind/unwarm.c wind/unwarm_thumb.c wind/unwarm_arm.c wind/unwarmmem.c)
 else()
-  set(CRASH_STACK_SRCS ${CRASH_STACK_SRCS} crash-stack-libelf.c)
+  if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64")
+    set(CRASH_STACK_SRCS ${CRASH_STACK_SRCS} crash-stack-aarch64.c)
+  elseif (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64")
+    set(CRASH_STACK_SRCS ${CRASH_STACK_SRCS} crash-stack-libelf.c)
+    set(CRASH_STACK_SRCS ${CRASH_STACK_SRCS} crash-stack-x86_64.c)
+  endif()
 endif()
 
 add_executable(${CRASH_STACK_BIN} ${CRASH_STACK_SRCS})
diff --git a/src/crash-stack/crash-stack-aarch64.c b/src/crash-stack/crash-stack-aarch64.c
new file mode 100644
index 0000000..f9cac96
--- /dev/null
+++ b/src/crash-stack/crash-stack-aarch64.c
@@ -0,0 +1,74 @@
+#include "crash-stack.h"
+#include <sys/user.h>
+#include <string.h>
+
+static struct user_regs_struct g_registers;
+
+struct Regs {
+	Dwarf_Addr x29;
+	Dwarf_Addr x30;
+	Dwarf_Addr pc;
+	Dwarf_Addr sp;
+};
+
+#define REG_SP 32
+#define REG_PC 33
+#define REG_X29 29
+#define REG_X30 30
+
+typedef struct Regs Regs;
+static Regs g_regs;
+
+void *crash_stack_get_memory_for_ptrace_registers(size_t *size)
+{
+	if (NULL != size)
+		*size = sizeof(g_registers);
+	return &g_registers;
+}
+
+void crash_stack_set_ptrace_registers(void *regbuf)
+{
+  struct user_regs_struct *regs = regbuf;
+
+  memcpy(get_place_for_register_value("sp", 0), &regs->sp, sizeof(regs->sp));
+  memcpy(get_place_for_register_value("pc", 0), &regs->pc, sizeof(regs->pc));
+  memcpy(get_place_for_register_value("x29", 0), &regs->regs[29], sizeof(regs->regs[29]));
+  memcpy(get_place_for_register_value("x30", 0), &regs->regs[30], sizeof(regs->regs[30]));
+}
+
+void create_crash_stack(Dwfl *dwfl, Elf *core, pid_t pid, Mappings *mappings, Callstack *callstack)
+{
+	callstack->elems = 0;
+	callstack->tab[callstack->elems++] = g_regs.pc;
+	callstack->tab[callstack->elems++] = g_regs.x30;
+
+	bool end = false;
+
+	do {
+		uint64_t newx29, newx30;
+		bool read29 = crash_stack_libelf_read_value(dwfl, core, pid,
+				g_regs.x29,
+				&newx29, sizeof(newx29), mappings);
+		bool read30 = crash_stack_libelf_read_value(dwfl, core, pid,
+				g_regs.x29 + sizeof(newx29),
+				&newx30, sizeof(newx30), mappings);
+		if (read29 && read30) {
+			callstack->tab[callstack->elems++] = newx30;
+			g_regs.x29 = newx29;
+		}
+		else end = true;
+	} while (!end);
+}
+
+void *get_place_for_register_value(const char *regname, int regnum)
+{
+	if (strcmp(regname, "pc") == 0 || REG_PC == regnum)
+		return &g_regs.pc;
+	else if (strcmp(regname, "sp") == 0 || REG_SP == regnum)
+		return &g_regs.sp;
+	else if (strcmp(regname, "x29") == 0 || REG_X29 == regnum)
+		return &g_regs.x29;
+	else if (strcmp(regname, "x30") == 0 || REG_X30 == regnum)
+		return &g_regs.x30;
+	return NULL;
+}
diff --git a/src/crash-stack/crash-stack-arm.c b/src/crash-stack/crash-stack-arm.c
index c1d2628..acb3ead 100644
--- a/src/crash-stack/crash-stack-arm.c
+++ b/src/crash-stack/crash-stack-arm.c
@@ -3,6 +3,7 @@
 
 #include <string.h>
 #include <sys/ptrace.h>
+#include <sys/user.h>
 
 static Elf *g_core = NULL;
 static Dwfl *g_dwfl = NULL;
@@ -20,6 +21,15 @@ struct Regs {
 typedef struct Regs Regs;
 static Regs g_regs;
 
+static struct user_regs g_ptrace_registers;
+
+void *crash_stack_get_memory_for_ptrace_registers(size_t *size)
+{
+	if (NULL != size)
+		*size = sizeof(g_ptrace_registers);
+	return &g_ptrace_registers;
+}
+
 void *get_place_for_register_value(const char *regname, int regnum)
 {
 	if (strcmp(regname, "pc") == 0 || REG_PC == regnum)
@@ -35,6 +45,17 @@ void *get_place_for_register_value(const char *regname, int regnum)
 	return NULL;
 }
 
+void crash_stack_set_ptrace_registers(void *regbuf)
+{
+	struct user_regs *registers = regbuf;
+	int i;
+	for (i = 0; i < sizeof(registers->uregs)/sizeof(registers->uregs[0]); i++) {
+		void *regmem = get_place_for_register_value("", i);
+		if (NULL != regmem)
+			memcpy(regmem, &registers->uregs[i], sizeof(registers->uregs[i]));
+	}
+}
+
 static Boolean report(void *data, Int32 address)
 {
 	Callstack *callstack = (Callstack *)(data);
@@ -45,68 +66,7 @@ static Boolean report(void *data, Int32 address)
 
 Boolean readT(Int32 a, void *v, size_t size)
 {
-	Dwfl_Module *module = 0;
-	Elf_Data *data = NULL;
-
-	int segment = dwfl_addrsegment(g_dwfl, a, &module);
-
-	if (module != NULL) {
-		Dwarf_Addr start;
-		dwfl_module_info(module, NULL, &start, NULL, NULL, NULL, NULL, NULL);
-
-		GElf_Addr bias;
-		Elf *elf = dwfl_module_getelf(module, &bias);
-
-		data = elf_getdata_rawchunk(elf, a-start, size, ELF_T_BYTE);
-	}
-	if (NULL == data && segment != -1) {
-		// get data from segment
-		GElf_Phdr mem;
-		GElf_Phdr *phdr;
-		Dwarf_Addr offset_in_segment;
-
-		phdr = gelf_getphdr(g_core, segment, &mem);
-		if (phdr != NULL) {
-			offset_in_segment = a - phdr->p_vaddr;
-			if (offset_in_segment < phdr->p_filesz) {
-				Dwarf_Addr offset_in_file = phdr->p_offset + offset_in_segment;
-
-				data = elf_getdata_rawchunk(g_core, offset_in_file, size, ELF_T_BYTE);
-			}
-		}
-	}
-
-	if (NULL == data && module != NULL) {
-		const char *name = dwfl_module_info(module, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
-		if (name != NULL && name[0] == '[') {
-			int i;
-			// get module from mappings
-			for (i = 0; i < g_mappings->elems; i++) {
-				if (g_mappings->tab[i].m_start <= a && a < g_mappings->tab[i].m_end) {
-					// compute offset relative to the start of the mapping
-					Int32 offset = a - g_mappings->tab[i].m_start;
-					// read from the file, but also account file offset
-					data = elf_getdata_rawchunk(g_mappings->tab[i].m_elf,
-							offset + g_mappings->tab[i].m_offset, size, ELF_T_BYTE);
-					break;
-				}
-			}
-		}
-	}
-
-	if (data != NULL) {
-		memcpy(v, data->d_buf, size);
-		return TRUE;
-	}
-
-	/* Still no data, but we have a process - read memory with ptrace */
-	if (NULL == data && g_pid > 1) {
-		long val = ptrace(PTRACE_PEEKDATA, g_pid, a, NULL);
-		memcpy(v, &val, size);
-		return TRUE;
-	}
-
-	return FALSE;
+  return crash_stack_libelf_read_value(g_dwfl, g_core, g_pid, a, v, size, g_mappings);
 }
 
 static Boolean readW(Int32 a, Int32 *v)
@@ -126,50 +86,7 @@ static Boolean readB(Int32 a, Int8 *v)
 
 static Int32 getProloguePC(Int32 current_pc)
 {
-	Int32 result = 0;
-	Dwfl_Module *module = dwfl_addrmodule(g_dwfl, current_pc);
-	if (module) {
-		//        GElf_Off offset;
-		GElf_Sym sym;
-		//        dwfl_module_addrinfo (module, current_pc, &offset, &sym, NULL, NULL, NULL);
-		dwfl_module_addrsym(module, current_pc, &sym, NULL);
-		//        result = current_pc - offset;
-		result = sym.st_value;
-	}
-	if (0 == result) {
-		int i;
-		for (i = 0; i < g_mappings->elems; i++) {
-			if (g_mappings->tab[i].m_start <= current_pc && current_pc < g_mappings->tab[i].m_end) {
-				/* go through symbols to find the nearest */
-				Elf_Scn *scn = NULL;
-				Elf *elf = g_mappings->tab[i].m_elf;
-				while ((scn = elf_nextscn(elf, scn)) != NULL) {
-					GElf_Shdr shdr_mem;
-					GElf_Shdr *shdr = gelf_getshdr(scn, &shdr_mem);
-					if (shdr != NULL && (shdr->sh_type == SHT_SYMTAB || shdr->sh_type == SHT_DYNSYM)) {
-						Elf_Data *sdata = elf_getdata(scn, NULL);
-						unsigned int nsyms = sdata->d_size / (gelf_getclass(elf) == ELFCLASS32 ?
-								sizeof(Elf32_Sym) :
-								sizeof(Elf64_Sym));
-						unsigned int cnt;
-						uintptr_t address_offset = current_pc;
-						if (shdr->sh_type == SHT_DYNSYM)
-							address_offset -= g_mappings->tab[i].m_start;
-						for (cnt = 0; cnt < nsyms; ++cnt) {
-							GElf_Sym sym_mem;
-							Elf32_Word xndx;
-							GElf_Sym *sym = gelf_getsymshndx(sdata, NULL, cnt, &sym_mem, &xndx);
-							if (sym != NULL && sym->st_shndx != SHN_UNDEF) {
-								if (sym->st_value <= address_offset && address_offset < sym->st_value + sym->st_size)
-									return sym->st_value;
-							}
-						}
-					}
-				}
-			}
-		}
-	}
-	return result;
+    return crash_stack_libelf_get_prologue_pc(g_dwfl, current_pc, g_mappings);
 }
 
 void create_crash_stack(Dwfl *dwfl, Elf *core, pid_t pid, Mappings *mappings, Callstack *callstack)
diff --git a/src/crash-stack/crash-stack-libelf-helpers.c b/src/crash-stack/crash-stack-libelf-helpers.c
new file mode 100644
index 0000000..7dab540
--- /dev/null
+++ b/src/crash-stack/crash-stack-libelf-helpers.c
@@ -0,0 +1,117 @@
+#include "crash-stack.h"
+#include <string.h>
+#include <sys/ptrace.h>
+#include <errno.h>
+
+bool crash_stack_libelf_read_value(Dwfl *dwfl, Elf *core, pid_t pid,
+		Dwarf_Addr a, void *v, size_t size,
+		Mappings *mappings)
+{
+	Dwfl_Module *module = 0;
+	Elf_Data *data = NULL;
+
+	int segment = dwfl_addrsegment(dwfl, a, &module);
+
+	if (module != NULL) {
+		Dwarf_Addr start;
+		dwfl_module_info(module, NULL, &start, NULL, NULL, NULL, NULL, NULL);
+
+		GElf_Addr bias;
+		Elf *elf = dwfl_module_getelf(module, &bias);
+
+		data = elf_getdata_rawchunk(elf, a-start, size, ELF_T_BYTE);
+	}
+	if (NULL == data && segment != -1) {
+		// get data from segment
+		GElf_Phdr mem;
+		GElf_Phdr *phdr = gelf_getphdr(core, segment, &mem);
+		Dwarf_Addr offset_in_segment = a - phdr->p_vaddr;
+		if (offset_in_segment < phdr->p_filesz) {
+			Dwarf_Addr offset_in_file = phdr->p_offset + offset_in_segment;
+
+			data = elf_getdata_rawchunk(core, offset_in_file, size, ELF_T_BYTE);
+		}
+	}
+
+	if (NULL == data && module != NULL) {
+		const char *name = dwfl_module_info(module, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+		if (name != NULL && name[0] == '[') {
+			int i;
+			// get module from mappings
+			for (i = 0; i < mappings->elems; i++) {
+				if (mappings->tab[i].m_start <= a && a < mappings->tab[i].m_end) {
+					// compute offset relative to the start of the mapping
+					long offset = a - mappings->tab[i].m_start;
+					// read from the file, but also account file offset
+					data = elf_getdata_rawchunk(mappings->tab[i].m_elf,
+							offset + mappings->tab[i].m_offset, size, ELF_T_BYTE);
+					break;
+				}
+			}
+		}
+	}
+
+	if (data != NULL) {
+		memcpy(v, data->d_buf, size);
+		return true;
+	}
+
+	/* Still no data, but we have a process - read memory with ptrace */
+	/* FIXME need to know if we are still in the mapped area */
+	/* Bigger issue is that dwfl does not have modules */
+	if (pid > 1) {
+		long val = ptrace(PTRACE_PEEKDATA, pid, a, NULL);
+		if (-1 == val && errno)
+			return false;
+		memcpy(v, &val, size);
+		return true;
+	}
+
+	return false;
+}
+
+Dwarf_Addr crash_stack_libelf_get_prologue_pc(Dwfl *dwfl, Dwarf_Addr current_pc, Mappings *mappings)
+{
+	Dwarf_Addr result = 0;
+	Dwfl_Module *module = dwfl_addrmodule(dwfl, current_pc);
+	if (module) {
+		GElf_Sym sym;
+		dwfl_module_addrsym(module, current_pc, &sym, NULL);
+		result = sym.st_value;
+	}
+	if (0 == result) {
+		int i;
+		for (i=0; i < mappings->elems; i++) {
+			if (mappings->tab[i].m_start <= current_pc && current_pc < mappings->tab[i].m_end) {
+				/* go through symbols to find the nearest */
+				Elf_Scn *scn = NULL;
+				Elf *elf = mappings->tab[i].m_elf;
+				while ((scn = elf_nextscn(elf, scn)) != NULL) {
+					GElf_Shdr shdr_mem;
+					GElf_Shdr *shdr = gelf_getshdr(scn, &shdr_mem);
+					if (shdr != NULL && (shdr->sh_type == SHT_SYMTAB || shdr->sh_type == SHT_DYNSYM)) {
+						Elf_Data *sdata = elf_getdata(scn, NULL);
+						unsigned int nsyms = sdata->d_size / (gelf_getclass(elf) == ELFCLASS32 ?
+								sizeof(Elf32_Sym) :
+								sizeof(Elf64_Sym));
+						unsigned int cnt;
+						uintptr_t address_offset = current_pc;
+						if (shdr->sh_type == SHT_DYNSYM)
+							address_offset -= mappings->tab[i].m_start;
+						for (cnt = 0; cnt < nsyms; ++cnt) {
+							GElf_Sym sym_mem;
+							Elf32_Word xndx;
+							GElf_Sym *sym = gelf_getsymshndx(sdata, NULL, cnt, &sym_mem, &xndx);
+							if (sym != NULL && sym->st_shndx != SHN_UNDEF) {
+								if (sym->st_value <= address_offset && address_offset < sym->st_value + sym->st_size) {
+									return sym->st_value;
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return result;
+}
diff --git a/src/crash-stack/crash-stack-libelf.c b/src/crash-stack/crash-stack-libelf.c
index f4c10bc..b152bea 100644
--- a/src/crash-stack/crash-stack-libelf.c
+++ b/src/crash-stack/crash-stack-libelf.c
@@ -1,6 +1,16 @@
 #include "crash-stack.h"
 #include <elfutils/libdwfl.h>
 #include <elfutils/version.h>
+#include <string.h>
+
+typedef union {
+	uint16_t reg16;
+	uint32_t reg32;
+	uint64_t reg64;
+} Register;
+
+static Register g_pc;
+static Register g_sp;
 
 #if _ELFUTILS_PREREQ(0, 158)
 static int frame_callback(Dwfl_Frame *state, void *arg)
@@ -19,9 +29,32 @@ static int thread_callback(Dwfl_Thread *thread, void *thread_arg)
 }
 #endif
 
+static const char *pc_names[] = {
+  "pc", "rip", "eip", "ip"
+};
+
+static const char *sp_names[] = {
+  "sp", "rsp", "esp"
+};
+
+static bool is_in(const char *name, const char **names, int elems)
+{
+	int nit;
+	for (nit = 0; nit < elems; ++nit) {
+		if (strcmp(name, names[nit]) == 0)
+			return true;
+	}
+	return false;
+}
+
+#define IS_IN(name,names) is_in((name), (names), sizeof(names)/sizeof(names[0]))
+
 void *get_place_for_register_value(const char *regname, int regnum)
 {
-	return 0;
+  if (IS_IN(regname, pc_names)) return &g_pc;
+  else if (IS_IN(regname, sp_names)) return &g_sp;
+
+  return 0;
 }
 
 void create_crash_stack(Dwfl *dwfl, Elf *core, pid_t pid, Mappings *mappings, Callstack *callstack)
diff --git a/src/crash-stack/crash-stack-x86_64.c b/src/crash-stack/crash-stack-x86_64.c
new file mode 100644
index 0000000..934ee72
--- /dev/null
+++ b/src/crash-stack/crash-stack-x86_64.c
@@ -0,0 +1,23 @@
+#include "crash-stack.h"
+#include <sys/user.h>
+#include <string.h>
+
+struct user_regs_struct g_registers;
+
+void *crash_stack_get_memory_for_ptrace_registers(size_t *size)
+{
+	if (NULL != size)
+		*size = sizeof(g_registers);
+	return &g_registers;
+}
+
+void crash_stack_set_ptrace_registers(void *regbuf)
+{
+	void *rsp = get_place_for_register_value("rsp", 0);
+	void *rip = get_place_for_register_value("rip", 0);
+
+	struct user_regs_struct *regs = regbuf;
+
+	memcpy(rsp, &regs->rsp, sizeof(regs->rsp));
+	memcpy(rip, &regs->rip, sizeof(regs->rip));
+}
diff --git a/src/crash-stack/crash-stack.c b/src/crash-stack/crash-stack.c
index 609ac2f..916f645 100644
--- a/src/crash-stack/crash-stack.c
+++ b/src/crash-stack/crash-stack.c
@@ -62,9 +62,19 @@ static int module_callback(Dwfl_Module *module, void **userdata,
 
 static void getvalue(Elf *core, const void *from, size_t size, void *to)
 {
+	Elf_Type type = ELF_T_BYTE;
+	switch (size) {
+	case 8: type = ELF_T_BYTE; break;
+	case 16: type = ELF_T_HALF; break;
+	case 32: type = ELF_T_WORD; break;
+	case 64: type = ELF_T_XWORD; break;
+	default:
+		 fprintf(stderr, "getvalue for strange size: %llu\n", (unsigned long long)size);
+		 break;
+	}
 	Elf_Data out = {
 		.d_buf = to,
-		.d_type = size == 32 ? ELF_T_WORD : ELF_T_XWORD,
+		.d_type = type,
 		.d_version = EV_CURRENT,
 		.d_size = size/8,
 		.d_off = 0,
@@ -282,25 +292,15 @@ static Dwfl *open_dwfl_with_core(Elf *core, const char *core_file_name)
 static int get_registers_ptrace(pid_t pid)
 {
 	struct iovec data;
-	uintptr_t regbuf[20];
-
-	data.iov_base = regbuf;
-	data.iov_len = sizeof(regbuf);
+	data.iov_base = crash_stack_get_memory_for_ptrace_registers( &data.iov_len );
 
 	if (ptrace(PTRACE_GETREGSET, pid, NT_PRSTATUS, &data) != 0) {
 		fprintf(errfile, "PTRACE_GETREGSET failed on PID %d: %m\n", pid);
 		return -1;
 	}
 
-	size_t i;
-	for (i = 0;
-			i * sizeof(regbuf[0]) < data.iov_len && i < sizeof(regbuf)/sizeof(regbuf[0]);
-			i++) {
-		void *reg = get_place_for_register_value("", i);
+	crash_stack_set_ptrace_registers(data.iov_base);
 
-		if (NULL != reg)
-			memcpy(reg, &regbuf[i], sizeof(regbuf[i]));
-	}
 	return 0;
 }
 
diff --git a/src/crash-stack/crash-stack.h b/src/crash-stack/crash-stack.h
index bf712c0..195bce5 100644
--- a/src/crash-stack/crash-stack.h
+++ b/src/crash-stack/crash-stack.h
@@ -31,4 +31,12 @@ typedef struct Mappings {
 void *get_place_for_register_value(const char *regname, int regnum);
 void create_crash_stack(Dwfl *dwfl, Elf *core, pid_t pid, Mappings *mappings, Callstack *callstack);
 
+Dwarf_Addr crash_stack_libelf_get_prologue_pc(Dwfl *dwfl, Dwarf_Addr current_pc, Mappings *mappings);
+bool crash_stack_libelf_read_value(Dwfl *dwfl, Elf *core, pid_t pid,
+                                   Dwarf_Addr a, void *v, size_t size,
+                                   Mappings *mappings);
+
+void *crash_stack_get_memory_for_ptrace_registers(size_t *size);
+void crash_stack_set_ptrace_registers(void *regbuf);
+
 #endif /* CRASH_STACK_H */