#define _GNU_SOURCE /* * Copyright 2007,2012 Intel Corporation * * This program file is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; version 2 of the License. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * for more details. * * You should have received a copy of the GNU General Public License * along with this program in a file named COPYING; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301 USA * * Authors: * Arjan van de Ven * William Douglas * Tim Pepper */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "corewatcher.h" /* * processing "queue" loop's condition variable and associated * lock. Note the queue is an implicit data structure consisting * of the non-submitted core files in the filesystem, but the bool pq is * used to mark whether the "queue" holds something to prevent the possible * race where the condition is set before the thread is awaiting it and * thus is not woken. */ GMutex *pq_mtx; static gboolean pq = FALSE; GCond *pq_work; static int diskfree = 100; static char *get_release(void) { FILE *file = NULL; char *line = NULL; size_t dummy = 0; file = fopen("/etc/os-release", "r"); if (!file) return NULL; while (!feof(file)) { if (getline(&line, &dummy, file) == -1) break; if (strstr(line, "VERSION_ID=")) { char *c = NULL; c = strchr(line, '\n'); if (c) { *c = 0; c = strdup(&line[11]); fclose(file); free(line); return c; } } } fclose(file); free(line); return NULL; } /* * Strip the directories from the path * given by fullname */ char *strip_directories(char *fullpath) { char *dfile = NULL, *c1 = NULL, *c2 = NULL, *r = NULL; char delim[] = "/"; char *saveptr; if (!fullpath) return NULL; dfile = strdup(fullpath); if (!dfile) return NULL; c1 = strtok_r(dfile, delim, &saveptr); while(c1) { c2 = c1; c1 = strtok_r(NULL, delim, &saveptr); } if (c2) r = strdup(c2); free(dfile); return r; } /* * Move corefile from core_folder to processed_folder subdir. * If this type of core has recently been seen, unlink this more recent * example in order to rate limit submissions of extremely crashy * applications. * Add extension and attempt to create directories if needed. */ static int move_core(char *fullpath, char *extension) { char *corefilename = NULL, *newpath = NULL, *coreprefix = NULL; char *s = NULL; size_t prefix_len; DIR *dir = NULL; struct dirent *entry = NULL; int ret = 0; if (!fullpath) return -1; corefilename = strip_directories(fullpath); if (!corefilename) return -ENOMEM; /* if the corefile's name minus any suffixes (such as .$PID) and * minus two additional characters (ie: last two digits of * timestamp assuming core_%e_%t) matches another core file in the * processed_folder, simply unlink it instead of processing it for * submission. TODO: consider a (configurable) time delta greater * than which the cores must be separated, stat'ing the files, etc. */ coreprefix = strdup(corefilename); if (!coreprefix) { free(corefilename); return -ENOMEM; } s = strstr(coreprefix, "."); if (!s) { free(coreprefix); free(corefilename); return -1; } *s = '\0'; prefix_len = strlen(coreprefix); if (prefix_len > 2) { s = strndup(coreprefix, prefix_len - 2); free(coreprefix); coreprefix = s; } else { ret = -1; goto out; } dir = opendir(processed_folder); if (!dir) { ret = -1; goto out; } while(1) { entry = readdir(dir); if (!entry || !entry->d_name) break; if (entry->d_name[0] == '.') continue; if (!strstr(entry->d_name, coreprefix)) continue; break; } closedir(dir); free(coreprefix); coreprefix = NULL; if (asprintf(&newpath, "%s%s.%s", processed_folder, corefilename, extension) == -1) { ret = -1; goto out; } free(corefilename); corefilename = NULL; rename(fullpath, newpath); free(newpath); return 0; out: free(coreprefix); free(corefilename); fprintf(stderr, "+ ...move failed, ignoring/unlinking %s\n", fullpath); unlink(fullpath); return ret; } static void skip_core(char *fullpath, char *extension) { char *procfn; int ret; procfn = replace_name(fullpath, extension, ".skipped"); if (!procfn) { fprintf(stderr, "+ Problems with filename manipulation for %s\n", fullpath); return; } ret = rename(fullpath, procfn); if (ret) { fprintf(stderr, "+ Unable to move %s to %s\n", fullpath, procfn); free(procfn); return; } fprintf(stderr, "+ Moved %s to %s\n", fullpath, procfn); free(procfn); return; } /* * Use GDB to extract backtrace information from corefile */ static struct oops *extract_core(char *fullpath, char *appfile, char *reportname) { struct oops *oops = NULL; int ret = 0; char *command = NULL, *h1 = NULL, *c1 = NULL, *c2 = NULL, *line = NULL; char *text = NULL, *coretime = NULL; char *m1 = NULL, *m2 = NULL; int bt_lines = 0, maps_lines = 0; FILE *file = NULL; char *badchar = NULL; char *release = get_release(); int parsing_maps = 0; struct stat stat_buf; size_t size = 0; ssize_t bytesread = 0; fprintf(stderr, "+ extract_core() called for %s\n", fullpath); if (asprintf(&command, "LANG=C gdb --batch -f '%s' '%s' -x /etc/corewatcher/gdb.command 2>&1", appfile, fullpath) == -1) return NULL; file = popen(command, "r"); free(command); if (!file) fprintf(stderr, "+ gdb failed for %s\n", fullpath); if (stat(fullpath, &stat_buf) != -1) { coretime = malloc(26); if (coretime) ctime_r(&stat_buf.st_mtime, coretime); } ret = asprintf(&h1, "cmdline: %s\n" "release: %s\n" "time: %s", appfile, release ? release : "Unknown", coretime ? coretime : "Unknown"); if (release) free(release); if (coretime) free(coretime); if (ret == -1) return NULL; while (file && !feof(file)) { bytesread = getline(&line, &size, file); if (!size) break; if (bytesread == -1) break; /* gdb outputs (to stderr) many "warning: " strings * We can't trust the gdb 'bt' if these two are seen. */ if ((strncmp(line, "warning: core file may not match specified executable file.", 59) == 0) || (strncmp(line, "warning: exec file is newer than core file.", 43) == 0)) { free(line); pclose(file); free(h1); fprintf(stderr, "+ core/executable mismatch for %s\n", fullpath); return NULL; } /* try to figure out if we're onto the maps output yet */ if (strncmp(line, "From", 4) == 0) { parsing_maps = 1; } /* maps might not be present */ if (strncmp(line, "No shared libraries", 19) == 0) { break; } if (!parsing_maps) { /* parsing backtrace */ c2 = c1; /* gdb's backtrace lines start with a line number */ if (line[0] != '#') continue; /* gdb prints some initial info which may include the * "#0" line of the backtrace, then prints the * backtrace in its entirety, leading to a * duplicate "#0" in our summary if we do do: */ if ((bt_lines == 1) && (strncmp(line, "#0 ", 3) == 0)) continue; bt_lines++; /* gdb outputs some 0x1a's which break XML */ do { badchar = memchr(line, 0x1a, bytesread); if (badchar) *badchar = ' '; } while (badchar); if (c1) { c1 = NULL; if (asprintf(&c1, "%s %s", c2, line) == -1) continue; free(c2); } else { /* keep going even if asprintf has errors */ ret = asprintf(&c1, " %s", line); } } else { /* parsing maps */ m2 = m1; maps_lines++; if (m1) { m1 = NULL; if (asprintf(&m1, "%s %s", m2, line) == -1) continue; free(m2); } else { /* keep going even if asprintf has errors */ ret = asprintf(&m1, " %s", line); } } } if (line) free(line); if (file) pclose(file); ret = asprintf(&text, "%s" "backtrace: |\n" "%s" "maps: |\n" "%s", h1, c1 ? c1 : " Unknown\n", m1 ? m1 : " Unknown\n"); free(h1); if (c1) free(c1); if (m1) free(m1); if (ret == -1) return NULL; oops = malloc(sizeof(struct oops)); if (!oops) { free(text); return NULL; } memset(oops, 0, sizeof(struct oops)); oops->next = NULL; oops->application = strdup(appfile); oops->text = text; oops->filename = strdup(fullpath); oops->detail_filename = strdup(reportname); return oops; } /* * input filename has the form: core_$APP_$TIMESTAMP[.$PID] * output filename has form of: $APP_$TIMESTAMP.txt */ static char *make_report_filename(char *filename) { char *name = NULL, *dotpid = NULL, *stamp = NULL, *detail_filename = NULL; if (!filename) return NULL; if (!(stamp = strstr(filename, "_"))) return NULL; if (!(++stamp)) return NULL; if (!(name = strdup(stamp))) return NULL; /* strip trailing .PID if present */ dotpid = strstr(name, "."); if (dotpid) *dotpid = '\0'; if ((asprintf(&detail_filename, "%s%s.txt", processed_folder, name)) == -1) { free(name); return NULL; } free(name); return detail_filename; } /* * Write the backtrace from the core file into a text * file named as $APP_$TIMESTAMP.txt */ static void write_core_detail_file(struct oops *oops) { int fd = 0; if (!oops->detail_filename) return; fd = open(oops->detail_filename, O_WRONLY | O_CREAT | O_TRUNC, 0); if (fd == -1) { fprintf(stderr, "+ Error creating/opening %s for write\n", oops->detail_filename); return; } if(write(fd, oops->text, strlen(oops->text)) >= 0) { fprintf(stderr, "+ Wrote %s\n", oops->detail_filename); fchmod(fd, 0644); } else { fprintf(stderr, "+ Error writing %s\n", oops->detail_filename); unlink(oops->detail_filename); } close(fd); } /* * Creates $APP_$TIMESTAMP.txt report summaries if they don't exist and * adds the oops struct to the submit queue */ static void *create_report(char *fullpath) { struct oops *oops = NULL; char *procfn = NULL; char *app = NULL, *appname = NULL, *appfile = NULL, *corefn = NULL, *reportname = NULL; char *old_ext = ".processed"; char *new_ext = ".to-process"; char *ext = NULL; struct stat stat_buf; int new = 0, ret; fprintf(stderr, "+ Entered create_report() for %s\n", fullpath); if (strstr(fullpath, ".to-process")) { new = 1; ext = new_ext; } else if (strstr(fullpath, ".processed")) { ext = old_ext; } else if (strstr(fullpath, ".skipped")) { fprintf(stderr, "+ Already skipped\n"); return NULL; } else { /* bad state */ fprintf(stderr, "+ Missing extension? (%s)\n", fullpath); unlink(fullpath); return NULL; } corefn = strip_directories(fullpath); if (!corefn) { fprintf(stderr, "+ No corefile? (%s)\n", fullpath); return NULL; } /* don't process rpm, gdb or corewatcher crashes */ appname = find_causingapp(fullpath); if (!appname) { fprintf(stderr, "+ No appname in %s\n", corefn); skip_core(fullpath, ext); free(corefn); return NULL; } app = strip_directories(appname); if (!app || !strncmp(app, "rpm", 3) || !strncmp(app, "gdb", 3) || !strncmp(app, "corewatcher", 11)) { fprintf(stderr, "+ ...skipping %s's %s\n", app, corefn); skip_core(fullpath, ext); free(corefn); free(appname); free(app); return NULL; } /* also skip apps which don't appear to be part of the OS */ appfile = find_apppath(appname); if (!appfile) { fprintf(stderr, "+ ...skipping %s's %s\n", appname, corefn); skip_core(fullpath, ext); free(corefn); free(appname); free(app); return NULL; } free(appname); free(app); reportname = make_report_filename(corefn); if (!reportname) { fprintf(stderr, "+ Couldn't make report name for %s\n", corefn); free(corefn); free(appfile); return NULL; } free(corefn); if (stat(reportname, &stat_buf) == 0) { int fd, ret; /* * TODO: * If the file already had trailing ".processed" but the txt file * is a low quality report, then create a new report. */ fprintf(stderr, "+ Report already exists in %s\n", reportname); oops = malloc(sizeof(struct oops)); if (!oops) { fprintf(stderr, "+ Malloc failed for struct oops\n"); free(reportname); free(appfile); return NULL; } memset(oops, 0, sizeof(struct oops)); oops->next = NULL; oops->application = strdup(appfile); oops->filename = strdup(fullpath); oops->detail_filename = strdup(reportname); free(reportname); free(appfile); oops->text = malloc(stat_buf.st_size + 1); if (!oops->text) { fprintf(stderr, "+ Malloc failed for oops text\n"); goto err; } fd = open(oops->detail_filename, O_RDONLY); if (fd == -1) { fprintf(stderr, "+ Open failed for oops text\n"); goto err; } ret = read(fd, oops->text, stat_buf.st_size); close(fd); if (ret != stat_buf.st_size) { fprintf(stderr, "+ Read failed for oops text\n"); goto err; } oops->text[stat_buf.st_size] = '\0'; } else { oops = extract_core(fullpath, appfile, reportname); free(reportname); free(appfile); if (!oops) { fprintf(stderr, "+ Did not generate struct oops for %s\n", fullpath); skip_core(fullpath, ext); return NULL; } write_core_detail_file(oops); } if (new) { fprintf(stderr, "+ Renaming %s (%s -> %s)\n", fullpath, new_ext, old_ext); procfn = replace_name(fullpath, new_ext, old_ext); if (!procfn) { fprintf(stderr, "+ Problems with filename manipulation for %s\n", fullpath); return oops; } ret = rename(fullpath, procfn); if (ret) { fprintf(stderr, "+ Unable to move %s to %s\n", fullpath, procfn); free(procfn); return oops; } free(oops->filename); oops->filename = strdup(procfn); free(procfn); } return oops; err: FREE_OOPS(oops); return NULL; } /* * scan once for core files in core_folder, moving any to the * processed_folder with ".to-process" appended to their name */ int scan_core_folder(void __unused *unused) { DIR *dir = NULL; struct dirent *entry = NULL; char *fullpath = NULL; int ret, work = 0; dir = opendir(core_folder); if (!dir) { fprintf(stderr, "+ Unable to open %s\n", core_folder); return -1; } fprintf(stderr, "+ Begin scanning %s...\n", core_folder); while(1) { entry = readdir(dir); if (!entry || !entry->d_name) break; if (entry->d_name[0] == '.') continue; if (strncmp(entry->d_name, "core_", 5)) continue; /* matched core_#### */ if (asprintf(&fullpath, "%s%s", core_folder, entry->d_name) == -1) { fullpath = NULL; continue; } /* If one were to prompt the user before submitting, that * might happen here. */ fprintf(stderr, "+ Looking at %s\n", fullpath); ret = move_core(fullpath, "to-process"); if (ret == 0) work++; free(fullpath); fullpath = NULL; } closedir(dir); if (work) { fprintf(stderr, "+ Found %d files, setting pq_work condition\n", work); g_mutex_lock(pq_mtx); g_cond_signal(pq_work); pq = TRUE; g_mutex_unlock(pq_mtx); } fprintf(stderr, "+ End scanning %s...\n", core_folder); return TRUE; } /* * scan for core_*.to-process and core_*.processed, * insure a summary *.txt report exists, then queue it */ void *scan_processed_folder(void __unused *unused) { DIR *dir = NULL; struct dirent *entry = NULL; char *fullpath = NULL; struct oops *oops = NULL; while(1) { g_mutex_lock(pq_mtx); while (pq != TRUE) { fprintf(stderr, "+ Awaiting work in %s...\n", processed_folder); g_cond_wait(pq_work, pq_mtx); } pq = FALSE; g_mutex_unlock(pq_mtx); fprintf(stderr, "+ Begin scanning %s...\n", processed_folder); dir = opendir(processed_folder); if (!dir) { fprintf(stderr, "+ Unable to open %s\n", processed_folder); continue; } while(1) { entry = readdir(dir); if (!entry || !entry->d_name) break; if (entry->d_name[0] == '.') continue; /* files with trailing ".to-process" or "processed" represent new work */ if (!strstr(entry->d_name, "process")) continue; if (asprintf(&fullpath, "%s%s", processed_folder, entry->d_name) == -1) { fullpath = NULL; continue; } fprintf(stderr, "+ Looking at %s\n", fullpath); oops = create_report(fullpath); if (oops) { fprintf(stderr, "+ Queued backtrace from %s\n", oops->detail_filename); queue_backtrace(oops); } free(fullpath); fullpath = NULL; } closedir(dir); fprintf(stderr, "+ End scanning %s...\n", processed_folder); } return NULL; } static void disable_corefiles(int diskfree) { int ret; ret = system("echo \"\" > /proc/sys/kernel/core_pattern"); if (ret != -1) { fprintf(stderr, "+ disabled core pattern, disk low %d%%\n", diskfree); syslog(LOG_WARNING, "corewatcher: disabled kernel core_pattern, %s only has %d%% available", core_folder, diskfree); } } void enable_corefiles(int diskfree) { int ret; char * proc_core_string = NULL; ret = asprintf(&proc_core_string, "echo \"%score_%%e_%%t\" > /proc/sys/kernel/core_pattern", core_folder); if (ret == -1) goto err; ret = system(proc_core_string); free(proc_core_string); if (ret == -1) goto err; proc_core_string = NULL; ret = asprintf(&proc_core_string, "echo 1 > /proc/sys/kernel/core_uses_pid"); if (ret == -1) goto err; ret = system(proc_core_string); free(proc_core_string); if (ret == -1) goto err; if (diskfree == -1) { fprintf(stderr, "+ enabled core pattern\n"); syslog(LOG_INFO, "corewatcher: enabled kernel core_pattern\n"); } else { fprintf(stderr, "+ reenabled core pattern, disk %d%%", diskfree); syslog(LOG_WARNING, "corewatcher: reenabled kernel core_pattern, %s now has %d%% available", core_folder, diskfree); } return; err: fprintf(stderr, "+ unable to enable core pattern\n"); syslog(LOG_WARNING, "corewatcher: unable to enable kernel core_pattern\n"); return; } /* do everything, called from timer event */ int scan_folders(void __unused *unused) { struct statvfs stat; int newdiskfree; if (statvfs(core_folder, &stat) == 0) { newdiskfree = (int)(100 * stat.f_bavail / stat.f_blocks); if ((newdiskfree < 10) && (diskfree >= 10)) disable_corefiles(newdiskfree); if ((newdiskfree > 12) && (diskfree <= 12)) enable_corefiles(newdiskfree); diskfree = newdiskfree; } scan_core_folder(NULL); g_mutex_lock(pq_mtx); g_cond_signal(pq_work); pq = TRUE; g_mutex_unlock(pq_mtx); return TRUE; }