diff options
| author | Yonghee Han <onstudy@samsung.com> | 2016-07-27 16:43:51 +0900 |
|---|---|---|
| committer | Yonghee Han <onstudy@samsung.com> | 2016-07-27 01:00:25 -0700 |
| commit | 186efde2677c31fb40d154a81a5f3731eab52414 (patch) | |
| tree | b43c1e7ee15fbdade66764b4b45f40dd3fad408e /scripts | |
| parent | a03c4728275d119af5f66c4a69e8d9d5a1730031 (diff) | |
| download | qemu-186efde2677c31fb40d154a81a5f3731eab52414.tar.gz qemu-186efde2677c31fb40d154a81a5f3731eab52414.tar.bz2 qemu-186efde2677c31fb40d154a81a5f3731eab52414.zip | |
Imported Upstream version 2.6.0upstream/2.6.0
Change-Id: I8e3ccf55257695533c385aa8706484c73a733251
Diffstat (limited to 'scripts')
38 files changed, 2191 insertions, 1977 deletions
diff --git a/scripts/acpi_extract.py b/scripts/acpi_extract.py deleted file mode 100755 index 10c1ffb36..000000000 --- a/scripts/acpi_extract.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2011 Red Hat, Inc., Michael S. Tsirkin <mst@redhat.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -# Process mixed ASL/AML listing (.lst file) produced by iasl -l -# Locate and execute ACPI_EXTRACT directives, output offset info -# -# Documentation of ACPI_EXTRACT_* directive tags: -# -# These directive tags output offset information from AML for BIOS runtime -# table generation. -# Each directive is of the form: -# ACPI_EXTRACT_<TYPE> <array_name> <Operator> (...) -# and causes the extractor to create an array -# named <array_name> with offset, in the generated AML, -# of an object of a given type in the following <Operator>. -# -# A directive must fit on a single code line. -# -# Object type in AML is verified, a mismatch causes a build failure. -# -# Directives and operators currently supported are: -# ACPI_EXTRACT_NAME_DWORD_CONST - extract a Dword Const object from Name() -# ACPI_EXTRACT_NAME_WORD_CONST - extract a Word Const object from Name() -# ACPI_EXTRACT_NAME_BYTE_CONST - extract a Byte Const object from Name() -# ACPI_EXTRACT_METHOD_STRING - extract a NameString from Method() -# ACPI_EXTRACT_NAME_STRING - extract a NameString from Name() -# ACPI_EXTRACT_PROCESSOR_START - start of Processor() block -# ACPI_EXTRACT_PROCESSOR_STRING - extract a NameString from Processor() -# ACPI_EXTRACT_PROCESSOR_END - offset at last byte of Processor() + 1 -# ACPI_EXTRACT_PKG_START - start of Package block -# -# ACPI_EXTRACT_ALL_CODE - create an array storing the generated AML bytecode -# -# ACPI_EXTRACT is not allowed anywhere else in code, except in comments. - -import re; -import sys; -import fileinput; - -aml = [] -asl = [] -output = {} -debug = "" - -class asl_line: - line = None - lineno = None - aml_offset = None - -def die(diag): - sys.stderr.write("Error: %s; %s\n" % (diag, debug)) - sys.exit(1) - -#Store an ASL command, matching AML offset, and input line (for debugging) -def add_asl(lineno, line): - l = asl_line() - l.line = line - l.lineno = lineno - l.aml_offset = len(aml) - asl.append(l) - -#Store an AML byte sequence -#Verify that offset output by iasl matches # of bytes so far -def add_aml(offset, line): - o = int(offset, 16); - # Sanity check: offset must match size of code so far - if (o != len(aml)): - die("Offset 0x%x != 0x%x" % (o, len(aml))) - # Strip any trailing dots and ASCII dump after " - line = re.sub(r'\s*\.*\s*".*$',"", line) - # Strip traling whitespace - line = re.sub(r'\s+$',"", line) - # Strip leading whitespace - line = re.sub(r'^\s+',"", line) - # Split on whitespace - code = re.split(r'\s+', line) - for c in code: - # Require a legal hex number, two digits - if (not(re.search(r'^[0-9A-Fa-f][0-9A-Fa-f]$', c))): - die("Unexpected octet %s" % c); - aml.append(int(c, 16)); - -# Process aml bytecode array, decoding AML -def aml_pkglen_bytes(offset): - # PkgLength can be multibyte. Bits 8-7 give the # of extra bytes. - pkglenbytes = aml[offset] >> 6; - return pkglenbytes + 1 - -def aml_pkglen(offset): - pkgstart = offset - pkglenbytes = aml_pkglen_bytes(offset) - pkglen = aml[offset] & 0x3F - # If multibyte, first nibble only uses bits 0-3 - if ((pkglenbytes > 1) and (pkglen & 0x30)): - die("PkgLen bytes 0x%x but first nibble 0x%x expected 0x0X" % - (pkglen, pkglen)) - offset += 1 - pkglenbytes -= 1 - for i in range(pkglenbytes): - pkglen |= aml[offset + i] << (i * 8 + 4) - if (len(aml) < pkgstart + pkglen): - die("PckgLen 0x%x at offset 0x%x exceeds AML size 0x%x" % - (pkglen, offset, len(aml))) - return pkglen - -# Given method offset, find its NameString offset -def aml_method_string(offset): - #0x14 MethodOp PkgLength NameString MethodFlags TermList - if (aml[offset] != 0x14): - die( "Method offset 0x%x: expected 0x14 actual 0x%x" % - (offset, aml[offset])); - offset += 1; - pkglenbytes = aml_pkglen_bytes(offset) - offset += pkglenbytes; - return offset; - -# Given name offset, find its NameString offset -def aml_name_string(offset): - #0x08 NameOp NameString DataRef - if (aml[offset] != 0x08): - die( "Name offset 0x%x: expected 0x08 actual 0x%x" % - (offset, aml[offset])); - offset += 1 - # Block Name Modifier. Skip it. - if (aml[offset] == 0x5c or aml[offset] == 0x5e): - offset += 1 - return offset; - -# Given data offset, find variable length byte buffer offset -def aml_data_buffer(offset, length): - #0x11 PkgLength BufferSize ByteList - if (length > 63): - die( "Name offset 0x%x: expected a one byte PkgLength (length<=63)" % - (offset)); - expect = [0x11, length+3, 0x0A, length] - if (aml[offset:offset+4] != expect): - die( "Name offset 0x%x: expected %s actual %s" % - (offset, expect, aml[offset:offset+4])) - return offset + len(expect) - -# Given data offset, find dword const offset -def aml_data_dword_const(offset): - #0x08 NameOp NameString DataRef - if (aml[offset] != 0x0C): - die( "Name offset 0x%x: expected 0x0C actual 0x%x" % - (offset, aml[offset])); - return offset + 1; - -# Given data offset, find word const offset -def aml_data_word_const(offset): - #0x08 NameOp NameString DataRef - if (aml[offset] != 0x0B): - die( "Name offset 0x%x: expected 0x0B actual 0x%x" % - (offset, aml[offset])); - return offset + 1; - -# Given data offset, find byte const offset -def aml_data_byte_const(offset): - #0x08 NameOp NameString DataRef - if (aml[offset] != 0x0A): - die( "Name offset 0x%x: expected 0x0A actual 0x%x" % - (offset, aml[offset])); - return offset + 1; - -# Find name'd buffer -def aml_name_buffer(offset, length): - return aml_data_buffer(aml_name_string(offset) + 4, length) - -# Given name offset, find dword const offset -def aml_name_dword_const(offset): - return aml_data_dword_const(aml_name_string(offset) + 4) - -# Given name offset, find word const offset -def aml_name_word_const(offset): - return aml_data_word_const(aml_name_string(offset) + 4) - -# Given name offset, find byte const offset -def aml_name_byte_const(offset): - return aml_data_byte_const(aml_name_string(offset) + 4) - -def aml_device_start(offset): - #0x5B 0x82 DeviceOp PkgLength NameString - if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x82)): - die( "Name offset 0x%x: expected 0x5B 0x82 actual 0x%x 0x%x" % - (offset, aml[offset], aml[offset + 1])); - return offset - -def aml_device_string(offset): - #0x5B 0x82 DeviceOp PkgLength NameString - start = aml_device_start(offset) - offset += 2 - pkglenbytes = aml_pkglen_bytes(offset) - offset += pkglenbytes - return offset - -def aml_device_end(offset): - start = aml_device_start(offset) - offset += 2 - pkglenbytes = aml_pkglen_bytes(offset) - pkglen = aml_pkglen(offset) - return offset + pkglen - -def aml_processor_start(offset): - #0x5B 0x83 ProcessorOp PkgLength NameString ProcID - if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x83)): - die( "Name offset 0x%x: expected 0x5B 0x83 actual 0x%x 0x%x" % - (offset, aml[offset], aml[offset + 1])); - return offset - -def aml_processor_string(offset): - #0x5B 0x83 ProcessorOp PkgLength NameString ProcID - start = aml_processor_start(offset) - offset += 2 - pkglenbytes = aml_pkglen_bytes(offset) - offset += pkglenbytes - return offset - -def aml_processor_end(offset): - start = aml_processor_start(offset) - offset += 2 - pkglenbytes = aml_pkglen_bytes(offset) - pkglen = aml_pkglen(offset) - return offset + pkglen - -def aml_package_start(offset): - offset = aml_name_string(offset) + 4 - # 0x12 PkgLength NumElements PackageElementList - if (aml[offset] != 0x12): - die( "Name offset 0x%x: expected 0x12 actual 0x%x" % - (offset, aml[offset])); - offset += 1 - return offset + aml_pkglen_bytes(offset) + 1 - -lineno = 0 -for line in fileinput.input(): - # Strip trailing newline - line = line.rstrip(); - # line number and debug string to output in case of errors - lineno = lineno + 1 - debug = "input line %d: %s" % (lineno, line) - #ASL listing: space, then line#, then ...., then code - pasl = re.compile('^\s+([0-9]+)(:\s\s|\.\.\.\.)\s*') - m = pasl.search(line) - if (m): - add_asl(lineno, pasl.sub("", line)); - # AML listing: offset in hex, then ...., then code - paml = re.compile('^([0-9A-Fa-f]+)(:\s\s|\.\.\.\.)\s*') - m = paml.search(line) - if (m): - add_aml(m.group(1), paml.sub("", line)) - -# Now go over code -# Track AML offset of a previous non-empty ASL command -prev_aml_offset = -1 -for i in range(len(asl)): - debug = "input line %d: %s" % (asl[i].lineno, asl[i].line) - - l = asl[i].line - - # skip if not an extract directive - a = len(re.findall(r'ACPI_EXTRACT', l)) - if (not a): - # If not empty, store AML offset. Will be used for sanity checks - # IASL seems to put {}. at random places in the listing. - # Ignore any non-words for the purpose of this test. - m = re.search(r'\w+', l) - if (m): - prev_aml_offset = asl[i].aml_offset - continue - - if (a > 1): - die("Expected at most one ACPI_EXTRACT per line, actual %d" % a) - - mext = re.search(r''' - ^\s* # leading whitespace - /\*\s* # start C comment - (ACPI_EXTRACT_\w+) # directive: group(1) - \s+ # whitspace separates directive from array name - (\w+) # array name: group(2) - \s*\*/ # end of C comment - \s*$ # trailing whitespace - ''', l, re.VERBOSE) - if (not mext): - die("Stray ACPI_EXTRACT in input") - - # previous command must have produced some AML, - # otherwise we are in a middle of a block - if (prev_aml_offset == asl[i].aml_offset): - die("ACPI_EXTRACT directive in the middle of a block") - - directive = mext.group(1) - array = mext.group(2) - offset = asl[i].aml_offset - - if (directive == "ACPI_EXTRACT_ALL_CODE"): - if array in output: - die("%s directive used more than once" % directive) - output[array] = aml - continue - if (directive == "ACPI_EXTRACT_NAME_BUFFER8"): - offset = aml_name_buffer(offset, 8) - elif (directive == "ACPI_EXTRACT_NAME_BUFFER16"): - offset = aml_name_buffer(offset, 16) - elif (directive == "ACPI_EXTRACT_NAME_DWORD_CONST"): - offset = aml_name_dword_const(offset) - elif (directive == "ACPI_EXTRACT_NAME_WORD_CONST"): - offset = aml_name_word_const(offset) - elif (directive == "ACPI_EXTRACT_NAME_BYTE_CONST"): - offset = aml_name_byte_const(offset) - elif (directive == "ACPI_EXTRACT_NAME_STRING"): - offset = aml_name_string(offset) - elif (directive == "ACPI_EXTRACT_METHOD_STRING"): - offset = aml_method_string(offset) - elif (directive == "ACPI_EXTRACT_DEVICE_START"): - offset = aml_device_start(offset) - elif (directive == "ACPI_EXTRACT_DEVICE_STRING"): - offset = aml_device_string(offset) - elif (directive == "ACPI_EXTRACT_DEVICE_END"): - offset = aml_device_end(offset) - elif (directive == "ACPI_EXTRACT_PROCESSOR_START"): - offset = aml_processor_start(offset) - elif (directive == "ACPI_EXTRACT_PROCESSOR_STRING"): - offset = aml_processor_string(offset) - elif (directive == "ACPI_EXTRACT_PROCESSOR_END"): - offset = aml_processor_end(offset) - elif (directive == "ACPI_EXTRACT_PKG_START"): - offset = aml_package_start(offset) - else: - die("Unsupported directive %s" % directive) - - if array not in output: - output[array] = [] - output[array].append(offset) - -debug = "at end of file" - -def get_value_type(maxvalue): - #Use type large enough to fit the table - if (maxvalue >= 0x10000): - return "int" - elif (maxvalue >= 0x100): - return "short" - else: - return "char" - -# Pretty print output -for array in output.keys(): - otype = get_value_type(max(output[array])) - odata = [] - for value in output[array]: - odata.append("0x%x" % value) - sys.stdout.write("static unsigned %s %s[] = {\n" % (otype, array)) - sys.stdout.write(",\n".join(odata)) - sys.stdout.write('\n};\n'); diff --git a/scripts/acpi_extract_preprocess.py b/scripts/acpi_extract_preprocess.py deleted file mode 100755 index 69d10d621..000000000 --- a/scripts/acpi_extract_preprocess.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2011 Red Hat, Inc., Michael S. Tsirkin <mst@redhat.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -# Read a preprocessed ASL listing and put each ACPI_EXTRACT -# directive in a comment, to make iasl skip it. -# We also put each directive on a new line, the machinery -# in tools/acpi_extract.py requires this. - -import re; -import sys; -import fileinput; - -def die(diag): - sys.stderr.write("Error: %s\n" % (diag)) - sys.exit(1) - -# Note: () around pattern make split return matched string as part of list -psplit = re.compile(r''' ( - \b # At word boundary - ACPI_EXTRACT_\w+ # directive - \s+ # some whitespace - \w+ # array name - )''', re.VERBOSE); - -lineno = 0 -for line in fileinput.input(): - # line number and debug string to output in case of errors - lineno = lineno + 1 - debug = "input line %d: %s" % (lineno, line.rstrip()) - - s = psplit.split(line); - # The way split works, each odd item is the matching ACPI_EXTRACT directive. - # Put each in a comment, and on a line by itself. - for i in range(len(s)): - if (i % 2): - sys.stdout.write("\n/* %s */\n" % s[i]) - else: - sys.stdout.write(s[i]) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index b0f6e113c..c9554ba64 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -212,6 +212,7 @@ our @typeList = ( qr{${Ident}_t}, qr{${Ident}_handler}, qr{${Ident}_handler_fn}, + qr{target_(?:u)?long}, ); # This can be modified by sub possible. Since it can be empty, be careful @@ -1715,11 +1716,15 @@ sub process { # 1. with a type on the left -- int [] a; # 2. at the beginning of a line for slice initialisers -- [0...10] = 5, # 3. inside a curly brace -- = { [0...10] = 5 } +# 4. after a comma -- [1] = 5, [2] = 6 +# 5. in a macro definition -- #define abc(x) [x] = y while ($line =~ /(.*?\s)\[/g) { my ($where, $prefix) = ($-[1], $1); if ($prefix !~ /$Type\s+$/ && ($where != 0 || $prefix !~ /^.\s+$/) && - $prefix !~ /{\s+$/) { + $prefix !~ /{\s+$/ && + $prefix !~ /\#\s*define[^(]*\([^)]*\)\s+$/ && + $prefix !~ /,\s+$/) { ERROR("space prohibited before open square bracket '['\n" . $herecurr); } } @@ -1890,19 +1895,6 @@ sub process { ERROR("space prohibited after that '$op' $at\n" . $hereptr); } - - # << and >> may either have or not have spaces both sides - } elsif ($op eq '<<' or $op eq '>>' or - $op eq '&' or $op eq '^' or $op eq '|' or - $op eq '+' or $op eq '-' or - $op eq '*' or $op eq '/' or - $op eq '%') - { - if ($ctx =~ /Wx[^WCE]|[^WCE]xW/) { - ERROR("need consistent spacing around '$op' $at\n" . - $hereptr); - } - # A colon needs no spaces before when it is # terminating a case value or a label. } elsif ($opv eq ':C' || $opv eq ':L') { @@ -2511,6 +2503,42 @@ sub process { WARN("use QEMU instead of Qemu or QEmu\n" . $herecurr); } +# Qemu error function tests + + # Find newlines in error messages + my $qemu_error_funcs = qr{error_setg| + error_setg_errno| + error_setg_win32| + error_set| + error_vreport| + error_report}x; + + if ($rawline =~ /\b(?:$qemu_error_funcs)\s*\(\s*\".*\\n/) { + WARN("Error messages should not contain newlines\n" . $herecurr); + } + + # Continue checking for error messages that contains newlines. This + # check handles cases where string literals are spread over multiple lines. + # Example: + # error_report("Error msg line #1" + # "Error msg line #2\n"); + my $quoted_newline_regex = qr{\+\s*\".*\\n.*\"}; + my $continued_str_literal = qr{\+\s*\".*\"}; + + if ($rawline =~ /$quoted_newline_regex/) { + # Backtrack to first line that does not contain only a quoted literal + # and assume that it is the start of the statement. + my $i = $linenr - 2; + + while (($i >= 0) & $rawlines[$i] =~ /$continued_str_literal/) { + $i--; + } + + if ($rawlines[$i] =~ /\b(?:$qemu_error_funcs)\s*\(/) { + WARN("Error messages should not contain newlines\n" . $herecurr); + } + } + # check for non-portable ffs() calls that have portable alternatives in QEMU if ($line =~ /\bffs\(/) { ERROR("use ctz32() instead of ffs()\n" . $herecurr); diff --git a/scripts/clean-includes b/scripts/clean-includes new file mode 100755 index 000000000..72b47f17f --- /dev/null +++ b/scripts/clean-includes @@ -0,0 +1,165 @@ +#!/bin/sh -e +# +# Clean up QEMU #include lines by ensuring that qemu/osdep.h +# is the first include listed in .c files, and no headers provided +# by osdep.h itself are redundantly included in either .c or .h files. +# +# Copyright (c) 2015 Linaro Limited +# +# Authors: +# Peter Maydell <peter.maydell@linaro.org> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +# Usage: +# clean-includes [--git subjectprefix] file ... +# or +# clean-includes [--git subjectprefix] --all +# +# If the --git subjectprefix option is given, then after making +# the changes to the files this script will create a git commit +# with the subject line "subjectprefix: Clean up includes" +# and a boilerplate commit message. +# +# Using --all will cause clean-includes to run on the whole source +# tree (excluding certain directories which are known not to need +# handling). + +# This script requires Coccinelle to be installed. + +# .c files will have the osdep.h included added, and redundant +# includes removed. +# .h files will have redundant includes (including includes of osdep.h) +# removed. +# Other files (including C++ and ObjectiveC) can't be handled by this script. + +# The following one-liner may be handy for finding files to run this on. +# However some caution is required regarding files that might be part +# of the guest agent or standalone tests. + +# for i in `git ls-tree --name-only HEAD` ; do test -f $i && \ +# grep -E '^# *include' $i | head -1 | grep 'osdep.h' ; test $? != 0 && \ +# echo $i ; done + + +GIT=no + +# Extended regular expression defining files to ignore when using --all +XDIRREGEX='^(tests/tcg|tests/multiboot|pc-bios|disas/libvixl)' + +if [ $# -ne 0 ] && [ "$1" = "--git" ]; then + if [ $# -eq 1 ]; then + echo "--git option requires an argument" + exit 1 + fi + GITSUBJ="$2" + GIT=yes + shift + shift +fi + +if [ $# -eq 0 ]; then + echo "Usage: clean-includes [--git subjectprefix] [--all | foo.c ...]" + echo "(modifies the files in place)" + exit 1 +fi + +if [ "$1" = "--all" ]; then + # We assume there are no files in the tree with spaces in their name + set -- $(git ls-files '*.[ch]' | grep -E -v "$XDIRREGEX") +fi + +# Annoyingly coccinelle won't read a scriptfile unless its +# name ends '.cocci', so write it out to a tempfile with the +# right kind of name. +COCCIFILE="$(mktemp --suffix=.cocci)" + +trap 'rm -f -- "$COCCIFILE"' INT TERM HUP EXIT + +cat >"$COCCIFILE" <<EOT +@@ +@@ + +( ++ #include "qemu/osdep.h" + #include "..." +| ++ #include "qemu/osdep.h" + #include <...> +) +EOT + + +for f in "$@"; do + case "$f" in + *.inc.c) + # These aren't standalone C source files + echo "SKIPPING $f (not a standalone source file)" + continue + ;; + *.c) + MODE=c + ;; + *include/qemu/osdep.h | \ + *include/qemu/compiler.h | \ + *include/standard-headers/ ) + # Removing include lines from osdep.h itself would be counterproductive. + echo "SKIPPING $f (special case header)" + continue + ;; + *include/standard-headers/*) + echo "SKIPPING $f (autogenerated header)" + continue + ;; + *.h) + MODE=h + ;; + *) + echo "WARNING: ignoring $f (cannot handle non-C files)" + continue + ;; + esac + + if [ "$MODE" = "c" ]; then + # First, use Coccinelle to add qemu/osdep.h before the first existing include + # (this will add two lines if the file uses both "..." and <...> #includes, + # but we will remove the extras in the next step) + spatch --in-place --no-show-diff --cocci-file "$COCCIFILE" "$f" + + # Now remove any duplicate osdep.h includes + perl -n -i -e 'print if !/#include "qemu\/osdep.h"/ || !$n++;' "$f" + else + # Remove includes of osdep.h itself + perl -n -i -e 'print if !/\s*#\s*include\s*(["<][^>"]*[">])/ || + ! (grep { $_ eq $1 } qw ("qemu/osdep.h"))' "$f" + fi + + # Remove includes that osdep.h already provides + perl -n -i -e 'print if !/\s*#\s*include\s*(["<][^>"]*[">])/ || + ! (grep { $_ eq $1 } qw ( + "config-host.h" "config-target.h" "qemu/compiler.h" + <setjmp.h> <stdarg.h> <stddef.h> <stdbool.h> <stdint.h> <sys/types.h> + <stdlib.h> <stdio.h> <string.h> <strings.h> <inttypes.h> + <limits.h> <unistd.h> <time.h> <ctype.h> <errno.h> <fcntl.h> + <sys/stat.h> <sys/time.h> <assert.h> <signal.h> + "sysemu/os-posix.h, sysemu/os-win32.h "glib-compat.h" + "qemu/typedefs.h" + ))' "$f" + +done + +if [ "$GIT" = "yes" ]; then + git add -- "$@" + git commit --signoff -F - <<EOF +$GITSUBJ: Clean up includes + +Clean up includes so that osdep.h is included first and headers +which it implies are not included manually. + +This commit was created with scripts/clean-includes. + +EOF + +fi diff --git a/scripts/coverity-model.c b/scripts/coverity-model.c index 617f67d71..ee5bf9d07 100644 --- a/scripts/coverity-model.c +++ b/scripts/coverity-model.c @@ -236,6 +236,23 @@ void *g_try_realloc(void *ptr, size_t size) return g_try_realloc_n(ptr, 1, size); } +/* Other memory allocation functions */ + +void *g_memdup(const void *ptr, unsigned size) +{ + unsigned char *dup; + unsigned i; + + if (!ptr) { + return NULL; + } + + dup = g_malloc(size); + for (i = 0; i < size; i++) + dup[i] = ((unsigned char *)ptr)[i]; + return dup; +} + /* * GLib string allocation functions */ @@ -325,6 +342,15 @@ char *g_strconcat(const char *s, ...) /* Other glib functions */ +typedef struct pollfd GPollFD; + +int poll(); + +int g_poll (GPollFD *fds, unsigned nfds, int timeout) +{ + return poll(fds, nfds, timeout); +} + typedef struct _GIOChannel GIOChannel; GIOChannel *g_io_channel_unix_new(int fd) { diff --git a/scripts/create_config b/scripts/create_config index 546f88914..9cb176f1b 100755 --- a/scripts/create_config +++ b/scripts/create_config @@ -61,6 +61,15 @@ case $line in value=${line#*=} echo "#define $name $value" ;; + HAVE_*=y) # configuration + name=${line%=*} + echo "#define $name 1" + ;; + HAVE_*=*) # configuration + name=${line%=*} + value=${line#*=} + echo "#define $name $value" + ;; ARCH=*) # configuration arch=${line#*=} arch_name=`echo $arch | LC_ALL=C tr '[a-z]' '[A-Z]'` diff --git a/scripts/dump-guest-memory.py b/scripts/dump-guest-memory.py index 08796fff8..c0a2e99f4 100644 --- a/scripts/dump-guest-memory.py +++ b/scripts/dump-guest-memory.py @@ -1,39 +1,456 @@ -# This python script adds a new gdb command, "dump-guest-memory". It -# should be loaded with "source dump-guest-memory.py" at the (gdb) -# prompt. -# -# Copyright (C) 2013, Red Hat, Inc. -# -# Authors: -# Laszlo Ersek <lersek@redhat.com> -# -# This work is licensed under the terms of the GNU GPL, version 2 or later. See -# the COPYING file in the top-level directory. -# +""" +This python script adds a new gdb command, "dump-guest-memory". It +should be loaded with "source dump-guest-memory.py" at the (gdb) +prompt. + +Copyright (C) 2013, Red Hat, Inc. + +Authors: + Laszlo Ersek <lersek@redhat.com> + Janosch Frank <frankja@linux.vnet.ibm.com> + +This work is licensed under the terms of the GNU GPL, version 2 or later. See +the COPYING file in the top-level directory. +""" + +import ctypes + +UINTPTR_T = gdb.lookup_type("uintptr_t") + +TARGET_PAGE_SIZE = 0x1000 +TARGET_PAGE_MASK = 0xFFFFFFFFFFFFF000 + +# Special value for e_phnum. This indicates that the real number of +# program headers is too large to fit into e_phnum. Instead the real +# value is in the field sh_info of section 0. +PN_XNUM = 0xFFFF + +EV_CURRENT = 1 + +ELFCLASS32 = 1 +ELFCLASS64 = 2 + +ELFDATA2LSB = 1 +ELFDATA2MSB = 2 + +ET_CORE = 4 + +PT_LOAD = 1 +PT_NOTE = 4 + +EM_386 = 3 +EM_PPC = 20 +EM_PPC64 = 21 +EM_S390 = 22 +EM_AARCH = 183 +EM_X86_64 = 62 + +class ELF(object): + """Representation of a ELF file.""" + + def __init__(self, arch): + self.ehdr = None + self.notes = [] + self.segments = [] + self.notes_size = 0 + self.endianess = None + self.elfclass = ELFCLASS64 + + if arch == 'aarch64-le': + self.endianess = ELFDATA2LSB + self.elfclass = ELFCLASS64 + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_AARCH + + elif arch == 'aarch64-be': + self.endianess = ELFDATA2MSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_AARCH + + elif arch == 'X86_64': + self.endianess = ELFDATA2LSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_X86_64 + + elif arch == '386': + self.endianess = ELFDATA2LSB + self.elfclass = ELFCLASS32 + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_386 + + elif arch == 's390': + self.endianess = ELFDATA2MSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_S390 + + elif arch == 'ppc64-le': + self.endianess = ELFDATA2LSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_PPC64 + + elif arch == 'ppc64-be': + self.endianess = ELFDATA2MSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_PPC64 + + else: + raise gdb.GdbError("No valid arch type specified.\n" + "Currently supported types:\n" + "aarch64-be, aarch64-le, X86_64, 386, s390, " + "ppc64-be, ppc64-le") + + self.add_segment(PT_NOTE, 0, 0) + + def add_note(self, n_name, n_desc, n_type): + """Adds a note to the ELF.""" + + note = get_arch_note(self.endianess, len(n_name), len(n_desc)) + note.n_namesz = len(n_name) + 1 + note.n_descsz = len(n_desc) + note.n_name = n_name.encode() + note.n_type = n_type + + # Desc needs to be 4 byte aligned (although the 64bit spec + # specifies 8 byte). When defining n_desc as uint32 it will be + # automatically aligned but we need the memmove to copy the + # string into it. + ctypes.memmove(note.n_desc, n_desc.encode(), len(n_desc)) + + self.notes.append(note) + self.segments[0].p_filesz += ctypes.sizeof(note) + self.segments[0].p_memsz += ctypes.sizeof(note) + + def add_segment(self, p_type, p_paddr, p_size): + """Adds a segment to the elf.""" + + phdr = get_arch_phdr(self.endianess, self.elfclass) + phdr.p_type = p_type + phdr.p_paddr = p_paddr + phdr.p_filesz = p_size + phdr.p_memsz = p_size + self.segments.append(phdr) + self.ehdr.e_phnum += 1 + + def to_file(self, elf_file): + """Writes all ELF structures to the the passed file. + + Structure: + Ehdr + Segment 0:PT_NOTE + Segment 1:PT_LOAD + Segment N:PT_LOAD + Note 0..N + Dump contents + """ + elf_file.write(self.ehdr) + off = ctypes.sizeof(self.ehdr) + \ + len(self.segments) * ctypes.sizeof(self.segments[0]) + + for phdr in self.segments: + phdr.p_offset = off + elf_file.write(phdr) + off += phdr.p_filesz + + for note in self.notes: + elf_file.write(note) + + +def get_arch_note(endianess, len_name, len_desc): + """Returns a Note class with the specified endianess.""" + + if endianess == ELFDATA2LSB: + superclass = ctypes.LittleEndianStructure + else: + superclass = ctypes.BigEndianStructure + + len_name = len_name + 1 + + class Note(superclass): + """Represents an ELF note, includes the content.""" + + _fields_ = [("n_namesz", ctypes.c_uint32), + ("n_descsz", ctypes.c_uint32), + ("n_type", ctypes.c_uint32), + ("n_name", ctypes.c_char * len_name), + ("n_desc", ctypes.c_uint32 * ((len_desc + 3) // 4))] + return Note() + + +class Ident(ctypes.Structure): + """Represents the ELF ident array in the ehdr structure.""" + + _fields_ = [('ei_mag0', ctypes.c_ubyte), + ('ei_mag1', ctypes.c_ubyte), + ('ei_mag2', ctypes.c_ubyte), + ('ei_mag3', ctypes.c_ubyte), + ('ei_class', ctypes.c_ubyte), + ('ei_data', ctypes.c_ubyte), + ('ei_version', ctypes.c_ubyte), + ('ei_osabi', ctypes.c_ubyte), + ('ei_abiversion', ctypes.c_ubyte), + ('ei_pad', ctypes.c_ubyte * 7)] + + def __init__(self, endianess, elfclass): + self.ei_mag0 = 0x7F + self.ei_mag1 = ord('E') + self.ei_mag2 = ord('L') + self.ei_mag3 = ord('F') + self.ei_class = elfclass + self.ei_data = endianess + self.ei_version = EV_CURRENT + + +def get_arch_ehdr(endianess, elfclass): + """Returns a EHDR64 class with the specified endianess.""" + + if endianess == ELFDATA2LSB: + superclass = ctypes.LittleEndianStructure + else: + superclass = ctypes.BigEndianStructure + + class EHDR64(superclass): + """Represents the 64 bit ELF header struct.""" + + _fields_ = [('e_ident', Ident), + ('e_type', ctypes.c_uint16), + ('e_machine', ctypes.c_uint16), + ('e_version', ctypes.c_uint32), + ('e_entry', ctypes.c_uint64), + ('e_phoff', ctypes.c_uint64), + ('e_shoff', ctypes.c_uint64), + ('e_flags', ctypes.c_uint32), + ('e_ehsize', ctypes.c_uint16), + ('e_phentsize', ctypes.c_uint16), + ('e_phnum', ctypes.c_uint16), + ('e_shentsize', ctypes.c_uint16), + ('e_shnum', ctypes.c_uint16), + ('e_shstrndx', ctypes.c_uint16)] + + def __init__(self): + super(superclass, self).__init__() + self.e_ident = Ident(endianess, elfclass) + self.e_type = ET_CORE + self.e_version = EV_CURRENT + self.e_ehsize = ctypes.sizeof(self) + self.e_phoff = ctypes.sizeof(self) + self.e_phentsize = ctypes.sizeof(get_arch_phdr(endianess, elfclass)) + self.e_phnum = 0 + + + class EHDR32(superclass): + """Represents the 32 bit ELF header struct.""" + + _fields_ = [('e_ident', Ident), + ('e_type', ctypes.c_uint16), + ('e_machine', ctypes.c_uint16), + ('e_version', ctypes.c_uint32), + ('e_entry', ctypes.c_uint32), + ('e_phoff', ctypes.c_uint32), + ('e_shoff', ctypes.c_uint32), + ('e_flags', ctypes.c_uint32), + ('e_ehsize', ctypes.c_uint16), + ('e_phentsize', ctypes.c_uint16), + ('e_phnum', ctypes.c_uint16), + ('e_shentsize', ctypes.c_uint16), + ('e_shnum', ctypes.c_uint16), + ('e_shstrndx', ctypes.c_uint16)] + + def __init__(self): + super(superclass, self).__init__() + self.e_ident = Ident(endianess, elfclass) + self.e_type = ET_CORE + self.e_version = EV_CURRENT + self.e_ehsize = ctypes.sizeof(self) + self.e_phoff = ctypes.sizeof(self) + self.e_phentsize = ctypes.sizeof(get_arch_phdr(endianess, elfclass)) + self.e_phnum = 0 + + # End get_arch_ehdr + if elfclass == ELFCLASS64: + return EHDR64() + else: + return EHDR32() + + +def get_arch_phdr(endianess, elfclass): + """Returns a 32 or 64 bit PHDR class with the specified endianess.""" + + if endianess == ELFDATA2LSB: + superclass = ctypes.LittleEndianStructure + else: + superclass = ctypes.BigEndianStructure + + class PHDR64(superclass): + """Represents the 64 bit ELF program header struct.""" + + _fields_ = [('p_type', ctypes.c_uint32), + ('p_flags', ctypes.c_uint32), + ('p_offset', ctypes.c_uint64), + ('p_vaddr', ctypes.c_uint64), + ('p_paddr', ctypes.c_uint64), + ('p_filesz', ctypes.c_uint64), + ('p_memsz', ctypes.c_uint64), + ('p_align', ctypes.c_uint64)] + + class PHDR32(superclass): + """Represents the 32 bit ELF program header struct.""" + + _fields_ = [('p_type', ctypes.c_uint32), + ('p_offset', ctypes.c_uint32), + ('p_vaddr', ctypes.c_uint32), + ('p_paddr', ctypes.c_uint32), + ('p_filesz', ctypes.c_uint32), + ('p_memsz', ctypes.c_uint32), + ('p_flags', ctypes.c_uint32), + ('p_align', ctypes.c_uint32)] + + # End get_arch_phdr + if elfclass == ELFCLASS64: + return PHDR64() + else: + return PHDR32() + + +def int128_get64(val): + """Returns low 64bit part of Int128 struct.""" + + assert val["hi"] == 0 + return val["lo"] + + +def qlist_foreach(head, field_str): + """Generator for qlists.""" + + var_p = head["lh_first"] + while var_p != 0: + var = var_p.dereference() + var_p = var[field_str]["le_next"] + yield var + + +def qemu_get_ram_block(ram_addr): + """Returns the RAMBlock struct to which the given address belongs.""" + + ram_blocks = gdb.parse_and_eval("ram_list.blocks") + + for block in qlist_foreach(ram_blocks, "next"): + if (ram_addr - block["offset"]) < block["used_length"]: + return block + + raise gdb.GdbError("Bad ram offset %x" % ram_addr) + + +def qemu_get_ram_ptr(ram_addr): + """Returns qemu vaddr for given guest physical address.""" + + block = qemu_get_ram_block(ram_addr) + return block["host"] + (ram_addr - block["offset"]) + + +def memory_region_get_ram_ptr(memory_region): + if memory_region["alias"] != 0: + return (memory_region_get_ram_ptr(memory_region["alias"].dereference()) + + memory_region["alias_offset"]) + + return qemu_get_ram_ptr(memory_region["ram_block"]["offset"]) + + +def get_guest_phys_blocks(): + """Returns a list of ram blocks. + + Each block entry contains: + 'target_start': guest block phys start address + 'target_end': guest block phys end address + 'host_addr': qemu vaddr of the block's start + """ + + guest_phys_blocks = [] + + print("guest RAM blocks:") + print("target_start target_end host_addr message " + "count") + print("---------------- ---------------- ---------------- ------- " + "-----") + + current_map_p = gdb.parse_and_eval("address_space_memory.current_map") + current_map = current_map_p.dereference() + + # Conversion to int is needed for python 3 + # compatibility. Otherwise range doesn't cast the value itself and + # breaks. + for cur in range(int(current_map["nr"])): + flat_range = (current_map["ranges"] + cur).dereference() + memory_region = flat_range["mr"].dereference() + + # we only care about RAM + if not memory_region["ram"]: + continue + + section_size = int128_get64(flat_range["addr"]["size"]) + target_start = int128_get64(flat_range["addr"]["start"]) + target_end = target_start + section_size + host_addr = (memory_region_get_ram_ptr(memory_region) + + flat_range["offset_in_region"]) + predecessor = None + + # find continuity in guest physical address space + if len(guest_phys_blocks) > 0: + predecessor = guest_phys_blocks[-1] + predecessor_size = (predecessor["target_end"] - + predecessor["target_start"]) + + # the memory API guarantees monotonically increasing + # traversal + assert predecessor["target_end"] <= target_start + + # we want continuity in both guest-physical and + # host-virtual memory + if (predecessor["target_end"] < target_start or + predecessor["host_addr"] + predecessor_size != host_addr): + predecessor = None + + if predecessor is None: + # isolated mapping, add it to the list + guest_phys_blocks.append({"target_start": target_start, + "target_end": target_end, + "host_addr": host_addr}) + message = "added" + else: + # expand predecessor until @target_end; predecessor's + # start doesn't change + predecessor["target_end"] = target_end + message = "joined" + + print("%016x %016x %016x %-7s %5u" % + (target_start, target_end, host_addr.cast(UINTPTR_T), + message, len(guest_phys_blocks))) + + return guest_phys_blocks + + # The leading docstring doesn't have idiomatic Python formatting. It is # printed by gdb's "help" command (the first line is printed in the # "help data" summary), and it should match how other help texts look in # gdb. - -import struct - class DumpGuestMemory(gdb.Command): """Extract guest vmcore from qemu process coredump. -The sole argument is FILE, identifying the target file to write the -guest vmcore to. +The two required arguments are FILE and ARCH: +FILE identifies the target file to write the guest vmcore to. +ARCH specifies the architecture for which the core will be generated. This GDB command reimplements the dump-guest-memory QMP command in python, using the representation of guest memory as captured in the qemu coredump. The qemu process that has been dumped must have had the -command line option "-machine dump-guest-core=on". +command line option "-machine dump-guest-core=on" which is the default. For simplicity, the "paging", "begin" and "end" parameters of the QMP command are not supported -- no attempt is made to get the guest's internal paging structures (ie. paging=false is hard-wired), and guest memory is always fully dumped. -Only x86_64 guests are supported. +Currently aarch64-be, aarch64-le, X86_64, 386, s390, ppc64-be, +ppc64-le guests are supported. The CORE/NT_PRSTATUS and QEMU notes (that is, the VCPUs' statuses) are not written to the vmcore. Preparing these would require context that is @@ -47,293 +464,66 @@ deliberately called abort(), or it was dumped in response to a signal at a halfway fortunate point, then its coredump should be in reasonable shape and this command should mostly work.""" - TARGET_PAGE_SIZE = 0x1000 - TARGET_PAGE_MASK = 0xFFFFFFFFFFFFF000 - - # Various ELF constants - EM_X86_64 = 62 # AMD x86-64 target machine - ELFDATA2LSB = 1 # little endian - ELFCLASS64 = 2 - ELFMAG = "\x7FELF" - EV_CURRENT = 1 - ET_CORE = 4 - PT_LOAD = 1 - PT_NOTE = 4 - - # Special value for e_phnum. This indicates that the real number of - # program headers is too large to fit into e_phnum. Instead the real - # value is in the field sh_info of section 0. - PN_XNUM = 0xFFFF - - # Format strings for packing and header size calculation. - ELF64_EHDR = ("4s" # e_ident/magic - "B" # e_ident/class - "B" # e_ident/data - "B" # e_ident/version - "B" # e_ident/osabi - "8s" # e_ident/pad - "H" # e_type - "H" # e_machine - "I" # e_version - "Q" # e_entry - "Q" # e_phoff - "Q" # e_shoff - "I" # e_flags - "H" # e_ehsize - "H" # e_phentsize - "H" # e_phnum - "H" # e_shentsize - "H" # e_shnum - "H" # e_shstrndx - ) - ELF64_PHDR = ("I" # p_type - "I" # p_flags - "Q" # p_offset - "Q" # p_vaddr - "Q" # p_paddr - "Q" # p_filesz - "Q" # p_memsz - "Q" # p_align - ) - def __init__(self): super(DumpGuestMemory, self).__init__("dump-guest-memory", gdb.COMMAND_DATA, gdb.COMPLETE_FILENAME) - self.uintptr_t = gdb.lookup_type("uintptr_t") - self.elf64_ehdr_le = struct.Struct("<%s" % self.ELF64_EHDR) - self.elf64_phdr_le = struct.Struct("<%s" % self.ELF64_PHDR) - - def int128_get64(self, val): - assert (val["hi"] == 0) - return val["lo"] - - def qlist_foreach(self, head, field_str): - var_p = head["lh_first"] - while (var_p != 0): - var = var_p.dereference() - yield var - var_p = var[field_str]["le_next"] - - def qemu_get_ram_block(self, ram_addr): - ram_blocks = gdb.parse_and_eval("ram_list.blocks") - for block in self.qlist_foreach(ram_blocks, "next"): - if (ram_addr - block["offset"] < block["used_length"]): - return block - raise gdb.GdbError("Bad ram offset %x" % ram_addr) - - def qemu_get_ram_ptr(self, ram_addr): - block = self.qemu_get_ram_block(ram_addr) - return block["host"] + (ram_addr - block["offset"]) - - def memory_region_get_ram_ptr(self, mr): - if (mr["alias"] != 0): - return (self.memory_region_get_ram_ptr(mr["alias"].dereference()) + - mr["alias_offset"]) - return self.qemu_get_ram_ptr(mr["ram_addr"] & self.TARGET_PAGE_MASK) - - def guest_phys_blocks_init(self): - self.guest_phys_blocks = [] - - def guest_phys_blocks_append(self): - print "guest RAM blocks:" - print ("target_start target_end host_addr message " - "count") - print ("---------------- ---------------- ---------------- ------- " - "-----") - - current_map_p = gdb.parse_and_eval("address_space_memory.current_map") - current_map = current_map_p.dereference() - for cur in range(current_map["nr"]): - flat_range = (current_map["ranges"] + cur).dereference() - mr = flat_range["mr"].dereference() - - # we only care about RAM - if (not mr["ram"]): - continue - - section_size = self.int128_get64(flat_range["addr"]["size"]) - target_start = self.int128_get64(flat_range["addr"]["start"]) - target_end = target_start + section_size - host_addr = (self.memory_region_get_ram_ptr(mr) + - flat_range["offset_in_region"]) - predecessor = None - - # find continuity in guest physical address space - if (len(self.guest_phys_blocks) > 0): - predecessor = self.guest_phys_blocks[-1] - predecessor_size = (predecessor["target_end"] - - predecessor["target_start"]) - - # the memory API guarantees monotonically increasing - # traversal - assert (predecessor["target_end"] <= target_start) - - # we want continuity in both guest-physical and - # host-virtual memory - if (predecessor["target_end"] < target_start or - predecessor["host_addr"] + predecessor_size != host_addr): - predecessor = None - - if (predecessor is None): - # isolated mapping, add it to the list - self.guest_phys_blocks.append({"target_start": target_start, - "target_end" : target_end, - "host_addr" : host_addr}) - message = "added" - else: - # expand predecessor until @target_end; predecessor's - # start doesn't change - predecessor["target_end"] = target_end - message = "joined" - - print ("%016x %016x %016x %-7s %5u" % - (target_start, target_end, host_addr.cast(self.uintptr_t), - message, len(self.guest_phys_blocks))) - - def cpu_get_dump_info(self): - # We can't synchronize the registers with KVM post-mortem, and - # the bits in (first_x86_cpu->env.hflags) seem to be stale; they - # may not reflect long mode for example. Hence just assume the - # most common values. This also means that instruction pointer - # etc. will be bogus in the dump, but at least the RAM contents - # should be valid. - self.dump_info = {"d_machine": self.EM_X86_64, - "d_endian" : self.ELFDATA2LSB, - "d_class" : self.ELFCLASS64} - - def encode_elf64_ehdr_le(self): - return self.elf64_ehdr_le.pack( - self.ELFMAG, # e_ident/magic - self.dump_info["d_class"], # e_ident/class - self.dump_info["d_endian"], # e_ident/data - self.EV_CURRENT, # e_ident/version - 0, # e_ident/osabi - "", # e_ident/pad - self.ET_CORE, # e_type - self.dump_info["d_machine"], # e_machine - self.EV_CURRENT, # e_version - 0, # e_entry - self.elf64_ehdr_le.size, # e_phoff - 0, # e_shoff - 0, # e_flags - self.elf64_ehdr_le.size, # e_ehsize - self.elf64_phdr_le.size, # e_phentsize - self.phdr_num, # e_phnum - 0, # e_shentsize - 0, # e_shnum - 0 # e_shstrndx - ) - - def encode_elf64_note_le(self): - return self.elf64_phdr_le.pack(self.PT_NOTE, # p_type - 0, # p_flags - (self.memory_offset - - len(self.note)), # p_offset - 0, # p_vaddr - 0, # p_paddr - len(self.note), # p_filesz - len(self.note), # p_memsz - 0 # p_align - ) - - def encode_elf64_load_le(self, offset, start_hwaddr, range_size): - return self.elf64_phdr_le.pack(self.PT_LOAD, # p_type - 0, # p_flags - offset, # p_offset - 0, # p_vaddr - start_hwaddr, # p_paddr - range_size, # p_filesz - range_size, # p_memsz - 0 # p_align - ) - - def note_init(self, name, desc, type): - # name must include a trailing NUL - namesz = (len(name) + 1 + 3) / 4 * 4 - descsz = (len(desc) + 3) / 4 * 4 - fmt = ("<" # little endian - "I" # n_namesz - "I" # n_descsz - "I" # n_type - "%us" # name - "%us" # desc - % (namesz, descsz)) - self.note = struct.pack(fmt, - len(name) + 1, len(desc), type, name, desc) - - def dump_init(self): - self.guest_phys_blocks_init() - self.guest_phys_blocks_append() - self.cpu_get_dump_info() - # we have no way to retrieve the VCPU status from KVM - # post-mortem - self.note_init("NONE", "EMPTY", 0) - - # Account for PT_NOTE. - self.phdr_num = 1 - - # We should never reach PN_XNUM for paging=false dumps: there's - # just a handful of discontiguous ranges after merging. - self.phdr_num += len(self.guest_phys_blocks) - assert (self.phdr_num < self.PN_XNUM) - - # Calculate the ELF file offset where the memory dump commences: - # - # ELF header - # PT_NOTE - # PT_LOAD: 1 - # PT_LOAD: 2 - # ... - # PT_LOAD: len(self.guest_phys_blocks) - # ELF note - # memory dump - self.memory_offset = (self.elf64_ehdr_le.size + - self.elf64_phdr_le.size * self.phdr_num + - len(self.note)) - - def dump_begin(self, vmcore): - vmcore.write(self.encode_elf64_ehdr_le()) - vmcore.write(self.encode_elf64_note_le()) - running = self.memory_offset + self.elf = None + self.guest_phys_blocks = None + + def dump_init(self, vmcore): + """Prepares and writes ELF structures to core file.""" + + # Needed to make crash happy, data for more useful notes is + # not available in a qemu core. + self.elf.add_note("NONE", "EMPTY", 0) + + # We should never reach PN_XNUM for paging=false dumps, + # there's just a handful of discontiguous ranges after + # merging. + # The constant is needed to account for the PT_NOTE segment. + phdr_num = len(self.guest_phys_blocks) + 1 + assert phdr_num < PN_XNUM + for block in self.guest_phys_blocks: - range_size = block["target_end"] - block["target_start"] - vmcore.write(self.encode_elf64_load_le(running, - block["target_start"], - range_size)) - running += range_size - vmcore.write(self.note) + block_size = block["target_end"] - block["target_start"] + self.elf.add_segment(PT_LOAD, block["target_start"], block_size) + + self.elf.to_file(vmcore) def dump_iterate(self, vmcore): + """Writes guest core to file.""" + qemu_core = gdb.inferiors()[0] for block in self.guest_phys_blocks: - cur = block["host_addr"] + cur = block["host_addr"] left = block["target_end"] - block["target_start"] - print ("dumping range at %016x for length %016x" % - (cur.cast(self.uintptr_t), left)) - while (left > 0): - chunk_size = min(self.TARGET_PAGE_SIZE, left) + print("dumping range at %016x for length %016x" % + (cur.cast(UINTPTR_T), left)) + + while left > 0: + chunk_size = min(TARGET_PAGE_SIZE, left) chunk = qemu_core.read_memory(cur, chunk_size) vmcore.write(chunk) - cur += chunk_size + cur += chunk_size left -= chunk_size - def create_vmcore(self, filename): - vmcore = open(filename, "wb") - self.dump_begin(vmcore) - self.dump_iterate(vmcore) - vmcore.close() - def invoke(self, args, from_tty): + """Handles command invocation from gdb.""" + # Unwittingly pressing the Enter key after the command should # not dump the same multi-gig coredump to the same file. self.dont_repeat() argv = gdb.string_to_argv(args) - if (len(argv) != 1): - raise gdb.GdbError("usage: dump-guest-memory FILE") + if len(argv) != 2: + raise gdb.GdbError("usage: dump-guest-memory FILE ARCH") + + self.elf = ELF(argv[1]) + self.guest_phys_blocks = get_guest_phys_blocks() - self.dump_init() - self.create_vmcore(argv[0]) + with open(argv[0], "wb") as vmcore: + self.dump_init(vmcore) + self.dump_iterate(vmcore) DumpGuestMemory() diff --git a/scripts/feature_to_c.sh b/scripts/feature_to_c.sh index 888548e58..fb1f3363f 100644 --- a/scripts/feature_to_c.sh +++ b/scripts/feature_to_c.sh @@ -36,7 +36,7 @@ for input; do arrayname=xml_feature_`echo $input | sed 's,.*/,,; s/[-.]/_/g'` ${AWK:-awk} 'BEGIN { n = 0 - printf "#include \"config.h\"\n" + printf "#include \"qemu/osdep.h\"\n" printf "#include \"qemu-common.h\"\n" printf "#include \"exec/gdbstub.h\"\n" print "static const char '$arrayname'[] = {" diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl index 7dacf32f4..8261bcb1a 100755 --- a/scripts/get_maintainer.pl +++ b/scripts/get_maintainer.pl @@ -636,7 +636,7 @@ sub get_maintainers { if ($email) { if (! $interactive) { - $email_git_fallback = 0 if @email_to > 0 || @list_to > 0 || $email_git || $email_git_blame; + $email_git_fallback = 0 if @email_to > 0 || $email_git || $email_git_blame; if ($email_git_fallback) { print STDERR "get_maintainer.pl: No maintainers found, printing recent contributors.\n"; print STDERR "get_maintainer.pl: Do not blindly cc: them on patches! Use common sense.\n"; diff --git a/scripts/kvm/kvm_stat b/scripts/kvm/kvm_stat index 7e5d25612..769d884b6 100755 --- a/scripts/kvm/kvm_stat +++ b/scripts/kvm/kvm_stat @@ -12,285 +12,312 @@ # the COPYING file in the top-level directory. import curses -import sys, os, time, optparse, ctypes -from ctypes import * - -class DebugfsProvider(object): - def __init__(self): - self.base = '/sys/kernel/debug/kvm' - self._fields = os.listdir(self.base) - def fields(self): - return self._fields - def select(self, fields): - self._fields = fields - def read(self): - def val(key): - return int(file(self.base + '/' + key).read()) - return dict([(key, val(key)) for key in self._fields]) - -vmx_exit_reasons = { - 0: 'EXCEPTION_NMI', - 1: 'EXTERNAL_INTERRUPT', - 2: 'TRIPLE_FAULT', - 7: 'PENDING_INTERRUPT', - 8: 'NMI_WINDOW', - 9: 'TASK_SWITCH', - 10: 'CPUID', - 12: 'HLT', - 14: 'INVLPG', - 15: 'RDPMC', - 16: 'RDTSC', - 18: 'VMCALL', - 19: 'VMCLEAR', - 20: 'VMLAUNCH', - 21: 'VMPTRLD', - 22: 'VMPTRST', - 23: 'VMREAD', - 24: 'VMRESUME', - 25: 'VMWRITE', - 26: 'VMOFF', - 27: 'VMON', - 28: 'CR_ACCESS', - 29: 'DR_ACCESS', - 30: 'IO_INSTRUCTION', - 31: 'MSR_READ', - 32: 'MSR_WRITE', - 33: 'INVALID_STATE', - 36: 'MWAIT_INSTRUCTION', - 39: 'MONITOR_INSTRUCTION', - 40: 'PAUSE_INSTRUCTION', - 41: 'MCE_DURING_VMENTRY', - 43: 'TPR_BELOW_THRESHOLD', - 44: 'APIC_ACCESS', - 48: 'EPT_VIOLATION', - 49: 'EPT_MISCONFIG', - 54: 'WBINVD', - 55: 'XSETBV', - 56: 'APIC_WRITE', - 58: 'INVPCID', +import sys +import os +import time +import optparse +import ctypes +import fcntl +import resource +import struct +import re +from collections import defaultdict +from time import sleep + +VMX_EXIT_REASONS = { + 'EXCEPTION_NMI': 0, + 'EXTERNAL_INTERRUPT': 1, + 'TRIPLE_FAULT': 2, + 'PENDING_INTERRUPT': 7, + 'NMI_WINDOW': 8, + 'TASK_SWITCH': 9, + 'CPUID': 10, + 'HLT': 12, + 'INVLPG': 14, + 'RDPMC': 15, + 'RDTSC': 16, + 'VMCALL': 18, + 'VMCLEAR': 19, + 'VMLAUNCH': 20, + 'VMPTRLD': 21, + 'VMPTRST': 22, + 'VMREAD': 23, + 'VMRESUME': 24, + 'VMWRITE': 25, + 'VMOFF': 26, + 'VMON': 27, + 'CR_ACCESS': 28, + 'DR_ACCESS': 29, + 'IO_INSTRUCTION': 30, + 'MSR_READ': 31, + 'MSR_WRITE': 32, + 'INVALID_STATE': 33, + 'MWAIT_INSTRUCTION': 36, + 'MONITOR_INSTRUCTION': 39, + 'PAUSE_INSTRUCTION': 40, + 'MCE_DURING_VMENTRY': 41, + 'TPR_BELOW_THRESHOLD': 43, + 'APIC_ACCESS': 44, + 'EPT_VIOLATION': 48, + 'EPT_MISCONFIG': 49, + 'WBINVD': 54, + 'XSETBV': 55, + 'APIC_WRITE': 56, + 'INVPCID': 58, } -svm_exit_reasons = { - 0x000: 'READ_CR0', - 0x003: 'READ_CR3', - 0x004: 'READ_CR4', - 0x008: 'READ_CR8', - 0x010: 'WRITE_CR0', - 0x013: 'WRITE_CR3', - 0x014: 'WRITE_CR4', - 0x018: 'WRITE_CR8', - 0x020: 'READ_DR0', - 0x021: 'READ_DR1', - 0x022: 'READ_DR2', - 0x023: 'READ_DR3', - 0x024: 'READ_DR4', - 0x025: 'READ_DR5', - 0x026: 'READ_DR6', - 0x027: 'READ_DR7', - 0x030: 'WRITE_DR0', - 0x031: 'WRITE_DR1', - 0x032: 'WRITE_DR2', - 0x033: 'WRITE_DR3', - 0x034: 'WRITE_DR4', - 0x035: 'WRITE_DR5', - 0x036: 'WRITE_DR6', - 0x037: 'WRITE_DR7', - 0x040: 'EXCP_BASE', - 0x060: 'INTR', - 0x061: 'NMI', - 0x062: 'SMI', - 0x063: 'INIT', - 0x064: 'VINTR', - 0x065: 'CR0_SEL_WRITE', - 0x066: 'IDTR_READ', - 0x067: 'GDTR_READ', - 0x068: 'LDTR_READ', - 0x069: 'TR_READ', - 0x06a: 'IDTR_WRITE', - 0x06b: 'GDTR_WRITE', - 0x06c: 'LDTR_WRITE', - 0x06d: 'TR_WRITE', - 0x06e: 'RDTSC', - 0x06f: 'RDPMC', - 0x070: 'PUSHF', - 0x071: 'POPF', - 0x072: 'CPUID', - 0x073: 'RSM', - 0x074: 'IRET', - 0x075: 'SWINT', - 0x076: 'INVD', - 0x077: 'PAUSE', - 0x078: 'HLT', - 0x079: 'INVLPG', - 0x07a: 'INVLPGA', - 0x07b: 'IOIO', - 0x07c: 'MSR', - 0x07d: 'TASK_SWITCH', - 0x07e: 'FERR_FREEZE', - 0x07f: 'SHUTDOWN', - 0x080: 'VMRUN', - 0x081: 'VMMCALL', - 0x082: 'VMLOAD', - 0x083: 'VMSAVE', - 0x084: 'STGI', - 0x085: 'CLGI', - 0x086: 'SKINIT', - 0x087: 'RDTSCP', - 0x088: 'ICEBP', - 0x089: 'WBINVD', - 0x08a: 'MONITOR', - 0x08b: 'MWAIT', - 0x08c: 'MWAIT_COND', - 0x08d: 'XSETBV', - 0x400: 'NPF', +SVM_EXIT_REASONS = { + 'READ_CR0': 0x000, + 'READ_CR3': 0x003, + 'READ_CR4': 0x004, + 'READ_CR8': 0x008, + 'WRITE_CR0': 0x010, + 'WRITE_CR3': 0x013, + 'WRITE_CR4': 0x014, + 'WRITE_CR8': 0x018, + 'READ_DR0': 0x020, + 'READ_DR1': 0x021, + 'READ_DR2': 0x022, + 'READ_DR3': 0x023, + 'READ_DR4': 0x024, + 'READ_DR5': 0x025, + 'READ_DR6': 0x026, + 'READ_DR7': 0x027, + 'WRITE_DR0': 0x030, + 'WRITE_DR1': 0x031, + 'WRITE_DR2': 0x032, + 'WRITE_DR3': 0x033, + 'WRITE_DR4': 0x034, + 'WRITE_DR5': 0x035, + 'WRITE_DR6': 0x036, + 'WRITE_DR7': 0x037, + 'EXCP_BASE': 0x040, + 'INTR': 0x060, + 'NMI': 0x061, + 'SMI': 0x062, + 'INIT': 0x063, + 'VINTR': 0x064, + 'CR0_SEL_WRITE': 0x065, + 'IDTR_READ': 0x066, + 'GDTR_READ': 0x067, + 'LDTR_READ': 0x068, + 'TR_READ': 0x069, + 'IDTR_WRITE': 0x06a, + 'GDTR_WRITE': 0x06b, + 'LDTR_WRITE': 0x06c, + 'TR_WRITE': 0x06d, + 'RDTSC': 0x06e, + 'RDPMC': 0x06f, + 'PUSHF': 0x070, + 'POPF': 0x071, + 'CPUID': 0x072, + 'RSM': 0x073, + 'IRET': 0x074, + 'SWINT': 0x075, + 'INVD': 0x076, + 'PAUSE': 0x077, + 'HLT': 0x078, + 'INVLPG': 0x079, + 'INVLPGA': 0x07a, + 'IOIO': 0x07b, + 'MSR': 0x07c, + 'TASK_SWITCH': 0x07d, + 'FERR_FREEZE': 0x07e, + 'SHUTDOWN': 0x07f, + 'VMRUN': 0x080, + 'VMMCALL': 0x081, + 'VMLOAD': 0x082, + 'VMSAVE': 0x083, + 'STGI': 0x084, + 'CLGI': 0x085, + 'SKINIT': 0x086, + 'RDTSCP': 0x087, + 'ICEBP': 0x088, + 'WBINVD': 0x089, + 'MONITOR': 0x08a, + 'MWAIT': 0x08b, + 'MWAIT_COND': 0x08c, + 'XSETBV': 0x08d, + 'NPF': 0x400, } # EC definition of HSR (from arch/arm64/include/asm/kvm_arm.h) -aarch64_exit_reasons = { - 0x00: 'UNKNOWN', - 0x01: 'WFI', - 0x03: 'CP15_32', - 0x04: 'CP15_64', - 0x05: 'CP14_MR', - 0x06: 'CP14_LS', - 0x07: 'FP_ASIMD', - 0x08: 'CP10_ID', - 0x0C: 'CP14_64', - 0x0E: 'ILL_ISS', - 0x11: 'SVC32', - 0x12: 'HVC32', - 0x13: 'SMC32', - 0x15: 'SVC64', - 0x16: 'HVC64', - 0x17: 'SMC64', - 0x18: 'SYS64', - 0x20: 'IABT', - 0x21: 'IABT_HYP', - 0x22: 'PC_ALIGN', - 0x24: 'DABT', - 0x25: 'DABT_HYP', - 0x26: 'SP_ALIGN', - 0x28: 'FP_EXC32', - 0x2C: 'FP_EXC64', - 0x2F: 'SERROR', - 0x30: 'BREAKPT', - 0x31: 'BREAKPT_HYP', - 0x32: 'SOFTSTP', - 0x33: 'SOFTSTP_HYP', - 0x34: 'WATCHPT', - 0x35: 'WATCHPT_HYP', - 0x38: 'BKPT32', - 0x3A: 'VECTOR32', - 0x3C: 'BRK64', +AARCH64_EXIT_REASONS = { + 'UNKNOWN': 0x00, + 'WFI': 0x01, + 'CP15_32': 0x03, + 'CP15_64': 0x04, + 'CP14_MR': 0x05, + 'CP14_LS': 0x06, + 'FP_ASIMD': 0x07, + 'CP10_ID': 0x08, + 'CP14_64': 0x0C, + 'ILL_ISS': 0x0E, + 'SVC32': 0x11, + 'HVC32': 0x12, + 'SMC32': 0x13, + 'SVC64': 0x15, + 'HVC64': 0x16, + 'SMC64': 0x17, + 'SYS64': 0x18, + 'IABT': 0x20, + 'IABT_HYP': 0x21, + 'PC_ALIGN': 0x22, + 'DABT': 0x24, + 'DABT_HYP': 0x25, + 'SP_ALIGN': 0x26, + 'FP_EXC32': 0x28, + 'FP_EXC64': 0x2C, + 'SERROR': 0x2F, + 'BREAKPT': 0x30, + 'BREAKPT_HYP': 0x31, + 'SOFTSTP': 0x32, + 'SOFTSTP_HYP': 0x33, + 'WATCHPT': 0x34, + 'WATCHPT_HYP': 0x35, + 'BKPT32': 0x38, + 'VECTOR32': 0x3A, + 'BRK64': 0x3C, } # From include/uapi/linux/kvm.h, KVM_EXIT_xxx -userspace_exit_reasons = { - 0: 'UNKNOWN', - 1: 'EXCEPTION', - 2: 'IO', - 3: 'HYPERCALL', - 4: 'DEBUG', - 5: 'HLT', - 6: 'MMIO', - 7: 'IRQ_WINDOW_OPEN', - 8: 'SHUTDOWN', - 9: 'FAIL_ENTRY', - 10: 'INTR', - 11: 'SET_TPR', - 12: 'TPR_ACCESS', - 13: 'S390_SIEIC', - 14: 'S390_RESET', - 15: 'DCR', - 16: 'NMI', - 17: 'INTERNAL_ERROR', - 18: 'OSI', - 19: 'PAPR_HCALL', - 20: 'S390_UCONTROL', - 21: 'WATCHDOG', - 22: 'S390_TSCH', - 23: 'EPR', - 24: 'SYSTEM_EVENT', +USERSPACE_EXIT_REASONS = { + 'UNKNOWN': 0, + 'EXCEPTION': 1, + 'IO': 2, + 'HYPERCALL': 3, + 'DEBUG': 4, + 'HLT': 5, + 'MMIO': 6, + 'IRQ_WINDOW_OPEN': 7, + 'SHUTDOWN': 8, + 'FAIL_ENTRY': 9, + 'INTR': 10, + 'SET_TPR': 11, + 'TPR_ACCESS': 12, + 'S390_SIEIC': 13, + 'S390_RESET': 14, + 'DCR': 15, + 'NMI': 16, + 'INTERNAL_ERROR': 17, + 'OSI': 18, + 'PAPR_HCALL': 19, + 'S390_UCONTROL': 20, + 'WATCHDOG': 21, + 'S390_TSCH': 22, + 'EPR': 23, + 'SYSTEM_EVENT': 24, } -x86_exit_reasons = { - 'vmx': vmx_exit_reasons, - 'svm': svm_exit_reasons, +IOCTL_NUMBERS = { + 'SET_FILTER': 0x40082406, + 'ENABLE': 0x00002400, + 'DISABLE': 0x00002401, + 'RESET': 0x00002403, } -sc_perf_evt_open = None -exit_reasons = None +class Arch(object): + """Class that encapsulates global architecture specific data like + syscall and ioctl numbers. + + """ + @staticmethod + def get_arch(): + machine = os.uname()[4] + + if machine.startswith('ppc'): + return ArchPPC() + elif machine.startswith('aarch64'): + return ArchA64() + elif machine.startswith('s390'): + return ArchS390() + else: + # X86_64 + for line in open('/proc/cpuinfo'): + if not line.startswith('flags'): + continue + + flags = line.split() + if 'vmx' in flags: + return ArchX86(VMX_EXIT_REASONS) + if 'svm' in flags: + return ArchX86(SVM_EXIT_REASONS) + return + +class ArchX86(Arch): + def __init__(self, exit_reasons): + self.sc_perf_evt_open = 298 + self.ioctl_numbers = IOCTL_NUMBERS + self.exit_reasons = exit_reasons + +class ArchPPC(Arch): + def __init__(self): + self.sc_perf_evt_open = 319 + self.ioctl_numbers = IOCTL_NUMBERS + self.ioctl_numbers['ENABLE'] = 0x20002400 + self.ioctl_numbers['DISABLE'] = 0x20002401 -ioctl_numbers = { - 'SET_FILTER' : 0x40082406, - 'ENABLE' : 0x00002400, - 'DISABLE' : 0x00002401, - 'RESET' : 0x00002403, -} + # PPC comes in 32 and 64 bit and some generated ioctl + # numbers depend on the wordsize. + char_ptr_size = ctypes.sizeof(ctypes.c_char_p) + self.ioctl_numbers['SET_FILTER'] = 0x80002406 | char_ptr_size << 16 + +class ArchA64(Arch): + def __init__(self): + self.sc_perf_evt_open = 241 + self.ioctl_numbers = IOCTL_NUMBERS + self.exit_reasons = AARCH64_EXIT_REASONS + +class ArchS390(Arch): + def __init__(self): + self.sc_perf_evt_open = 331 + self.ioctl_numbers = IOCTL_NUMBERS + self.exit_reasons = None + +ARCH = Arch.get_arch() + + +def walkdir(path): + """Returns os.walk() data for specified directory. + + As it is only a wrapper it returns the same 3-tuple of (dirpath, + dirnames, filenames). + """ + return next(os.walk(path)) + + +def parse_int_list(list_string): + """Returns an int list from a string of comma separated integers and + integer ranges.""" + integers = [] + members = list_string.split(',') -def x86_init(flag): - globals().update({ - 'sc_perf_evt_open' : 298, - 'exit_reasons' : x86_exit_reasons[flag], - }) - -def s390_init(): - globals().update({ - 'sc_perf_evt_open' : 331 - }) - -def ppc_init(): - globals().update({ - 'sc_perf_evt_open' : 319, - 'ioctl_numbers' : { - 'SET_FILTER' : 0x80002406 | (ctypes.sizeof(ctypes.c_char_p) << 16), - 'ENABLE' : 0x20002400, - 'DISABLE' : 0x20002401, - } - }) - -def aarch64_init(): - globals().update({ - 'sc_perf_evt_open' : 241, - 'exit_reasons' : aarch64_exit_reasons, - }) - -def detect_platform(): - if os.uname()[4].startswith('ppc'): - ppc_init() - return - elif os.uname()[4].startswith('aarch64'): - aarch64_init() - return - - for line in file('/proc/cpuinfo').readlines(): - if line.startswith('flags'): - for flag in line.split(): - if flag in x86_exit_reasons: - x86_init(flag) - return - elif line.startswith('vendor_id'): - for flag in line.split(): - if flag == 'IBM/S390': - s390_init() - return - -detect_platform() - -def invert(d): - return dict((x[1], x[0]) for x in d.iteritems()) - -filters = {} -filters['kvm_userspace_exit'] = ('reason', invert(userspace_exit_reasons)) -if exit_reasons: - filters['kvm_exit'] = ('exit_reason', invert(exit_reasons)) - -import struct, array - -libc = ctypes.CDLL('libc.so.6') + for member in members: + if '-' not in member: + integers.append(int(member)) + else: + int_range = member.split('-') + integers.extend(range(int(int_range[0]), + int(int_range[1]) + 1)) + + return integers + + +def get_online_cpus(): + with open('/sys/devices/system/cpu/online') as cpu_list: + cpu_string = cpu_list.readline() + return parse_int_list(cpu_string) + + +def get_filters(): + filters = {} + filters['kvm_userspace_exit'] = ('reason', USERSPACE_EXIT_REASONS) + if ARCH.exit_reasons: + filters['kvm_exit'] = ('exit_reason', ARCH.exit_reasons) + return filters + +libc = ctypes.CDLL('libc.so.6', use_errno=True) syscall = libc.syscall -get_errno = libc.__errno_location -get_errno.restype = POINTER(c_int) class perf_event_attr(ctypes.Structure): _fields_ = [('type', ctypes.c_uint32), @@ -305,262 +332,350 @@ class perf_event_attr(ctypes.Structure): ('bp_addr', ctypes.c_uint64), ('bp_len', ctypes.c_uint64), ] -def _perf_event_open(attr, pid, cpu, group_fd, flags): - return syscall(sc_perf_evt_open, ctypes.pointer(attr), ctypes.c_int(pid), - ctypes.c_int(cpu), ctypes.c_int(group_fd), - ctypes.c_long(flags)) - -PERF_TYPE_HARDWARE = 0 -PERF_TYPE_SOFTWARE = 1 -PERF_TYPE_TRACEPOINT = 2 -PERF_TYPE_HW_CACHE = 3 -PERF_TYPE_RAW = 4 -PERF_TYPE_BREAKPOINT = 5 - -PERF_SAMPLE_IP = 1 << 0 -PERF_SAMPLE_TID = 1 << 1 -PERF_SAMPLE_TIME = 1 << 2 -PERF_SAMPLE_ADDR = 1 << 3 -PERF_SAMPLE_READ = 1 << 4 -PERF_SAMPLE_CALLCHAIN = 1 << 5 -PERF_SAMPLE_ID = 1 << 6 -PERF_SAMPLE_CPU = 1 << 7 -PERF_SAMPLE_PERIOD = 1 << 8 -PERF_SAMPLE_STREAM_ID = 1 << 9 -PERF_SAMPLE_RAW = 1 << 10 - -PERF_FORMAT_TOTAL_TIME_ENABLED = 1 << 0 -PERF_FORMAT_TOTAL_TIME_RUNNING = 1 << 1 -PERF_FORMAT_ID = 1 << 2 -PERF_FORMAT_GROUP = 1 << 3 -import re + def __init__(self): + super(self.__class__, self).__init__() + self.type = PERF_TYPE_TRACEPOINT + self.size = ctypes.sizeof(self) + self.read_format = PERF_FORMAT_GROUP + +def perf_event_open(attr, pid, cpu, group_fd, flags): + return syscall(ARCH.sc_perf_evt_open, ctypes.pointer(attr), + ctypes.c_int(pid), ctypes.c_int(cpu), + ctypes.c_int(group_fd), ctypes.c_long(flags)) -sys_tracing = '/sys/kernel/debug/tracing' +PERF_TYPE_TRACEPOINT = 2 +PERF_FORMAT_GROUP = 1 << 3 + +PATH_DEBUGFS_TRACING = '/sys/kernel/debug/tracing' +PATH_DEBUGFS_KVM = '/sys/kernel/debug/kvm' class Group(object): - def __init__(self, cpu): + def __init__(self): self.events = [] - self.group_leader = None - self.cpu = cpu - def add_event(self, name, event_set, tracepoint, filter = None): - self.events.append(Event(group = self, - name = name, event_set = event_set, - tracepoint = tracepoint, filter = filter)) - if len(self.events) == 1: - self.file = os.fdopen(self.events[0].fd) + + def add_event(self, event): + self.events.append(event) + def read(self): - bytes = 8 * (1 + len(self.events)) - fmt = 'xxxxxxxx' + 'q' * len(self.events) + length = 8 * (1 + len(self.events)) + read_format = 'xxxxxxxx' + 'Q' * len(self.events) return dict(zip([event.name for event in self.events], - struct.unpack(fmt, self.file.read(bytes)))) + struct.unpack(read_format, + os.read(self.events[0].fd, length)))) class Event(object): - def __init__(self, group, name, event_set, tracepoint, filter = None): + def __init__(self, name, group, trace_cpu, trace_point, trace_filter, + trace_set='kvm'): self.name = name - attr = perf_event_attr() - attr.type = PERF_TYPE_TRACEPOINT - attr.size = ctypes.sizeof(attr) - id_path = os.path.join(sys_tracing, 'events', event_set, - tracepoint, 'id') - id = int(file(id_path).read()) - attr.config = id - attr.sample_type = (PERF_SAMPLE_RAW - | PERF_SAMPLE_TIME - | PERF_SAMPLE_CPU) - attr.sample_period = 1 - attr.read_format = PERF_FORMAT_GROUP + self.fd = None + self.setup_event(group, trace_cpu, trace_point, trace_filter, + trace_set) + + def setup_event_attribute(self, trace_set, trace_point): + id_path = os.path.join(PATH_DEBUGFS_TRACING, 'events', trace_set, + trace_point, 'id') + + event_attr = perf_event_attr() + event_attr.config = int(open(id_path).read()) + return event_attr + + def setup_event(self, group, trace_cpu, trace_point, trace_filter, + trace_set): + event_attr = self.setup_event_attribute(trace_set, trace_point) + group_leader = -1 if group.events: group_leader = group.events[0].fd - fd = _perf_event_open(attr, -1, group.cpu, group_leader, 0) + + fd = perf_event_open(event_attr, -1, trace_cpu, + group_leader, 0) if fd == -1: - err = get_errno()[0] - raise Exception('perf_event_open failed, errno = ' + err.__str__()) - if filter: - import fcntl - fcntl.ioctl(fd, ioctl_numbers['SET_FILTER'], filter) + err = ctypes.get_errno() + raise OSError(err, os.strerror(err), + 'while calling sys_perf_event_open().') + + if trace_filter: + fcntl.ioctl(fd, ARCH.ioctl_numbers['SET_FILTER'], + trace_filter) + self.fd = fd + def enable(self): - import fcntl - fcntl.ioctl(self.fd, ioctl_numbers['ENABLE'], 0) + fcntl.ioctl(self.fd, ARCH.ioctl_numbers['ENABLE'], 0) + def disable(self): - import fcntl - fcntl.ioctl(self.fd, ioctl_numbers['DISABLE'], 0) + fcntl.ioctl(self.fd, ARCH.ioctl_numbers['DISABLE'], 0) + def reset(self): - import fcntl - fcntl.ioctl(self.fd, ioctl_numbers['RESET'], 0) + fcntl.ioctl(self.fd, ARCH.ioctl_numbers['RESET'], 0) class TracepointProvider(object): def __init__(self): - path = os.path.join(sys_tracing, 'events', 'kvm') - fields = [f - for f in os.listdir(path) - if os.path.isdir(os.path.join(path, f))] + self.group_leaders = [] + self.filters = get_filters() + self._fields = self.get_available_fields() + self.setup_traces() + self.fields = self._fields + + def get_available_fields(self): + path = os.path.join(PATH_DEBUGFS_TRACING, 'events', 'kvm') + fields = walkdir(path)[1] extra = [] - for f in fields: - if f in filters: - subfield, values = filters[f] - for name, number in values.iteritems(): - extra.append(f + '(' + name + ')') + for field in fields: + if field in self.filters: + filter_name_, filter_dicts = self.filters[field] + for name in filter_dicts: + extra.append(field + '(' + name + ')') fields += extra - self._setup(fields) - self.select(fields) - def fields(self): - return self._fields + return fields + + def setup_traces(self): + cpus = get_online_cpus() + + # The constant is needed as a buffer for python libs, std + # streams and other files that the script opens. + newlim = len(cpus) * len(self._fields) + 50 + try: + softlim_, hardlim = resource.getrlimit(resource.RLIMIT_NOFILE) + + if hardlim < newlim: + # Now we need CAP_SYS_RESOURCE, to increase the hard limit. + resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, newlim)) + else: + # Raising the soft limit is sufficient. + resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, hardlim)) + + except ValueError: + sys.exit("NOFILE rlimit could not be raised to {0}".format(newlim)) - def _online_cpus(self): - l = [] - pattern = r'cpu([0-9]+)' - basedir = '/sys/devices/system/cpu' - for entry in os.listdir(basedir): - match = re.match(pattern, entry) - if not match: - continue - path = os.path.join(basedir, entry, 'online') - if os.path.exists(path) and open(path).read().strip() != '1': - continue - l.append(int(match.group(1))) - return l - - def _setup(self, _fields): - self._fields = _fields - cpus = self._online_cpus() - import resource - nfiles = len(cpus) * 1000 - resource.setrlimit(resource.RLIMIT_NOFILE, (nfiles, nfiles)) - events = [] - self.group_leaders = [] for cpu in cpus: - group = Group(cpu) - for name in _fields: + group = Group() + for name in self._fields: tracepoint = name - filter = None - m = re.match(r'(.*)\((.*)\)', name) - if m: - tracepoint, sub = m.groups() - filter = '%s==%d\0' % (filters[tracepoint][0], - filters[tracepoint][1][sub]) - event = group.add_event(name, event_set = 'kvm', - tracepoint = tracepoint, - filter = filter) + tracefilter = None + match = re.match(r'(.*)\((.*)\)', name) + if match: + tracepoint, sub = match.groups() + tracefilter = ('%s==%d\0' % + (self.filters[tracepoint][0], + self.filters[tracepoint][1][sub])) + + group.add_event(Event(name=name, + group=group, + trace_cpu=cpu, + trace_point=tracepoint, + trace_filter=tracefilter)) self.group_leaders.append(group) - def select(self, fields): + + def available_fields(self): + return self.get_available_fields() + + @property + def fields(self): + return self._fields + + @fields.setter + def fields(self, fields): + self._fields = fields for group in self.group_leaders: - for event in group.events: + for index, event in enumerate(group.events): if event.name in fields: event.reset() event.enable() else: - event.disable() + # Do not disable the group leader. + # It would disable all of its events. + if index != 0: + event.disable() + def read(self): - from collections import defaultdict ret = defaultdict(int) for group in self.group_leaders: for name, val in group.read().iteritems(): - ret[name] += val + if name in self._fields: + ret[name] += val return ret -class Stats: - def __init__(self, providers, fields = None): +class DebugfsProvider(object): + def __init__(self): + self._fields = self.get_available_fields() + + def get_available_fields(self): + return walkdir(PATH_DEBUGFS_KVM)[2] + + @property + def fields(self): + return self._fields + + @fields.setter + def fields(self, fields): + self._fields = fields + + def read(self): + def val(key): + return int(file(PATH_DEBUGFS_KVM + '/' + key).read()) + return dict([(key, val(key)) for key in self._fields]) + +class Stats(object): + def __init__(self, providers, fields=None): self.providers = providers - self.fields_filter = fields - self._update() - def _update(self): + self._fields_filter = fields + self.values = {} + self.update_provider_filters() + + def update_provider_filters(self): def wanted(key): - import re - if not self.fields_filter: + if not self._fields_filter: return True - return re.match(self.fields_filter, key) is not None - self.values = dict() - for d in providers: - provider_fields = [key for key in d.fields() if wanted(key)] - for key in provider_fields: - self.values[key] = None - d.select(provider_fields) - def set_fields_filter(self, fields_filter): - self.fields_filter = fields_filter - self._update() + return re.match(self._fields_filter, key) is not None + + # As we reset the counters when updating the fields we can + # also clear the cache of old values. + self.values = {} + for provider in self.providers: + provider_fields = [key for key in provider.get_available_fields() + if wanted(key)] + provider.fields = provider_fields + + @property + def fields_filter(self): + return self._fields_filter + + @fields_filter.setter + def fields_filter(self, fields_filter): + self._fields_filter = fields_filter + self.update_provider_filters() + def get(self): - for d in providers: - new = d.read() - for key in d.fields(): + for provider in self.providers: + new = provider.read() + for key in provider.fields: oldval = self.values.get(key, (0, 0)) - newval = new[key] + newval = new.get(key, 0) newdelta = None if oldval is not None: newdelta = newval - oldval[0] self.values[key] = (newval, newdelta) return self.values -if not os.access('/sys/kernel/debug', os.F_OK): - print 'Please enable CONFIG_DEBUG_FS in your kernel' - sys.exit(1) -if not os.access('/sys/kernel/debug/kvm', os.F_OK): - print "Please mount debugfs ('mount -t debugfs debugfs /sys/kernel/debug')" - print "and ensure the kvm modules are loaded" - sys.exit(1) - -label_width = 40 -number_width = 10 - -def tui(screen, stats): - curses.use_default_colors() - curses.noecho() - drilldown = False - fields_filter = stats.fields_filter - def update_drilldown(): - if not fields_filter: - if drilldown: - stats.set_fields_filter(None) - else: - stats.set_fields_filter(r'^[^\(]*$') - update_drilldown() - def refresh(sleeptime): - screen.erase() - screen.addstr(0, 0, 'kvm statistics') - screen.addstr(2, 1, 'Event') - screen.addstr(2, 1 + label_width + number_width - len('Total'), 'Total') - screen.addstr(2, 1 + label_width + number_width + 8 - len('Current'), 'Current') +LABEL_WIDTH = 40 +NUMBER_WIDTH = 10 + +class Tui(object): + def __init__(self, stats): + self.stats = stats + self.screen = None + self.drilldown = False + self.update_drilldown() + + def __enter__(self): + """Initialises curses for later use. Based on curses.wrapper + implementation from the Python standard library.""" + self.screen = curses.initscr() + curses.noecho() + curses.cbreak() + + # The try/catch works around a minor bit of + # over-conscientiousness in the curses module, the error + # return from C start_color() is ignorable. + try: + curses.start_color() + except: + pass + + curses.use_default_colors() + return self + + def __exit__(self, *exception): + """Resets the terminal to its normal state. Based on curses.wrappre + implementation from the Python standard library.""" + if self.screen: + self.screen.keypad(0) + curses.echo() + curses.nocbreak() + curses.endwin() + + def update_drilldown(self): + if not self.stats.fields_filter: + self.stats.fields_filter = r'^[^\(]*$' + + elif self.stats.fields_filter == r'^[^\(]*$': + self.stats.fields_filter = None + + def refresh(self, sleeptime): + self.screen.erase() + self.screen.addstr(0, 0, 'kvm statistics - summary', curses.A_BOLD) + self.screen.addstr(2, 1, 'Event') + self.screen.addstr(2, 1 + LABEL_WIDTH + NUMBER_WIDTH - + len('Total'), 'Total') + self.screen.addstr(2, 1 + LABEL_WIDTH + NUMBER_WIDTH + 8 - + len('Current'), 'Current') row = 3 - s = stats.get() + stats = self.stats.get() def sortkey(x): - if s[x][1]: - return (-s[x][1], -s[x][0]) + if stats[x][1]: + return (-stats[x][1], -stats[x][0]) else: - return (0, -s[x][0]) - for key in sorted(s.keys(), key = sortkey): - if row >= screen.getmaxyx()[0]: + return (0, -stats[x][0]) + for key in sorted(stats.keys(), key=sortkey): + + if row >= self.screen.getmaxyx()[0]: break - values = s[key] + values = stats[key] if not values[0] and not values[1]: break col = 1 - screen.addstr(row, col, key) - col += label_width - screen.addstr(row, col, '%10d' % (values[0],)) - col += number_width + self.screen.addstr(row, col, key) + col += LABEL_WIDTH + self.screen.addstr(row, col, '%10d' % (values[0],)) + col += NUMBER_WIDTH if values[1] is not None: - screen.addstr(row, col, '%8d' % (values[1] / sleeptime,)) + self.screen.addstr(row, col, '%8d' % (values[1] / sleeptime,)) row += 1 - screen.refresh() + self.screen.refresh() + + def show_filter_selection(self): + while True: + self.screen.erase() + self.screen.addstr(0, 0, + "Show statistics for events matching a regex.", + curses.A_BOLD) + self.screen.addstr(2, 0, + "Current regex: {0}" + .format(self.stats.fields_filter)) + self.screen.addstr(3, 0, "New regex: ") + curses.echo() + regex = self.screen.getstr() + curses.noecho() + if len(regex) == 0: + return + try: + re.compile(regex) + self.stats.fields_filter = regex + return + except re.error: + continue - sleeptime = 0.25 - while True: - refresh(sleeptime) - curses.halfdelay(int(sleeptime * 10)) - sleeptime = 3 - try: - c = screen.getkey() - if c == 'x': - drilldown = not drilldown - update_drilldown() - if c == 'q': + def show_stats(self): + sleeptime = 0.25 + while True: + self.refresh(sleeptime) + curses.halfdelay(int(sleeptime * 10)) + sleeptime = 3 + try: + char = self.screen.getkey() + if char == 'x': + self.drilldown = not self.drilldown + self.update_drilldown() + if char == 'q': + break + if char == 'f': + self.show_filter_selection() + except KeyboardInterrupt: break - except KeyboardInterrupt: - break - except curses.error: - continue + except curses.error: + continue def batch(stats): s = stats.get() @@ -568,13 +683,13 @@ def batch(stats): s = stats.get() for key in sorted(s.keys()): values = s[key] - print '%-22s%10d%10d' % (key, values[0], values[1]) + print '%-42s%10d%10d' % (key, values[0], values[1]) def log(stats): keys = sorted(stats.get().iterkeys()) def banner(): for k in keys: - print '%10s' % k[0:9], + print '%s' % k, print def statline(): s = stats.get() @@ -590,57 +705,121 @@ def log(stats): statline() line += 1 -options = optparse.OptionParser() -options.add_option('-1', '--once', '--batch', - action = 'store_true', - default = False, - dest = 'once', - help = 'run in batch mode for one second', - ) -options.add_option('-l', '--log', - action = 'store_true', - default = False, - dest = 'log', - help = 'run in logging mode (like vmstat)', - ) -options.add_option('-t', '--tracepoints', - action = 'store_true', - default = False, - dest = 'tracepoints', - help = 'retrieve statistics from tracepoints', - ) -options.add_option('-d', '--debugfs', - action = 'store_true', - default = False, - dest = 'debugfs', - help = 'retrieve statistics from debugfs', - ) -options.add_option('-f', '--fields', - action = 'store', - default = None, - dest = 'fields', - help = 'fields to display (regex)', - ) -(options, args) = options.parse_args(sys.argv) - -providers = [] -if options.tracepoints: - providers.append(TracepointProvider()) -if options.debugfs: - providers.append(DebugfsProvider()) - -if len(providers) == 0: - try: - providers = [TracepointProvider()] - except: - providers = [DebugfsProvider()] - -stats = Stats(providers, fields = options.fields) - -if options.log: - log(stats) -elif not options.once: - import curses.wrapper - curses.wrapper(tui, stats) -else: - batch(stats) +def get_options(): + description_text = """ +This script displays various statistics about VMs running under KVM. +The statistics are gathered from the KVM debugfs entries and / or the +currently available perf traces. + +The monitoring takes additional cpu cycles and might affect the VM's +performance. + +Requirements: +- Access to: + /sys/kernel/debug/kvm + /sys/kernel/debug/trace/events/* + /proc/pid/task +- /proc/sys/kernel/perf_event_paranoid < 1 if user has no + CAP_SYS_ADMIN and perf events are used. +- CAP_SYS_RESOURCE if the hard limit is not high enough to allow + the large number of files that are possibly opened. +""" + + class PlainHelpFormatter(optparse.IndentedHelpFormatter): + def format_description(self, description): + if description: + return description + "\n" + else: + return "" + + optparser = optparse.OptionParser(description=description_text, + formatter=PlainHelpFormatter()) + optparser.add_option('-1', '--once', '--batch', + action='store_true', + default=False, + dest='once', + help='run in batch mode for one second', + ) + optparser.add_option('-l', '--log', + action='store_true', + default=False, + dest='log', + help='run in logging mode (like vmstat)', + ) + optparser.add_option('-t', '--tracepoints', + action='store_true', + default=False, + dest='tracepoints', + help='retrieve statistics from tracepoints', + ) + optparser.add_option('-d', '--debugfs', + action='store_true', + default=False, + dest='debugfs', + help='retrieve statistics from debugfs', + ) + optparser.add_option('-f', '--fields', + action='store', + default=None, + dest='fields', + help='fields to display (regex)', + ) + (options, _) = optparser.parse_args(sys.argv) + return options + +def get_providers(options): + providers = [] + + if options.tracepoints: + providers.append(TracepointProvider()) + if options.debugfs: + providers.append(DebugfsProvider()) + if len(providers) == 0: + providers.append(TracepointProvider()) + + return providers + +def check_access(options): + if not os.path.exists('/sys/kernel/debug'): + sys.stderr.write('Please enable CONFIG_DEBUG_FS in your kernel.') + sys.exit(1) + + if not os.path.exists(PATH_DEBUGFS_KVM): + sys.stderr.write("Please make sure, that debugfs is mounted and " + "readable by the current user:\n" + "('mount -t debugfs debugfs /sys/kernel/debug')\n" + "Also ensure, that the kvm modules are loaded.\n") + sys.exit(1) + + if not os.path.exists(PATH_DEBUGFS_TRACING) and (options.tracepoints + or not options.debugfs): + sys.stderr.write("Please enable CONFIG_TRACING in your kernel " + "when using the option -t (default).\n" + "If it is enabled, make {0} readable by the " + "current user.\n" + .format(PATH_DEBUGFS_TRACING)) + if options.tracepoints: + sys.exit(1) + + sys.stderr.write("Falling back to debugfs statistics!\n") + options.debugfs = True + sleep(5) + + return options + +def main(): + options = get_options() + options = check_access(options) + providers = get_providers(options) + stats = Stats(providers, fields=options.fields) + + if options.log: + log(stats) + elif not options.once: + with Tui(stats) as tui: + tui.show_stats() + else: + batch(stats) + +if __name__ == "__main__": + main() diff --git a/scripts/qapi-commands.py b/scripts/qapi-commands.py index 561e47a42..b570069fa 100644 --- a/scripts/qapi-commands.py +++ b/scripts/qapi-commands.py @@ -2,7 +2,7 @@ # QAPI command marshaller generator # # Copyright IBM, Corp. 2011 -# Copyright (C) 2014-2015 Red Hat, Inc. +# Copyright (C) 2014-2016 Red Hat, Inc. # # Authors: # Anthony Liguori <aliguori@us.ibm.com> @@ -30,10 +30,11 @@ def gen_call(name, arg_type, ret_type): argstr = '' if arg_type: + assert not arg_type.variants for memb in arg_type.members: if memb.optional: - argstr += 'has_%s, ' % c_name(memb.name) - argstr += '%s, ' % c_name(memb.name) + argstr += 'arg.has_%s, ' % c_name(memb.name) + argstr += 'arg.%s, ' % c_name(memb.name) lhs = '' if ret_type: @@ -54,72 +55,6 @@ def gen_call(name, arg_type, ret_type): return ret -def gen_marshal_vars(arg_type, ret_type): - ret = mcgen(''' - Error *err = NULL; -''') - - if ret_type: - ret += mcgen(''' - %(c_type)s retval; -''', - c_type=ret_type.c_type()) - - if arg_type: - ret += mcgen(''' - QmpInputVisitor *qiv = qmp_input_visitor_new_strict(QOBJECT(args)); - QapiDeallocVisitor *qdv; - Visitor *v; -''') - - for memb in arg_type.members: - if memb.optional: - ret += mcgen(''' - bool has_%(c_name)s = false; -''', - c_name=c_name(memb.name)) - ret += mcgen(''' - %(c_type)s %(c_name)s = %(c_null)s; -''', - c_name=c_name(memb.name), - c_type=memb.type.c_type(), - c_null=memb.type.c_null()) - ret += '\n' - else: - ret += mcgen(''' - - (void)args; -''') - - return ret - - -def gen_marshal_input_visit(arg_type, dealloc=False): - ret = '' - - if not arg_type: - return ret - - if dealloc: - ret += mcgen(''' - qmp_input_visitor_cleanup(qiv); - qdv = qapi_dealloc_visitor_new(); - v = qapi_dealloc_get_visitor(qdv); -''') - else: - ret += mcgen(''' - v = qmp_input_get_visitor(qiv); -''') - - ret += gen_visit_fields(arg_type.members, skiperr=dealloc) - - if dealloc: - ret += mcgen(''' - qapi_dealloc_visitor_cleanup(qdv); -''') - return ret - - def gen_marshal_output(ret_type): return mcgen(''' @@ -131,7 +66,7 @@ static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in, QObject **ret_out, Visitor *v; v = qmp_output_get_visitor(qov); - visit_type_%(c_name)s(v, &ret_in, "unused", &err); + visit_type_%(c_name)s(v, "unused", &ret_in, &err); if (err) { goto out; } @@ -142,7 +77,7 @@ out: qmp_output_visitor_cleanup(qov); qdv = qapi_dealloc_visitor_new(); v = qapi_dealloc_get_visitor(qdv); - visit_type_%(c_name)s(v, &ret_in, "unused", NULL); + visit_type_%(c_name)s(v, "unused", &ret_in, NULL); qapi_dealloc_visitor_cleanup(qdv); } ''', @@ -168,15 +103,40 @@ def gen_marshal(name, arg_type, ret_type): %(proto)s { + Error *err = NULL; ''', proto=gen_marshal_proto(name)) - ret += gen_marshal_vars(arg_type, ret_type) - ret += gen_marshal_input_visit(arg_type) + if ret_type: + ret += mcgen(''' + %(c_type)s retval; +''', + c_type=ret_type.c_type()) + + if arg_type and arg_type.members: + ret += mcgen(''' + QmpInputVisitor *qiv = qmp_input_visitor_new_strict(QOBJECT(args)); + QapiDeallocVisitor *qdv; + Visitor *v; + %(c_name)s arg = {0}; + + v = qmp_input_get_visitor(qiv); + visit_type_%(c_name)s_members(v, &arg, &err); + if (err) { + goto out; + } +''', + c_name=arg_type.c_name()) + + else: + ret += mcgen(''' + + (void)args; +''') + ret += gen_call(name, arg_type, ret_type) - # 'goto out' produced by gen_marshal_input_visit->gen_visit_fields() - # for each arg_type member, and by gen_call() for ret_type + # 'goto out' produced above for arg_type, and by gen_call() for ret_type if (arg_type and arg_type.members) or ret_type: ret += mcgen(''' @@ -185,7 +145,16 @@ out: ret += mcgen(''' error_propagate(errp, err); ''') - ret += gen_marshal_input_visit(arg_type, dealloc=True) + if arg_type and arg_type.members: + ret += mcgen(''' + qmp_input_visitor_cleanup(qiv); + qdv = qapi_dealloc_visitor_new(); + v = qapi_dealloc_get_visitor(qdv); + visit_type_%(c_name)s_members(v, &arg, NULL); + qapi_dealloc_visitor_cleanup(qdv); +''', + c_name=arg_type.c_name()) + ret += mcgen(''' } ''') @@ -297,6 +266,7 @@ h_comment = ''' c_comment, h_comment) fdef.write(mcgen(''' +#include "qemu/osdep.h" #include "qemu-common.h" #include "qemu/module.h" #include "qapi/qmp/types.h" diff --git a/scripts/qapi-event.py b/scripts/qapi-event.py index 720486f06..9b5c5b535 100644 --- a/scripts/qapi-event.py +++ b/scripts/qapi-event.py @@ -2,7 +2,7 @@ # QAPI event generator # # Copyright (c) 2014 Wenchao Xia -# Copyright (c) 2015 Red Hat Inc. +# Copyright (c) 2015-2016 Red Hat Inc. # # Authors: # Wenchao Xia <wenchaoqemu@gmail.com> @@ -28,7 +28,37 @@ def gen_event_send_decl(name, arg_type): proto=gen_event_send_proto(name, arg_type)) +# Declare and initialize an object 'qapi' using parameters from gen_params() +def gen_param_var(typ): + assert not typ.variants + ret = mcgen(''' + %(c_name)s param = { +''', + c_name=typ.c_name()) + sep = ' ' + for memb in typ.members: + ret += sep + sep = ', ' + if memb.optional: + ret += 'has_' + c_name(memb.name) + sep + if memb.type.name == 'str': + # Cast away const added in gen_params() + ret += '(char *)' + ret += c_name(memb.name) + ret += mcgen(''' + + }; +''') + return ret + + def gen_event_send(name, arg_type): + # FIXME: Our declaration of local variables (and of 'errp' in the + # parameter list) can collide with exploded members of the event's + # data type passed in as parameters. If this collision ever hits in + # practice, we can rename our local variables with a leading _ prefix, + # or split the code into a wrapper function that creates a boxed + # 'param' object then calls another to do the real work. ret = mcgen(''' %(proto)s @@ -43,11 +73,11 @@ def gen_event_send(name, arg_type): ret += mcgen(''' QmpOutputVisitor *qov; Visitor *v; - QObject *obj; - ''') + ret += gen_param_var(arg_type) ret += mcgen(''' + emit = qmp_event_get_func_emit(); if (!emit) { return; @@ -61,28 +91,21 @@ def gen_event_send(name, arg_type): if arg_type and arg_type.members: ret += mcgen(''' qov = qmp_output_visitor_new(); - g_assert(qov); - v = qmp_output_get_visitor(qov); - g_assert(v); - /* Fake visit, as if all members are under a structure */ - visit_start_struct(v, NULL, "", "%(name)s", 0, &err); -''', - name=name) - ret += gen_err_check() - ret += gen_visit_fields(arg_type.members, need_cast=True) - ret += mcgen(''' - visit_end_struct(v, &err); + visit_start_struct(v, "%(name)s", NULL, 0, &err); + if (err) { + goto out; + } + visit_type_%(c_name)s_members(v, ¶m, &err); + visit_end_struct(v, err ? NULL : &err); if (err) { goto out; } - obj = qmp_output_get_qobject(qov); - g_assert(obj != NULL); - - qdict_put_obj(qmp, "data", obj); -''') + qdict_put_obj(qmp, "data", qmp_output_get_qobject(qov)); +''', + name=name, c_name=arg_type.c_name()) ret += mcgen(''' emit(%(c_enum)s, qmp, &err); @@ -161,6 +184,7 @@ h_comment = ''' c_comment, h_comment) fdef.write(mcgen(''' +#include "qemu/osdep.h" #include "qemu-common.h" #include "%(prefix)sqapi-event.h" #include "%(prefix)sqapi-visit.h" diff --git a/scripts/qapi-introspect.py b/scripts/qapi-introspect.py index 64f2cd063..e0f926be0 100644 --- a/scripts/qapi-introspect.py +++ b/scripts/qapi-introspect.py @@ -1,7 +1,7 @@ # # QAPI introspection generator # -# Copyright (C) 2015 Red Hat, Inc. +# Copyright (C) 2015-2016 Red Hat, Inc. # # Authors: # Markus Armbruster <armbru@redhat.com> @@ -204,6 +204,7 @@ h_comment = ''' c_comment, h_comment) fdef.write(mcgen(''' +#include "qemu/osdep.h" #include "%(prefix)sqmp-introspect.h" ''', diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py index b37900f6f..437cf6c8e 100644 --- a/scripts/qapi-types.py +++ b/scripts/qapi-types.py @@ -2,7 +2,7 @@ # QAPI types generator # # Copyright IBM, Corp. 2011 -# Copyright (c) 2013-2015 Red Hat Inc. +# Copyright (c) 2013-2016 Red Hat Inc. # # Authors: # Anthony Liguori <aliguori@us.ibm.com> @@ -14,6 +14,11 @@ from qapi import * +# variants must be emitted before their container; track what has already +# been output +objects_seen = set() + + def gen_fwd_object_or_array(name): return mcgen(''' @@ -26,66 +31,69 @@ def gen_array(name, element_type): return mcgen(''' struct %(c_name)s { - union { - %(c_type)s value; - uint64_t padding; - }; %(c_name)s *next; + %(c_type)s value; }; ''', c_name=c_name(name), c_type=element_type.c_type()) -def gen_struct_field(member): +def gen_struct_members(members): ret = '' - - if member.optional: - ret += mcgen(''' + for memb in members: + if memb.optional: + ret += mcgen(''' bool has_%(c_name)s; ''', - c_name=c_name(member.name)) - ret += mcgen(''' + c_name=c_name(memb.name)) + ret += mcgen(''' %(c_type)s %(c_name)s; ''', - c_type=member.type.c_type(), c_name=c_name(member.name)) + c_type=memb.type.c_type(), c_name=c_name(memb.name)) return ret -def gen_struct_fields(local_members, base=None): +def gen_object(name, base, members, variants): + if name in objects_seen: + return '' + objects_seen.add(name) + ret = '' + if variants: + for v in variants.variants: + if isinstance(v.type, QAPISchemaObjectType): + ret += gen_object(v.type.name, v.type.base, + v.type.local_members, v.type.variants) + + ret += mcgen(''' + +struct %(c_name)s { +''', + c_name=c_name(name)) if base: - ret += mcgen(''' + if not base.is_implicit(): + ret += mcgen(''' /* Members inherited from %(c_name)s: */ ''', - c_name=base.c_name()) - for memb in base.members: - ret += gen_struct_field(memb) - ret += mcgen(''' + c_name=base.c_name()) + ret += gen_struct_members(base.members) + if not base.is_implicit(): + ret += mcgen(''' /* Own members: */ ''') + ret += gen_struct_members(members) - for memb in local_members: - ret += gen_struct_field(memb) - return ret - - -def gen_struct(name, base, members): - ret = mcgen(''' - -struct %(c_name)s { -''', - c_name=c_name(name)) + if variants: + ret += gen_variants(variants) - ret += gen_struct_fields(members, base) - - # Make sure that all structs have at least one field; this avoids + # Make sure that all structs have at least one member; this avoids # potential issues with attempting to malloc space for zero-length # structs in C, and also incompatibility with C++ (where an empty # struct is size 1). - if not (base and base.members) and not members: + if not (base and base.members) and not members and not variants: ret += mcgen(''' - char qapi_dummy_field_for_empty_struct; + char qapi_dummy_for_empty_struct; ''') ret += mcgen(''' @@ -108,75 +116,21 @@ static inline %(base)s *qapi_%(c_name)s_base(const %(c_name)s *obj) c_name=c_name(name), base=base.c_name()) -def gen_alternate_qtypes_decl(name): - return mcgen(''' - -extern const int %(c_name)s_qtypes[]; -''', - c_name=c_name(name)) - - -def gen_alternate_qtypes(name, variants): +def gen_variants(variants): ret = mcgen(''' - -const int %(c_name)s_qtypes[QTYPE_MAX] = { -''', - c_name=c_name(name)) - - for var in variants.variants: - qtype = var.type.alternate_qtype() - assert qtype - - ret += mcgen(''' - [%(qtype)s] = %(enum_const)s, -''', - qtype=qtype, - enum_const=c_enum_const(variants.tag_member.type.name, - var.name)) - - ret += mcgen(''' -}; -''') - return ret - - -def gen_union(name, base, variants): - ret = mcgen(''' - -struct %(c_name)s { -''', - c_name=c_name(name)) - if base: - ret += gen_struct_fields([], base) - else: - ret += gen_struct_field(variants.tag_member) - - # FIXME: What purpose does data serve, besides preventing a union that - # has a branch named 'data'? We use it in qapi-visit.py to decide - # whether to bypass the switch statement if visiting the discriminator - # failed; but since we 0-initialize structs, and cannot tell what - # branch of the union is in use if the discriminator is invalid, there - # should not be any data leaks even without a data pointer. Or, if - # 'data' is merely added to guarantee we don't have an empty union, - # shouldn't we enforce that at .json parse time? - ret += mcgen(''' union { /* union tag is @%(c_name)s */ - void *data; ''', - c_name=c_name(variants.tag_member.name)) + c_name=c_name(variants.tag_member.name)) for var in variants.variants: - # Ugly special case for simple union TODO get rid of it - typ = var.simple_union_type() or var.type ret += mcgen(''' %(c_type)s %(c_name)s; ''', - c_type=typ.c_type(), + c_type=var.type.c_unboxed_type(), c_name=c_name(var.name)) ret += mcgen(''' } u; -}; ''') return ret @@ -205,7 +159,7 @@ void qapi_free_%(c_name)s(%(c_name)s *obj) qdv = qapi_dealloc_visitor_new(); v = qapi_dealloc_get_visitor(qdv); - visit_type_%(c_name)s(v, &obj, NULL, NULL); + visit_type_%(c_name)s(v, NULL, &obj, NULL); qapi_dealloc_visitor_cleanup(qdv); } ''', @@ -218,21 +172,19 @@ class QAPISchemaGenTypeVisitor(QAPISchemaVisitor): self.decl = None self.defn = None self._fwdecl = None - self._fwdefn = None self._btin = None def visit_begin(self, schema): + # gen_object() is recursive, ensure it doesn't visit the empty type + objects_seen.add(schema.the_empty_object_type.name) self.decl = '' self.defn = '' self._fwdecl = '' - self._fwdefn = '' self._btin = guardstart('QAPI_TYPES_BUILTIN') def visit_end(self): self.decl = self._fwdecl + self.decl self._fwdecl = None - self.defn = self._fwdefn + self.defn - self._fwdefn = None # To avoid header dependency hell, we always generate # declarations for built-in types in our header files and # simply guard them. See also do_builtins (command line @@ -241,18 +193,20 @@ class QAPISchemaGenTypeVisitor(QAPISchemaVisitor): self.decl = self._btin + self.decl self._btin = None - def visit_needed(self, entity): - # Visit everything except implicit objects - return not (entity.is_implicit() and - isinstance(entity, QAPISchemaObjectType)) - def _gen_type_cleanup(self, name): self.decl += gen_type_cleanup_decl(name) self.defn += gen_type_cleanup(name) def visit_enum_type(self, name, info, values, prefix): - self._fwdecl += gen_enum(name, values, prefix) - self._fwdefn += gen_enum_lookup(name, values, prefix) + # Special case for our lone builtin enum type + # TODO use something cleaner than existence of info + if not info: + self._btin += gen_enum(name, values, prefix) + if do_builtins: + self.defn += gen_enum_lookup(name, values, prefix) + else: + self._fwdecl += gen_enum(name, values, prefix) + self.defn += gen_enum_lookup(name, values, prefix) def visit_array_type(self, name, info, element_type): if isinstance(element_type, QAPISchemaBuiltinType): @@ -267,21 +221,22 @@ class QAPISchemaGenTypeVisitor(QAPISchemaVisitor): self._gen_type_cleanup(name) def visit_object_type(self, name, info, base, members, variants): + # Nothing to do for the special empty builtin + if name == 'q_empty': + return self._fwdecl += gen_fwd_object_or_array(name) - if variants: - assert not members # not implemented - self.decl += gen_union(name, base, variants) - else: - self.decl += gen_struct(name, base, members) - if base: + self.decl += gen_object(name, base, members, variants) + if base and not base.is_implicit(): self.decl += gen_upcast(name, base) - self._gen_type_cleanup(name) + # TODO Worth changing the visitor signature, so we could + # directly use rather than repeat type.is_implicit()? + if not name.startswith('q_'): + # implicit types won't be directly allocated/freed + self._gen_type_cleanup(name) def visit_alternate_type(self, name, info, variants): self._fwdecl += gen_fwd_object_or_array(name) - self._fwdefn += gen_alternate_qtypes(name, variants) - self.decl += gen_union(name, None, variants) - self.decl += gen_alternate_qtypes_decl(name) + self.decl += gen_object(name, None, [variants.tag_member], variants) self._gen_type_cleanup(name) # If you link code generated from multiple schemata, you want only one @@ -332,18 +287,13 @@ h_comment = ''' c_comment, h_comment) fdef.write(mcgen(''' +#include "qemu/osdep.h" #include "qapi/dealloc-visitor.h" #include "%(prefix)sqapi-types.h" #include "%(prefix)sqapi-visit.h" ''', prefix=prefix)) -fdecl.write(mcgen(''' -#include <stdbool.h> -#include <stdint.h> -#include "qapi/qmp/qobject.h" -''')) - schema = QAPISchema(input_file) gen = QAPISchemaGenTypeVisitor() schema.visit(gen) diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py index 3ef5c16a6..31d233035 100644 --- a/scripts/qapi-visit.py +++ b/scripts/qapi-visit.py @@ -2,7 +2,7 @@ # QAPI visitor generator # # Copyright IBM, Corp. 2011 -# Copyright (C) 2014-2015 Red Hat, Inc. +# Copyright (C) 2014-2016 Red Hat, Inc. # # Authors: # Anthony Liguori <aliguori@us.ibm.com> @@ -15,89 +15,87 @@ from qapi import * import re -# visit_type_FOO_implicit() is emitted as needed; track if it has already -# been output. -implicit_structs_seen = set() - -# visit_type_FOO_fields() is always emitted; track if a forward declaration -# or implementation has already been output. -struct_fields_seen = set() - def gen_visit_decl(name, scalar=False): c_type = c_name(name) + ' *' if not scalar: c_type += '*' return mcgen(''' -void visit_type_%(c_name)s(Visitor *v, %(c_type)sobj, const char *name, Error **errp); +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_type)sobj, Error **errp); ''', c_name=c_name(name), c_type=c_type) -def gen_visit_fields_decl(typ): - ret = '' - if typ.name not in struct_fields_seen: - ret += mcgen(''' +def gen_visit_members_decl(name): + return mcgen(''' -static void visit_type_%(c_type)s_fields(Visitor *v, %(c_type)s **obj, Error **errp); +void visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp); ''', - c_type=typ.c_name()) - struct_fields_seen.add(typ.name) - return ret - - -def gen_visit_implicit_struct(typ): - if typ in implicit_structs_seen: - return '' - implicit_structs_seen.add(typ) + c_name=c_name(name)) - ret = gen_visit_fields_decl(typ) - ret += mcgen(''' +def gen_visit_object_members(name, base, members, variants): + ret = mcgen(''' -static void visit_type_implicit_%(c_type)s(Visitor *v, %(c_type)s **obj, Error **errp) +void visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp) { Error *err = NULL; - visit_start_implicit_struct(v, (void **)obj, sizeof(%(c_type)s), &err); - if (!err) { - visit_type_%(c_type)s_fields(v, obj, errp); - visit_end_implicit_struct(v, &err); - } - error_propagate(errp, err); -} ''', - c_type=typ.c_name()) - return ret - - -def gen_visit_struct_fields(name, base, members): - ret = '' + c_name=c_name(name)) if base: - ret += gen_visit_fields_decl(base) - - struct_fields_seen.add(name) - ret += mcgen(''' - -static void visit_type_%(c_name)s_fields(Visitor *v, %(c_name)s **obj, Error **errp) -{ - Error *err = NULL; - + ret += mcgen(''' + visit_type_%(c_type)s_members(v, (%(c_type)s *)obj, &err); ''', - c_name=c_name(name)) + c_type=base.c_name()) + ret += gen_err_check() - if base: + for memb in members: + if memb.optional: + ret += mcgen(''' + if (visit_optional(v, "%(name)s", &obj->has_%(c_name)s)) { +''', + name=memb.name, c_name=c_name(memb.name)) + push_indent() ret += mcgen(''' - visit_type_%(c_type)s_fields(v, (%(c_type)s **)obj, &err); + visit_type_%(c_type)s(v, "%(name)s", &obj->%(c_name)s, &err); ''', - c_type=base.c_name()) + c_type=memb.type.c_name(), name=memb.name, + c_name=c_name(memb.name)) ret += gen_err_check() + if memb.optional: + pop_indent() + ret += mcgen(''' + } +''') + + if variants: + ret += mcgen(''' + switch (obj->%(c_name)s) { +''', + c_name=c_name(variants.tag_member.name)) + + for var in variants.variants: + ret += mcgen(''' + case %(case)s: + visit_type_%(c_type)s_members(v, &obj->u.%(c_name)s, &err); + break; +''', + case=c_enum_const(variants.tag_member.type.name, + var.name, + variants.tag_member.type.prefix), + c_type=var.type.c_name(), c_name=c_name(var.name)) - ret += gen_visit_fields(members, prefix='(*obj)->') + ret += mcgen(''' + default: + abort(); + } +''') - # 'goto out' produced for base, and by gen_visit_fields() for each member - if base or members: + # 'goto out' produced for base, for each member, and if variants were + # present + if base or members or variants: ret += mcgen(''' out: @@ -109,34 +107,6 @@ out: return ret -def gen_visit_struct(name, base, members): - ret = gen_visit_struct_fields(name, base, members) - - # FIXME: if *obj is NULL on entry, and visit_start_struct() assigns to - # *obj, but then visit_type_FOO_fields() fails, we should clean up *obj - # rather than leaving it non-NULL. As currently written, the caller must - # call qapi_free_FOO() to avoid a memory leak of the partial FOO. - ret += mcgen(''' - -void visit_type_%(c_name)s(Visitor *v, %(c_name)s **obj, const char *name, Error **errp) -{ - Error *err = NULL; - - visit_start_struct(v, (void **)obj, "%(name)s", name, sizeof(%(c_name)s), &err); - if (!err) { - if (*obj) { - visit_type_%(c_name)s_fields(v, obj, errp); - } - visit_end_struct(v, &err); - } - error_propagate(errp, err); -} -''', - name=name, c_name=c_name(name)) - - return ret - - def gen_visit_list(name, element_type): # FIXME: if *obj is NULL on entry, and the first visit_next_list() # assigns to *obj, while a later one fails, we should clean up *obj @@ -144,7 +114,7 @@ def gen_visit_list(name, element_type): # call qapi_free_FOOList() to avoid a memory leak of the partial FOOList. return mcgen(''' -void visit_type_%(c_name)s(Visitor *v, %(c_name)s **obj, const char *name, Error **errp) +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp) { Error *err = NULL; GenericList *i, **prev; @@ -155,15 +125,13 @@ void visit_type_%(c_name)s(Visitor *v, %(c_name)s **obj, const char *name, Error } for (prev = (GenericList **)obj; - !err && (i = visit_next_list(v, prev, &err)) != NULL; + !err && (i = visit_next_list(v, prev, sizeof(**obj))) != NULL; prev = &i) { %(c_name)s *native_i = (%(c_name)s *)i; - visit_type_%(c_elt_type)s(v, &native_i->value, NULL, &err); + visit_type_%(c_elt_type)s(v, NULL, &native_i->value, &err); } - error_propagate(errp, err); - err = NULL; - visit_end_list(v, &err); + visit_end_list(v); out: error_propagate(errp, err); } @@ -174,151 +142,109 @@ out: def gen_visit_enum(name): return mcgen(''' -void visit_type_%(c_name)s(Visitor *v, %(c_name)s *obj, const char *name, Error **errp) +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s *obj, Error **errp) { - visit_type_enum(v, (int *)obj, %(c_name)s_lookup, "%(name)s", name, errp); + int value = *obj; + visit_type_enum(v, name, &value, %(c_name)s_lookup, errp); + *obj = value; } ''', - c_name=c_name(name), name=name) + c_name=c_name(name)) def gen_visit_alternate(name, variants): - ret = mcgen(''' + promote_int = 'true' + ret = '' + for var in variants.variants: + if var.type.alternate_qtype() == 'QTYPE_QINT': + promote_int = 'false' + + ret += mcgen(''' -void visit_type_%(c_name)s(Visitor *v, %(c_name)s **obj, const char *name, Error **errp) +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp) { Error *err = NULL; - visit_start_implicit_struct(v, (void**) obj, sizeof(%(c_name)s), &err); + visit_start_alternate(v, name, (GenericAlternate **)obj, sizeof(**obj), + %(promote_int)s, &err); if (err) { goto out; } - visit_get_next_type(v, (int*) &(*obj)->type, %(c_name)s_qtypes, name, &err); - if (err) { - goto out_obj; - } switch ((*obj)->type) { ''', - c_name=c_name(name)) + c_name=c_name(name), promote_int=promote_int) for var in variants.variants: ret += mcgen(''' case %(case)s: - visit_type_%(c_type)s(v, &(*obj)->u.%(c_name)s, name, &err); - break; ''', - case=c_enum_const(variants.tag_member.type.name, - var.name), - c_type=var.type.c_name(), - c_name=c_name(var.name)) + case=var.type.alternate_qtype()) + if isinstance(var.type, QAPISchemaObjectType): + ret += mcgen(''' + visit_start_struct(v, name, NULL, 0, &err); + if (err) { + break; + } + visit_type_%(c_type)s_members(v, &(*obj)->u.%(c_name)s, &err); + error_propagate(errp, err); + err = NULL; + visit_end_struct(v, &err); +''', + c_type=var.type.c_name(), + c_name=c_name(var.name)) + else: + ret += mcgen(''' + visit_type_%(c_type)s(v, name, &(*obj)->u.%(c_name)s, &err); +''', + c_type=var.type.c_name(), + c_name=c_name(var.name)) + ret += mcgen(''' + break; +''') ret += mcgen(''' default: - abort(); + error_setg(&err, QERR_INVALID_PARAMETER_TYPE, name ? name : "null", + "%(name)s"); } -out_obj: - error_propagate(errp, err); - err = NULL; - visit_end_implicit_struct(v, &err); + visit_end_alternate(v); out: error_propagate(errp, err); } -''') +''', + name=name) return ret -def gen_visit_union(name, base, variants): - ret = '' - - if base: - ret += gen_visit_fields_decl(base) - - for var in variants.variants: - # Ugly special case for simple union TODO get rid of it - if not var.simple_union_type(): - ret += gen_visit_implicit_struct(var.type) - - ret += mcgen(''' +def gen_visit_object(name, base, members, variants): + # FIXME: if *obj is NULL on entry, and visit_start_struct() assigns to + # *obj, but then visit_type_FOO_members() fails, we should clean up *obj + # rather than leaving it non-NULL. As currently written, the caller must + # call qapi_free_FOO() to avoid a memory leak of the partial FOO. + return mcgen(''' -void visit_type_%(c_name)s(Visitor *v, %(c_name)s **obj, const char *name, Error **errp) +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp) { Error *err = NULL; - visit_start_struct(v, (void **)obj, "%(name)s", name, sizeof(%(c_name)s), &err); + visit_start_struct(v, name, (void **)obj, sizeof(%(c_name)s), &err); if (err) { goto out; } if (!*obj) { goto out_obj; } -''', - c_name=c_name(name), name=name) - - if base: - ret += mcgen(''' - visit_type_%(c_name)s_fields(v, (%(c_name)s **)obj, &err); -''', - c_name=base.c_name()) - else: - ret += mcgen(''' - visit_type_%(c_type)s(v, &(*obj)->%(c_name)s, "%(name)s", &err); -''', - c_type=variants.tag_member.type.c_name(), - c_name=c_name(variants.tag_member.name), - name=variants.tag_member.name) - ret += gen_err_check(label='out_obj') - ret += mcgen(''' - if (!visit_start_union(v, !!(*obj)->u.data, &err) || err) { - goto out_obj; - } - switch ((*obj)->%(c_name)s) { -''', - c_name=c_name(variants.tag_member.name)) - - for var in variants.variants: - # TODO ugly special case for simple union - simple_union_type = var.simple_union_type() - ret += mcgen(''' - case %(case)s: -''', - case=c_enum_const(variants.tag_member.type.name, - var.name)) - if simple_union_type: - ret += mcgen(''' - visit_type_%(c_type)s(v, &(*obj)->u.%(c_name)s, "data", &err); -''', - c_type=simple_union_type.c_name(), - c_name=c_name(var.name)) - else: - ret += mcgen(''' - visit_type_implicit_%(c_type)s(v, &(*obj)->u.%(c_name)s, &err); -''', - c_type=var.type.c_name(), - c_name=c_name(var.name)) - ret += mcgen(''' - break; -''') - - ret += mcgen(''' - default: - abort(); - } -out_obj: - error_propagate(errp, err); - err = NULL; - if (*obj) { - visit_end_union(v, !!(*obj)->u.data, &err); - } + visit_type_%(c_name)s_members(v, *obj, &err); error_propagate(errp, err); err = NULL; +out_obj: visit_end_struct(v, &err); out: error_propagate(errp, err); } -''') - - return ret +''', + c_name=c_name(name)) class QAPISchemaGenVisitVisitor(QAPISchemaVisitor): @@ -341,14 +267,16 @@ class QAPISchemaGenVisitVisitor(QAPISchemaVisitor): self.decl = self._btin + self.decl self._btin = None - def visit_needed(self, entity): - # Visit everything except implicit objects - return not (entity.is_implicit() and - isinstance(entity, QAPISchemaObjectType)) - def visit_enum_type(self, name, info, values, prefix): - self.decl += gen_visit_decl(name, scalar=True) - self.defn += gen_visit_enum(name) + # Special case for our lone builtin enum type + # TODO use something cleaner than existence of info + if not info: + self._btin += gen_visit_decl(name, scalar=True) + if do_builtins: + self.defn += gen_visit_enum(name) + else: + self.decl += gen_visit_decl(name, scalar=True) + self.defn += gen_visit_enum(name) def visit_array_type(self, name, info, element_type): decl = gen_visit_decl(name) @@ -362,12 +290,17 @@ class QAPISchemaGenVisitVisitor(QAPISchemaVisitor): self.defn += defn def visit_object_type(self, name, info, base, members, variants): - self.decl += gen_visit_decl(name) - if variants: - assert not members # not implemented - self.defn += gen_visit_union(name, base, variants) - else: - self.defn += gen_visit_struct(name, base, members) + # Nothing to do for the special empty builtin + if name == 'q_empty': + return + self.decl += gen_visit_members_decl(name) + self.defn += gen_visit_object_members(name, base, members, variants) + # TODO Worth changing the visitor signature, so we could + # directly use rather than repeat type.is_implicit()? + if not name.startswith('q_'): + # only explicit types need an allocating visit + self.decl += gen_visit_decl(name) + self.defn += gen_visit_object(name, base, members, variants) def visit_alternate_type(self, name, info, variants): self.decl += gen_visit_decl(name) @@ -420,13 +353,16 @@ h_comment = ''' c_comment, h_comment) fdef.write(mcgen(''' +#include "qemu/osdep.h" #include "qemu-common.h" +#include "qapi/error.h" #include "%(prefix)sqapi-visit.h" ''', prefix=prefix)) fdecl.write(mcgen(''' #include "qapi/visitor.h" +#include "qapi/qmp/qerror.h" #include "%(prefix)sqapi-types.h" ''', diff --git a/scripts/qapi.py b/scripts/qapi.py index 7c50cc4c8..b13ae4789 100644 --- a/scripts/qapi.py +++ b/scripts/qapi.py @@ -2,7 +2,7 @@ # QAPI helper library # # Copyright IBM, Corp. 2011 -# Copyright (c) 2013-2015 Red Hat Inc. +# Copyright (c) 2013-2016 Red Hat Inc. # # Authors: # Anthony Liguori <aliguori@us.ibm.com> @@ -33,7 +33,8 @@ builtin_types = { 'uint32': 'QTYPE_QINT', 'uint64': 'QTYPE_QINT', 'size': 'QTYPE_QINT', - 'any': None, # any qtype_code possible, actually + 'any': None, # any QType possible, actually + 'QType': 'QTYPE_QSTRING', } # Whitelist of commands allowed to return a non-dictionary @@ -58,6 +59,18 @@ returns_whitelist = [ 'guest-sync-delimited', ] +# Whitelist of entities allowed to violate case conventions +case_whitelist = [ + # From QMP: + 'ACPISlotType', # DIMM, visible through query-acpi-ospm-status + 'CpuInfoMIPS', # PC, visible through query-cpu + 'CpuInfoTricore', # PC, visible through query-cpu + 'QapiErrorClass', # all members, visible through errors + 'UuidInfo', # UUID, visible through query-uuid + 'X86CPURegister32', # all members, visible indirectly through qom-get + 'q_obj_CpuInfo-base', # CPU, visible through query-cpu +] + enum_types = [] struct_types = [] union_types = [] @@ -152,7 +165,7 @@ class QAPISchemaParser(object): continue try: fobj = open(incl_abs_fname, 'r') - except IOError, e: + except IOError as e: raise QAPIExprError(expr_info, '%s: %s' % (e.strerror, include)) exprs_include = QAPISchemaParser(fobj, previously_included, @@ -313,7 +326,9 @@ class QAPISchemaParser(object): # -def find_base_fields(base): +def find_base_members(base): + if isinstance(base, dict): + return base base_struct_define = find_struct(base) if not base_struct_define: return None @@ -342,20 +357,22 @@ def discriminator_find_enum_define(expr): if not (discriminator and base): return None - base_fields = find_base_fields(base) - if not base_fields: + base_members = find_base_members(base) + if not base_members: return None - discriminator_type = base_fields.get(discriminator) + discriminator_type = base_members.get(discriminator) if not discriminator_type: return None return find_enum(discriminator_type) -# FIXME should enforce "other than downstream extensions [...], all -# names should begin with a letter". -valid_name = re.compile('^[a-zA-Z_][a-zA-Z0-9_.-]*$') +# Names must be letters, numbers, -, and _. They must start with letter, +# except for downstream extensions which must start with __RFQDN_. +# Dots are only valid in the downstream extension prefix. +valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?' + '[a-zA-Z][a-zA-Z0-9_-]*$') def check_name(expr_info, source, name, allow_optional=False, @@ -374,9 +391,10 @@ def check_name(expr_info, source, name, allow_optional=False, % (source, name)) # Enum members can start with a digit, because the generated C # code always prefixes it with the enum name - if enum_member: - membername = '_' + membername - # Reserve the entire 'q_' namespace for c_name() + if enum_member and membername[0].isdigit(): + membername = 'D' + membername + # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' + # and 'q_obj_*' implicit type names. if not valid_name.match(membername) or \ c_name(membername, False).startswith('q_'): raise QAPIExprError(expr_info, @@ -502,21 +520,6 @@ def check_type(expr_info, source, value, allow_array=False, 'enum']) -def check_member_clash(expr_info, base_name, data, source=""): - base = find_struct(base_name) - assert base - base_members = base['data'] - for key in data.keys(): - if key.startswith('*'): - key = key[1:] - if key in base_members or "*" + key in base_members: - raise QAPIExprError(expr_info, - "Member name '%s'%s clashes with base '%s'" - % (key, source, base_name)) - if base.get('base'): - check_member_clash(expr_info, base['base'], data, source) - - def check_command(expr, expr_info): name = expr['command'] @@ -535,8 +538,6 @@ def check_event(expr, expr_info): global events name = expr['event'] - if name.upper() == 'MAX': - raise QAPIExprError(expr_info, "Event name 'MAX' cannot be created") events.append(name) check_type(expr_info, "'data' for event '%s'" % name, expr.get('data'), allow_dict=True, allow_optional=True, @@ -548,8 +549,6 @@ def check_union(expr, expr_info): base = expr.get('base') discriminator = expr.get('discriminator') members = expr['data'] - values = {'MAX': '(automatic)', 'KIND': '(automatic)', - 'TYPE': '(automatic)'} # Two types of unions, determined by discriminator. @@ -564,21 +563,22 @@ def check_union(expr, expr_info): # Else, it's a flat union. else: - # The object must have a string member 'base'. + # The object must have a string or dictionary 'base'. check_type(expr_info, "'base' for union '%s'" % name, - base, allow_metas=['struct']) + base, allow_dict=True, allow_optional=True, + allow_metas=['struct']) if not base: raise QAPIExprError(expr_info, "Flat union '%s' must have a base" % name) - base_fields = find_base_fields(base) - assert base_fields + base_members = find_base_members(base) + assert base_members # The value of member 'discriminator' must name a non-optional # member of the base struct. check_name(expr_info, "Discriminator of flat union '%s'" % name, discriminator) - discriminator_type = base_fields.get(discriminator) + discriminator_type = base_members.get(discriminator) if not discriminator_type: raise QAPIExprError(expr_info, "Discriminator '%s' is not a member of base " @@ -592,69 +592,49 @@ def check_union(expr, expr_info): "Discriminator '%s' must be of enumeration " "type" % discriminator) - # Check every branch + # Check every branch; don't allow an empty union + if len(members) == 0: + raise QAPIExprError(expr_info, + "Union '%s' cannot have empty 'data'" % name) for (key, value) in members.items(): check_name(expr_info, "Member of union '%s'" % name, key) - # Each value must name a known type; furthermore, in flat unions, - # branches must be a struct with no overlapping member names + # Each value must name a known type check_type(expr_info, "Member '%s' of union '%s'" % (key, name), value, allow_array=not base, allow_metas=allow_metas) - if base: - branch_struct = find_struct(value) - assert branch_struct - check_member_clash(expr_info, base, branch_struct['data'], - " of branch '%s'" % key) # If the discriminator names an enum type, then all members - # of 'data' must also be members of the enum type, which in turn - # must not collide with the discriminator name. + # of 'data' must also be members of the enum type. if enum_define: if key not in enum_define['enum_values']: raise QAPIExprError(expr_info, "Discriminator value '%s' is not found in " "enum '%s'" % (key, enum_define["enum_name"])) - if discriminator in enum_define['enum_values']: - raise QAPIExprError(expr_info, - "Discriminator name '%s' collides with " - "enum value in '%s'" % - (discriminator, enum_define["enum_name"])) - - # Otherwise, check for conflicts in the generated enum - else: - c_key = camel_to_upper(key) - if c_key in values: - raise QAPIExprError(expr_info, - "Union '%s' member '%s' clashes with '%s'" - % (name, key, values[c_key])) - values[c_key] = key def check_alternate(expr, expr_info): name = expr['alternate'] members = expr['data'] - values = {'MAX': '(automatic)'} types_seen = {} - # Check every branch + # Check every branch; require at least two branches + if len(members) < 2: + raise QAPIExprError(expr_info, + "Alternate '%s' should have at least two branches " + "in 'data'" % name) for (key, value) in members.items(): check_name(expr_info, "Member of alternate '%s'" % name, key) - # Check for conflicts in the generated enum - c_key = camel_to_upper(key) - if c_key in values: - raise QAPIExprError(expr_info, - "Alternate '%s' member '%s' clashes with '%s'" - % (name, key, values[c_key])) - values[c_key] = key - # Ensure alternates have no type conflicts. check_type(expr_info, "Member '%s' of alternate '%s'" % (key, name), value, allow_metas=['built-in', 'union', 'struct', 'enum']) qtype = find_alternate_member_qtype(value) - assert qtype + if not qtype: + raise QAPIExprError(expr_info, + "Alternate '%s' member '%s' cannot use " + "type '%s'" % (name, key, value)) if qtype in types_seen: raise QAPIExprError(expr_info, "Alternate '%s' member '%s' can't " @@ -667,7 +647,6 @@ def check_enum(expr, expr_info): name = expr['enum'] members = expr.get('data') prefix = expr.get('prefix') - values = {'MAX': '(automatic)'} if not isinstance(members, list): raise QAPIExprError(expr_info, @@ -678,12 +657,6 @@ def check_enum(expr, expr_info): for member in members: check_name(expr_info, "Member of enum '%s'" % name, member, enum_member=True) - key = camel_to_upper(member) - if key in values: - raise QAPIExprError(expr_info, - "Enum '%s' member '%s' clashes with '%s'" - % (name, member, values[key])) - values[key] = member def check_struct(expr, expr_info): @@ -694,8 +667,6 @@ def check_struct(expr, expr_info): allow_dict=True, allow_optional=True) check_type(expr_info, "'base' for struct '%s'" % name, expr.get('base'), allow_metas=['struct']) - if expr.get('base'): - check_member_clash(expr_info, expr['base'], expr['data']) def check_keys(expr_elem, meta, required, optional=[]): @@ -855,11 +826,18 @@ class QAPISchemaVisitor(object): class QAPISchemaType(QAPISchemaEntity): - def c_type(self, is_param=False): - return c_name(self.name) + pointer_suffix + # Return the C type for common use. + # For the types we commonly box, this is a pointer type. + def c_type(self): + pass - def c_null(self): - return 'NULL' + # Return the C type to be used in a parameter list. + def c_param_type(self): + return self.c_type() + + # Return the C type to be used where we suppress boxing. + def c_unboxed_type(self): + return self.c_type() def json_type(self): pass @@ -876,25 +854,24 @@ class QAPISchemaType(QAPISchemaEntity): class QAPISchemaBuiltinType(QAPISchemaType): - def __init__(self, name, json_type, c_type, c_null): + def __init__(self, name, json_type, c_type): QAPISchemaType.__init__(self, name, None) assert not c_type or isinstance(c_type, str) assert json_type in ('string', 'number', 'int', 'boolean', 'null', 'value') self._json_type_name = json_type self._c_type_name = c_type - self._c_null_val = c_null def c_name(self): return self.name - def c_type(self, is_param=False): - if is_param and self.name == 'str': - return 'const ' + self._c_type_name + def c_type(self): return self._c_type_name - def c_null(self): - return self._c_null_val + def c_param_type(self): + if self.name == 'str': + return 'const ' + self._c_type_name + return self._c_type_name def json_type(self): return self._json_type_name @@ -907,31 +884,33 @@ class QAPISchemaEnumType(QAPISchemaType): def __init__(self, name, info, values, prefix): QAPISchemaType.__init__(self, name, info) for v in values: - assert isinstance(v, str) + assert isinstance(v, QAPISchemaMember) + v.set_owner(name) assert prefix is None or isinstance(prefix, str) self.values = values self.prefix = prefix def check(self, schema): - assert len(set(self.values)) == len(self.values) + seen = {} + for v in self.values: + v.check_clash(self.info, seen) def is_implicit(self): # See QAPISchema._make_implicit_enum_type() return self.name.endswith('Kind') - def c_type(self, is_param=False): + def c_type(self): return c_name(self.name) - def c_null(self): - return c_enum_const(self.name, (self.values + ['MAX'])[0], - self.prefix) + def member_names(self): + return [v.name for v in self.values] def json_type(self): return 'string' def visit(self, visitor): visitor.visit_enum_type(self.name, self.info, - self.values, self.prefix) + self.member_names(), self.prefix) class QAPISchemaArrayType(QAPISchemaType): @@ -948,6 +927,9 @@ class QAPISchemaArrayType(QAPISchemaType): def is_implicit(self): return True + def c_type(self): + return c_name(self.name) + pointer_suffix + def json_type(self): return 'array' @@ -957,12 +939,17 @@ class QAPISchemaArrayType(QAPISchemaType): class QAPISchemaObjectType(QAPISchemaType): def __init__(self, name, info, base, local_members, variants): + # struct has local_members, optional base, and no variants + # flat union has base, variants, and no local_members + # simple union has local_members, variants, and no base QAPISchemaType.__init__(self, name, info) assert base is None or isinstance(base, str) for m in local_members: assert isinstance(m, QAPISchemaObjectTypeMember) - assert (variants is None or - isinstance(variants, QAPISchemaObjectTypeVariants)) + m.set_owner(name) + if variants is not None: + assert isinstance(variants, QAPISchemaObjectTypeVariants) + variants.set_owner(name) self._base_name = base self.base = None self.local_members = local_members @@ -970,39 +957,49 @@ class QAPISchemaObjectType(QAPISchemaType): self.members = None def check(self, schema): - assert self.members is not False # not running in cycles + if self.members is False: # check for cycles + raise QAPIExprError(self.info, + "Object %s contains itself" % self.name) if self.members: return self.members = False # mark as being checked + seen = OrderedDict() if self._base_name: self.base = schema.lookup_type(self._base_name) assert isinstance(self.base, QAPISchemaObjectType) - assert not self.base.variants # not implemented self.base.check(schema) - members = list(self.base.members) - else: - members = [] - seen = {} - for m in members: - assert c_name(m.name) not in seen - seen[m.name] = m + self.base.check_clash(schema, self.info, seen) for m in self.local_members: - m.check(schema, members, seen) + m.check(schema) + m.check_clash(self.info, seen) + self.members = seen.values() if self.variants: - self.variants.check(schema, members, seen) - self.members = members + self.variants.check(schema, seen) + assert self.variants.tag_member in self.members + self.variants.check_clash(schema, self.info, seen) + + # Check that the members of this type do not cause duplicate JSON members, + # and update seen to track the members seen so far. Report any errors + # on behalf of info, which is not necessarily self.info + def check_clash(self, schema, info, seen): + assert not self.variants # not implemented + for m in self.members: + m.check_clash(info, seen) def is_implicit(self): - # See QAPISchema._make_implicit_object_type() - return self.name[0] == ':' + # See QAPISchema._make_implicit_object_type(), as well as + # _def_predefineds() + return self.name.startswith('q_') def c_name(self): - assert not self.is_implicit() return QAPISchemaType.c_name(self) - def c_type(self, is_param=False): + def c_type(self): assert not self.is_implicit() - return QAPISchemaType.c_type(self) + return c_name(self.name) + pointer_suffix + + def c_unboxed_type(self): + return c_name(self.name) def json_type(self): return 'object' @@ -1014,22 +1011,65 @@ class QAPISchemaObjectType(QAPISchemaType): self.members, self.variants) -class QAPISchemaObjectTypeMember(object): - def __init__(self, name, typ, optional): +class QAPISchemaMember(object): + role = 'member' + + def __init__(self, name): assert isinstance(name, str) + self.name = name + self.owner = None + + def set_owner(self, name): + assert not self.owner + self.owner = name + + def check_clash(self, info, seen): + cname = c_name(self.name) + if cname.lower() != cname and self.owner not in case_whitelist: + raise QAPIExprError(info, + "%s should not use uppercase" % self.describe()) + if cname in seen: + raise QAPIExprError(info, + "%s collides with %s" + % (self.describe(), seen[cname].describe())) + seen[cname] = self + + def _pretty_owner(self): + owner = self.owner + if owner.startswith('q_obj_'): + # See QAPISchema._make_implicit_object_type() - reverse the + # mapping there to create a nice human-readable description + owner = owner[6:] + if owner.endswith('-arg'): + return '(parameter of %s)' % owner[:-4] + elif owner.endswith('-base'): + return '(base of %s)' % owner[:-5] + else: + assert owner.endswith('-wrapper') + # Unreachable and not implemented + assert False + if owner.endswith('Kind'): + # See QAPISchema._make_implicit_enum_type() + return '(branch of %s)' % owner[:-4] + return '(%s of %s)' % (self.role, owner) + + def describe(self): + return "'%s' %s" % (self.name, self._pretty_owner()) + + +class QAPISchemaObjectTypeMember(QAPISchemaMember): + def __init__(self, name, typ, optional): + QAPISchemaMember.__init__(self, name) assert isinstance(typ, str) assert isinstance(optional, bool) - self.name = name self._type_name = typ self.type = None self.optional = optional - def check(self, schema, all_members, seen): - assert self.name not in seen + def check(self, schema): + assert self.owner self.type = schema.lookup_type(self._type_name) assert self.type - all_members.append(self) - seen[self.name] = self class QAPISchemaObjectTypeVariants(object): @@ -1041,51 +1081,68 @@ class QAPISchemaObjectTypeVariants(object): assert bool(tag_member) != bool(tag_name) assert (isinstance(tag_name, str) or isinstance(tag_member, QAPISchemaObjectTypeMember)) + assert len(variants) > 0 for v in variants: assert isinstance(v, QAPISchemaObjectTypeVariant) self.tag_name = tag_name self.tag_member = tag_member self.variants = variants - def check(self, schema, members, seen): - if self.tag_name: - self.tag_member = seen[self.tag_name] - else: - self.tag_member.check(schema, members, seen) + def set_owner(self, name): + for v in self.variants: + v.set_owner(name) + + def check(self, schema, seen): + if not self.tag_member: # flat union + self.tag_member = seen[c_name(self.tag_name)] + assert self.tag_name == self.tag_member.name assert isinstance(self.tag_member.type, QAPISchemaEnumType) for v in self.variants: - vseen = dict(seen) - v.check(schema, self.tag_member.type, vseen) + v.check(schema) + # Union names must match enum values; alternate names are + # checked separately. Use 'seen' to tell the two apart. + if seen: + assert v.name in self.tag_member.type.member_names() + assert isinstance(v.type, QAPISchemaObjectType) + v.type.check(schema) + + def check_clash(self, schema, info, seen): + for v in self.variants: + # Reset seen map for each variant, since qapi names from one + # branch do not affect another branch + assert isinstance(v.type, QAPISchemaObjectType) + v.type.check_clash(schema, info, dict(seen)) class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember): + role = 'branch' + def __init__(self, name, typ): QAPISchemaObjectTypeMember.__init__(self, name, typ, False) - def check(self, schema, tag_type, seen): - QAPISchemaObjectTypeMember.check(self, schema, [], seen) - assert self.name in tag_type.values - - # This function exists to support ugly simple union special cases - # TODO get rid of them, and drop the function - def simple_union_type(self): - if (self.type.is_implicit() and - isinstance(self.type, QAPISchemaObjectType)): - assert len(self.type.members) == 1 - assert not self.type.variants - return self.type.members[0].type - return None - class QAPISchemaAlternateType(QAPISchemaType): def __init__(self, name, info, variants): QAPISchemaType.__init__(self, name, info) assert isinstance(variants, QAPISchemaObjectTypeVariants) assert not variants.tag_name + variants.set_owner(name) + variants.tag_member.set_owner(self.name) self.variants = variants def check(self, schema): - self.variants.check(schema, [], {}) + self.variants.tag_member.check(schema) + # Not calling self.variants.check_clash(), because there's nothing + # to clash with + self.variants.check(schema, {}) + # Alternate branch names have no relation to the tag enum values; + # so we have to check for potential name collisions ourselves. + seen = {} + for v in self.variants.variants: + v.check_clash(self.info, seen) + + def c_type(self): + return c_name(self.name) + pointer_suffix def json_type(self): return 'value' @@ -1148,7 +1205,7 @@ class QAPISchema(object): self._predefining = False self._def_exprs() self.check() - except (QAPISchemaError, QAPIExprError), err: + except (QAPISchemaError, QAPIExprError) as err: print >>sys.stderr, err exit(1) @@ -1167,9 +1224,8 @@ class QAPISchema(object): def lookup_type(self, name): return self.lookup_entity(name, QAPISchemaType) - def _def_builtin_type(self, name, json_type, c_type, c_null): - self._def_entity(QAPISchemaBuiltinType(name, json_type, - c_type, c_null)) + def _def_builtin_type(self, name, json_type, c_type): + self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type)) # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple # qapi-types.h from a single .c, all arrays of builtins must be # declared in the first file whether or not they are used. Nicer @@ -1178,28 +1234,38 @@ class QAPISchema(object): self._make_array_type(name, None) def _def_predefineds(self): - for t in [('str', 'string', 'char' + pointer_suffix, 'NULL'), - ('number', 'number', 'double', '0'), - ('int', 'int', 'int64_t', '0'), - ('int8', 'int', 'int8_t', '0'), - ('int16', 'int', 'int16_t', '0'), - ('int32', 'int', 'int32_t', '0'), - ('int64', 'int', 'int64_t', '0'), - ('uint8', 'int', 'uint8_t', '0'), - ('uint16', 'int', 'uint16_t', '0'), - ('uint32', 'int', 'uint32_t', '0'), - ('uint64', 'int', 'uint64_t', '0'), - ('size', 'int', 'uint64_t', '0'), - ('bool', 'boolean', 'bool', 'false'), - ('any', 'value', 'QObject' + pointer_suffix, 'NULL')]: + for t in [('str', 'string', 'char' + pointer_suffix), + ('number', 'number', 'double'), + ('int', 'int', 'int64_t'), + ('int8', 'int', 'int8_t'), + ('int16', 'int', 'int16_t'), + ('int32', 'int', 'int32_t'), + ('int64', 'int', 'int64_t'), + ('uint8', 'int', 'uint8_t'), + ('uint16', 'int', 'uint16_t'), + ('uint32', 'int', 'uint32_t'), + ('uint64', 'int', 'uint64_t'), + ('size', 'int', 'uint64_t'), + ('bool', 'boolean', 'bool'), + ('any', 'value', 'QObject' + pointer_suffix)]: self._def_builtin_type(*t) - self.the_empty_object_type = QAPISchemaObjectType(':empty', None, None, - [], None) + self.the_empty_object_type = QAPISchemaObjectType('q_empty', None, + None, [], None) self._def_entity(self.the_empty_object_type) + qtype_values = self._make_enum_members(['none', 'qnull', 'qint', + 'qstring', 'qdict', 'qlist', + 'qfloat', 'qbool']) + self._def_entity(QAPISchemaEnumType('QType', None, qtype_values, + 'QTYPE')) + + def _make_enum_members(self, values): + return [QAPISchemaMember(v) for v in values] def _make_implicit_enum_type(self, name, info, values): + # See also QAPISchemaObjectTypeMember._pretty_owner() name = name + 'Kind' # Use namespace reserved by add_name() - self._def_entity(QAPISchemaEnumType(name, info, values, None)) + self._def_entity(QAPISchemaEnumType( + name, info, self._make_enum_members(values), None)) return name def _make_array_type(self, element_type, info): @@ -1211,7 +1277,8 @@ class QAPISchema(object): def _make_implicit_object_type(self, name, info, role, members): if not members: return None - name = ':obj-%s-%s' % (name, role) + # See also QAPISchemaObjectTypeMember._pretty_owner() + name = 'q_obj_%s-%s' % (name, role) if not self.lookup_entity(name, QAPISchemaObjectType): self._def_entity(QAPISchemaObjectType(name, info, None, members, None)) @@ -1221,7 +1288,8 @@ class QAPISchema(object): name = expr['enum'] data = expr['data'] prefix = expr.get('prefix') - self._def_entity(QAPISchemaEnumType(name, info, data, prefix)) + self._def_entity(QAPISchemaEnumType( + name, info, self._make_enum_members(data), prefix)) def _make_member(self, name, typ, info): optional = False @@ -1256,27 +1324,28 @@ class QAPISchema(object): typ, info, 'wrapper', [self._make_member('data', typ, info)]) return QAPISchemaObjectTypeVariant(case, typ) - def _make_implicit_tag(self, type_name, info, variants): - typ = self._make_implicit_enum_type(type_name, info, - [v.name for v in variants]) - return QAPISchemaObjectTypeMember('type', typ, False) - def _def_union_type(self, expr, info): name = expr['union'] data = expr['data'] base = expr.get('base') tag_name = expr.get('discriminator') tag_member = None + if isinstance(base, dict): + base = (self._make_implicit_object_type( + name, info, 'base', self._make_members(base, info))) if tag_name: variants = [self._make_variant(key, value) for (key, value) in data.iteritems()] + members = [] else: variants = [self._make_simple_variant(key, value, info) for (key, value) in data.iteritems()] - tag_member = self._make_implicit_tag(name, info, variants) + typ = self._make_implicit_enum_type(name, info, + [v.name for v in variants]) + tag_member = QAPISchemaObjectTypeMember('type', typ, False) + members = [tag_member] self._def_entity( - QAPISchemaObjectType(name, info, base, - self._make_members(OrderedDict(), info), + QAPISchemaObjectType(name, info, base, members, QAPISchemaObjectTypeVariants(tag_name, tag_member, variants))) @@ -1286,7 +1355,7 @@ class QAPISchema(object): data = expr['data'] variants = [self._make_variant(key, value) for (key, value) in data.iteritems()] - tag_member = self._make_implicit_tag(name, info, variants) + tag_member = QAPISchemaObjectTypeMember('type', 'QType', False) self._def_entity( QAPISchemaAlternateType(name, info, QAPISchemaObjectTypeVariants(None, @@ -1390,7 +1459,7 @@ def camel_to_upper(value): def c_enum_const(type_name, const_name, prefix=None): if prefix is not None: type_name = prefix - return camel_to_upper(type_name + '_' + const_name) + return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper() c_name_trans = string.maketrans('.-', '__') @@ -1431,11 +1500,12 @@ def c_name(name, protect=True): 'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not', 'not_eq', 'or', 'or_eq', 'xor', 'xor_eq']) # namespace pollution: - polluted_words = set(['unix', 'errno']) + polluted_words = set(['unix', 'errno', 'mips', 'sparc']) + name = name.translate(c_name_trans) if protect and (name in c89_words | c99_words | c11_words | gcc_words | cpp_words | polluted_words): return "q_" + name - return name.translate(c_name_trans) + return name eatspace = '\033EATSPACE.' pointer_suffix = ' *' + eatspace @@ -1515,7 +1585,7 @@ const char *const %(c_name)s_lookup[] = { ''', index=index, value=value) - max_index = c_enum_const(name, 'MAX', prefix) + max_index = c_enum_const(name, '_MAX', prefix) ret += mcgen(''' [%(max_index)s] = NULL, }; @@ -1526,7 +1596,7 @@ const char *const %(c_name)s_lookup[] = { def gen_enum(name, values, prefix=None): # append automatically generated _MAX value - enum_values = values + ['MAX'] + enum_values = values + ['_MAX'] ret = mcgen(''' @@ -1567,64 +1637,18 @@ def gen_params(arg_type, extra): sep = ', ' if memb.optional: ret += 'bool has_%s, ' % c_name(memb.name) - ret += '%s %s' % (memb.type.c_type(is_param=True), c_name(memb.name)) + ret += '%s %s' % (memb.type.c_param_type(), c_name(memb.name)) if extra: ret += sep + extra return ret -def gen_err_check(label='out', skiperr=False): - if skiperr: - return '' +def gen_err_check(): return mcgen(''' if (err) { - goto %(label)s; - } -''', - label=label) - - -def gen_visit_fields(members, prefix='', need_cast=False, skiperr=False): - ret = '' - if skiperr: - errparg = 'NULL' - else: - errparg = '&err' - - for memb in members: - if memb.optional: - ret += mcgen(''' - visit_optional(v, &%(prefix)shas_%(c_name)s, "%(name)s", %(errp)s); -''', - prefix=prefix, c_name=c_name(memb.name), - name=memb.name, errp=errparg) - ret += gen_err_check(skiperr=skiperr) - ret += mcgen(''' - if (%(prefix)shas_%(c_name)s) { -''', - prefix=prefix, c_name=c_name(memb.name)) - push_indent() - - # Ugly: sometimes we need to cast away const - if need_cast and memb.type.name == 'str': - cast = '(char **)' - else: - cast = '' - - ret += mcgen(''' - visit_type_%(c_type)s(v, %(cast)s&%(prefix)s%(c_name)s, "%(name)s", %(errp)s); -''', - c_type=memb.type.c_name(), prefix=prefix, cast=cast, - c_name=c_name(memb.name), name=memb.name, - errp=errparg) - ret += gen_err_check(skiperr=skiperr) - - if memb.optional: - pop_indent() - ret += mcgen(''' + goto out; } ''') - return ret # @@ -1639,7 +1663,7 @@ def parse_command_line(extra_options="", extra_long_options=[]): "chp:o:" + extra_options, ["source", "header", "prefix=", "output-dir="] + extra_long_options) - except getopt.GetoptError, err: + except getopt.GetoptError as err: print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err)) sys.exit(1) @@ -1693,7 +1717,7 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file, if output_dir: try: os.makedirs(output_dir) - except os.error, e: + except os.error as e: if e.errno != errno.EEXIST: raise diff --git a/scripts/qemugdb/mtree.py b/scripts/qemugdb/mtree.py index 06011c30c..cc8131c2e 100644 --- a/scripts/qemugdb/mtree.py +++ b/scripts/qemugdb/mtree.py @@ -21,7 +21,7 @@ def isnull(ptr): return ptr == gdb.Value(0).cast(ptr.type) def int128(p): - return long(p['lo']) + (long(p['hi']) << 64) + return int(p['lo']) + (int(p['hi']) << 64) class MtreeCommand(gdb.Command): '''Display the memory tree hierarchy''' @@ -40,11 +40,11 @@ class MtreeCommand(gdb.Command): def process_queue(self): while self.queue: ptr = self.queue.pop(0) - if long(ptr) in self.seen: + if int(ptr) in self.seen: continue self.print_item(ptr) def print_item(self, ptr, offset = gdb.Value(0), level = 0): - self.seen.add(long(ptr)) + self.seen.add(int(ptr)) addr = ptr['addr'] addr += offset size = int128(ptr['size']) @@ -58,8 +58,8 @@ class MtreeCommand(gdb.Command): klass = ' (RAM)' gdb.write('%s%016x-%016x %s%s (@ %s)\n' % (' ' * level, - long(addr), - long(addr + (size - 1)), + int(addr), + int(addr + (size - 1)), ptr['name'].string(), klass, ptr, diff --git a/scripts/qmp/qemu-ga-client b/scripts/qmp/qemu-ga-client index 9908f2109..fd056056f 100755 --- a/scripts/qmp/qemu-ga-client +++ b/scripts/qmp/qemu-ga-client @@ -259,7 +259,7 @@ def main(address, cmd, args): try: client = QemuGuestAgentClient(address) - except QemuGuestAgent.error, e: + except QemuGuestAgent.error as e: import errno print(e) diff --git a/scripts/qmp/qmp b/scripts/qmp/qmp index 1db3c7ffe..514b539a6 100755 --- a/scripts/qmp/qmp +++ b/scripts/qmp/qmp @@ -91,8 +91,8 @@ def main(args): try: os.environ['QMP_PATH'] = path os.execvp(fullcmd, [fullcmd] + args) - except OSError, (errno, msg): - if errno == 2: + except OSError as exc: + if exc.errno == 2: print 'Command "%s" not found.' % (fullcmd) return 1 raise diff --git a/scripts/qmp/qmp-shell b/scripts/qmp/qmp-shell index fa39bf0d7..0373b24b2 100755 --- a/scripts/qmp/qmp-shell +++ b/scripts/qmp/qmp-shell @@ -70,7 +70,6 @@ import json import ast import readline import sys -import pprint class QMPCompleter(list): def complete(self, text, state): @@ -103,11 +102,11 @@ class FuzzyJSON(ast.NodeTransformer): # TODO: QMPShell's interface is a bit ugly (eg. _fill_completion() and # _execute_cmd()). Let's design a better one. class QMPShell(qmp.QEMUMonitorProtocol): - def __init__(self, address, pp=None): + def __init__(self, address, pretty=False): qmp.QEMUMonitorProtocol.__init__(self, self.__get_address(address)) self._greeting = None self._completer = None - self._pp = pp + self._pretty = pretty self._transmode = False self._actions = list() @@ -231,16 +230,16 @@ class QMPShell(qmp.QEMUMonitorProtocol): return qmpcmd def _print(self, qmp): - jsobj = json.dumps(qmp) - if self._pp is not None: - self._pp.pprint(jsobj) - else: - print str(jsobj) + indent = None + if self._pretty: + indent = 4 + jsobj = json.dumps(qmp, indent=indent) + print str(jsobj) def _execute_cmd(self, cmdline): try: qmpcmd = self.__build_cmd(cmdline) - except Exception, e: + except Exception as e: print 'Error while parsing command line: %s' % e print 'command format: <command-name> ', print '[arg-name1=arg1] ... [arg-nameN=argN]' @@ -377,7 +376,7 @@ def main(): addr = '' qemu = None hmp = False - pp = None + pretty = False verbose = False try: @@ -387,9 +386,7 @@ def main(): fail_cmdline(arg) hmp = True elif arg == "-p": - if pp is not None: - fail_cmdline(arg) - pp = pprint.PrettyPrinter(indent=4) + pretty = True elif arg == "-v": verbose = True else: @@ -398,7 +395,7 @@ def main(): if hmp: qemu = HMPShell(arg) else: - qemu = QMPShell(arg, pp) + qemu = QMPShell(arg, pretty) addr = arg if qemu is None: diff --git a/scripts/qmp/qmp.py b/scripts/qmp/qmp.py index 1d38e3e9e..779332f32 100644 --- a/scripts/qmp/qmp.py +++ b/scripts/qmp/qmp.py @@ -92,7 +92,7 @@ class QEMUMonitorProtocol: self.__sock.setblocking(0) try: self.__json_read() - except socket.error, err: + except socket.error as err: if err[0] == errno.EAGAIN: # No data available pass @@ -150,7 +150,7 @@ class QEMUMonitorProtocol: """ try: self.__sock.sendall(json.dumps(qmp_cmd)) - except socket.error, err: + except socket.error as err: if err[0] == errno.EPIPE: return raise socket.error(err) diff --git a/scripts/tracetool.py b/scripts/tracetool.py index 83bde7bda..7b82959e8 100755 --- a/scripts/tracetool.py +++ b/scripts/tracetool.py @@ -71,7 +71,7 @@ def main(args): try: opts, args = getopt.getopt(args[1:], "", long_opts) - except getopt.GetoptError, err: + except getopt.GetoptError as err: error_opt(str(err)) check_backends = False @@ -132,7 +132,7 @@ def main(args): try: tracetool.generate(sys.stdin, arg_format, arg_backends, binary=binary, probe_prefix=probe_prefix) - except tracetool.TracetoolError, e: + except tracetool.TracetoolError as e: error_opt(str(e)) if __name__ == "__main__": diff --git a/scripts/tracetool/__init__.py b/scripts/tracetool/__init__.py index 181675f00..be24039c5 100644 --- a/scripts/tracetool/__init__.py +++ b/scripts/tracetool/__init__.py @@ -6,7 +6,7 @@ Machinery for generating tracing-related intermediate files. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -50,9 +50,14 @@ class Arguments: Parameters ---------- args : - List of (type, name) tuples. + List of (type, name) tuples or Arguments objects. """ - self._args = args + self._args = [] + for arg in args: + if isinstance(arg, Arguments): + self._args.extend(arg._args) + else: + self._args.append(arg) def copy(self): """Create a new copy.""" @@ -83,6 +88,12 @@ class Arguments: res.append((arg_type, identifier)) return Arguments(res) + def __getitem__(self, index): + if isinstance(index, slice): + return Arguments(self._args[index]) + else: + return self._args[index] + def __iter__(self): """Iterate over the (type, name) pairs.""" return iter(self._args) @@ -110,6 +121,10 @@ class Arguments: """List of argument types.""" return [ type_ for type_, _ in self._args ] + def casted(self): + """List of argument names casted to their type.""" + return ["(%s)%s" % (type_, name) for type_, name in self._args] + def transform(self, *trans): """Return a new Arguments instance with transformed types. @@ -146,9 +161,10 @@ class Event(object): "(?:(?:(?P<fmt_trans>\".+),)?\s*(?P<fmt>\".+))?" "\s*") - _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec"]) + _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec", "vcpu"]) - def __init__(self, name, props, fmt, args, orig=None): + def __init__(self, name, props, fmt, args, orig=None, + event_trans=None, event_exec=None): """ Parameters ---------- @@ -161,13 +177,19 @@ class Event(object): args : Arguments Event arguments. orig : Event or None - Original Event before transformation. + Original Event before transformation/generation. + event_trans : Event or None + Generated translation-time event ("tcg" property). + event_exec : Event or None + Generated execution-time event ("tcg" property). """ self.name = name self.properties = props self.fmt = fmt self.args = args + self.event_trans = event_trans + self.event_exec = event_exec if orig is None: self.original = weakref.ref(self) @@ -183,7 +205,7 @@ class Event(object): def copy(self): """Create a new copy.""" return Event(self.name, list(self.properties), self.fmt, - self.args.copy(), self) + self.args.copy(), self, self.event_trans, self.event_exec) @staticmethod def build(line_str): @@ -215,7 +237,13 @@ class Event(object): if "tcg" in props and isinstance(fmt, str): raise ValueError("Events with 'tcg' property must have two formats") - return Event(name, props, fmt, args) + event = Event(name, props, fmt, args) + + # add implicit arguments when using the 'vcpu' property + import tracetool.vcpu + event = tracetool.vcpu.transform_event(event) + + return event def __repr__(self): """Evaluable string representation for this object.""" @@ -270,6 +298,7 @@ def _read_events(fobj): event_trans.name += "_trans" event_trans.properties += ["tcg-trans"] event_trans.fmt = event.fmt[0] + # ignore TCG arguments args_trans = [] for atrans, aorig in zip( event_trans.transform(tracetool.transform.TCG_2_HOST).args, @@ -277,13 +306,12 @@ def _read_events(fobj): if atrans == aorig: args_trans.append(atrans) event_trans.args = Arguments(args_trans) - event_trans = event_trans.copy() event_exec = event.copy() event_exec.name += "_exec" event_exec.properties += ["tcg-exec"] event_exec.fmt = event.fmt[1] - event_exec = event_exec.transform(tracetool.transform.TCG_2_HOST) + event_exec.args = event_exec.args.transform(tracetool.transform.TCG_2_HOST) new_event = [event_trans, event_exec] event.event_trans, event.event_exec = new_event diff --git a/scripts/tracetool/backend/stderr.py b/scripts/tracetool/backend/log.py index ca5805462..e409b7326 100644 --- a/scripts/tracetool/backend/stderr.py +++ b/scripts/tracetool/backend/log.py @@ -20,11 +20,8 @@ PUBLIC = True def generate_h_begin(events): - out('#include <stdio.h>', - '#include <sys/time.h>', - '#include <sys/types.h>', - '#include <unistd.h>', - '#include "trace/control.h"', + out('#include "trace/control.h"', + '#include "qemu/log.h"', '') @@ -36,10 +33,10 @@ def generate_h(event): out(' if (trace_event_get_state(%(event_id)s)) {', ' struct timeval _now;', ' gettimeofday(&_now, NULL);', - ' fprintf(stderr, "%%d@%%zd.%%06zd:%(name)s " %(fmt)s "\\n",', - ' getpid(),', - ' (size_t)_now.tv_sec, (size_t)_now.tv_usec', - ' %(argnames)s);', + ' qemu_log_mask(LOG_TRACE, "%%d@%%zd.%%06zd:%(name)s " %(fmt)s "\\n",', + ' getpid(),', + ' (size_t)_now.tv_sec, (size_t)_now.tv_usec', + ' %(argnames)s);', ' }', event_id="TRACE_" + event.name.upper(), name=event.name, diff --git a/scripts/tracetool/backend/simple.py b/scripts/tracetool/backend/simple.py index e8c2cd57e..3246c2001 100644 --- a/scripts/tracetool/backend/simple.py +++ b/scripts/tracetool/backend/simple.py @@ -42,7 +42,8 @@ def generate_h(event): def generate_c_begin(events): - out('#include "trace.h"', + out('#include "qemu/osdep.h"', + '#include "trace.h"', '#include "trace/control.h"', '#include "trace/simple.h"', '') diff --git a/scripts/tracetool/format/events_c.py b/scripts/tracetool/format/events_c.py index 2d97fa310..1cc6a49a7 100644 --- a/scripts/tracetool/format/events_c.py +++ b/scripts/tracetool/format/events_c.py @@ -19,6 +19,7 @@ from tracetool import out def generate(events, backend): out('/* This file is autogenerated by tracetool, do not edit. */', '', + '#include "qemu/osdep.h"', '#include "trace.h"', '#include "trace/generated-events.h"', '#include "trace/control.h"', @@ -27,7 +28,7 @@ def generate(events, backend): out('TraceEvent trace_events[TRACE_EVENT_COUNT] = {') for e in events: - out(' { .id = %(id)s, .name = \"%(name)s\", .sstate = %(sstate)s, .dstate = 0 },', + out(' { .id = %(id)s, .name = \"%(name)s\", .sstate = %(sstate)s },', id = "TRACE_" + e.name.upper(), name = e.name, sstate = "TRACE_%s_ENABLED" % e.name.upper()) diff --git a/scripts/tracetool/format/events_h.py b/scripts/tracetool/format/events_h.py index 9f114a349..4529263e0 100644 --- a/scripts/tracetool/format/events_h.py +++ b/scripts/tracetool/format/events_h.py @@ -6,7 +6,7 @@ trace/generated-events.h """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -21,8 +21,6 @@ def generate(events, backend): '', '#ifndef TRACE__GENERATED_EVENTS_H', '#define TRACE__GENERATED_EVENTS_H', - '', - '#include <stdbool.h>', '') # event identifiers @@ -43,7 +41,7 @@ def generate(events, backend): if "tcg-trans" in e.properties: # a single define for the two "sub-events" out('#define TRACE_%(name)s_ENABLED %(enabled)d', - name=e.original.original.name.upper(), + name=e.original.name.upper(), enabled=enabled) out('#define TRACE_%s_ENABLED %d' % (e.name.upper(), enabled)) diff --git a/scripts/tracetool/format/h.py b/scripts/tracetool/format/h.py index 9b3943002..083540621 100644 --- a/scripts/tracetool/format/h.py +++ b/scripts/tracetool/format/h.py @@ -6,7 +6,7 @@ trace/generated-tracers.h """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" diff --git a/scripts/tracetool/format/tcg_h.py b/scripts/tracetool/format/tcg_h.py index f676b6662..e2331f251 100644 --- a/scripts/tracetool/format/tcg_h.py +++ b/scripts/tracetool/format/tcg_h.py @@ -6,14 +6,25 @@ Generate .h file for TCG code generation. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@linux.vnet.ibm.com" -from tracetool import out +from tracetool import out, Arguments +import tracetool.vcpu + + +def vcpu_transform_args(args): + assert len(args) == 1 + return Arguments([ + args, + # NOTE: this name must be kept in sync with the one in "tcg_h" + # NOTE: Current helper code uses TCGv_env (CPUArchState*) + ("TCGv_env", "__tcg_" + args.names()[0]), + ]) def generate(events, backend): @@ -23,8 +34,6 @@ def generate(events, backend): '#ifndef TRACE__GENERATED_TCG_TRACERS_H', '#define TRACE__GENERATED_TCG_TRACERS_H', '', - '#include <stdint.h>', - '', '#include "trace.h"', '#include "exec/helper-proto.h"', '', @@ -35,21 +44,21 @@ def generate(events, backend): if "tcg-trans" not in e.properties: continue - # get the original event definition - e = e.original.original - out('static inline void %(name_tcg)s(%(args)s)', '{', - name_tcg=e.api(e.QEMU_TRACE_TCG), - args=e.args) + name_tcg=e.original.api(e.QEMU_TRACE_TCG), + args=tracetool.vcpu.transform_args("tcg_h", e.original)) if "disable" not in e.properties: + args_trans = e.original.event_trans.args + args_exec = tracetool.vcpu.transform_args( + "tcg_helper_c", e.original.event_exec, "wrapper") out(' %(name_trans)s(%(argnames_trans)s);', ' gen_helper_%(name_exec)s(%(argnames_exec)s);', - name_trans=e.event_trans.api(e.QEMU_TRACE), - name_exec=e.event_exec.api(e.QEMU_TRACE), - argnames_trans=", ".join(e.event_trans.args.names()), - argnames_exec=", ".join(e.event_exec.args.names())) + name_trans=e.original.event_trans.api(e.QEMU_TRACE), + name_exec=e.original.event_exec.api(e.QEMU_TRACE), + argnames_trans=", ".join(args_trans.names()), + argnames_exec=", ".join(args_exec.names())) out('}') diff --git a/scripts/tracetool/format/tcg_helper_c.py b/scripts/tracetool/format/tcg_helper_c.py index 96655a059..a089b0bf0 100644 --- a/scripts/tracetool/format/tcg_helper_c.py +++ b/scripts/tracetool/format/tcg_helper_c.py @@ -6,15 +6,38 @@ Generate trace/generated-helpers.c. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@linux.vnet.ibm.com" -from tracetool import out +from tracetool import Arguments, out from tracetool.transform import * +import tracetool.vcpu + + +def vcpu_transform_args(args, mode): + assert len(args) == 1 + # NOTE: this name must be kept in sync with the one in "tcg_h" + args = Arguments([(args.types()[0], "__tcg_" + args.names()[0])]) + if mode == "code": + return Arguments([ + # Does cast from helper requirements to tracing types + ("CPUState *", "ENV_GET_CPU(%s)" % args.names()[0]), + ]) + else: + args = Arguments([ + # NOTE: Current helper code uses TCGv_env (CPUArchState*) + ("CPUArchState *", args.names()[0]), + ]) + if mode == "header": + return args + elif mode == "wrapper": + return args.transform(HOST_2_TCG) + else: + assert False def generate(events, backend): @@ -23,6 +46,7 @@ def generate(events, backend): out('/* This file is autogenerated by tracetool, do not edit. */', '', + '#include "qemu/osdep.h"', '#include "qemu-common.h"', '#include "trace.h"', '#include "exec/helper-proto.h"', @@ -33,18 +57,18 @@ def generate(events, backend): if "tcg-exec" not in e.properties: continue - # tracetool.generate always transforms types to host - e_args = e.original.args - - values = ["(%s)%s" % (t, n) - for t, n in e.args.transform(TCG_2_TCG_HELPER_DEF)] + e_args_api = tracetool.vcpu.transform_args( + "tcg_helper_c", e.original, "header").transform( + HOST_2_TCG_COMPAT, TCG_2_TCG_HELPER_DEF) + e_args_call = tracetool.vcpu.transform_args( + "tcg_helper_c", e, "code") - out('void %(name_tcg)s(%(args)s)', + out('void %(name_tcg)s(%(args_api)s)', '{', - ' %(name)s(%(values)s);', + ' %(name)s(%(args_call)s);', '}', name_tcg="helper_%s_proxy" % e.api(), name=e.api(), - args=e_args.transform(HOST_2_TCG_COMPAT, TCG_2_TCG_HELPER_DEF), - values=", ".join(values), + args_api=e_args_api, + args_call=", ".join(e_args_call.casted()), ) diff --git a/scripts/tracetool/format/tcg_helper_h.py b/scripts/tracetool/format/tcg_helper_h.py index a8ba7ba8e..dc76c15eb 100644 --- a/scripts/tracetool/format/tcg_helper_h.py +++ b/scripts/tracetool/format/tcg_helper_h.py @@ -6,7 +6,7 @@ Generate trace/generated-helpers.h. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -15,6 +15,7 @@ __email__ = "stefanha@linux.vnet.ibm.com" from tracetool import out from tracetool.transform import * +import tracetool.vcpu def generate(events, backend): @@ -29,11 +30,9 @@ def generate(events, backend): if "tcg-exec" not in e.properties: continue - # tracetool.generate always transforms types to host - e_args = e.original.args - # TCG helper proxy declaration fmt = "DEF_HELPER_FLAGS_%(argc)d(%(name)s, %(flags)svoid%(types)s)" + e_args = tracetool.vcpu.transform_args("tcg_helper_c", e.original, "header") args = e_args.transform(HOST_2_TCG_COMPAT, HOST_2_TCG, TCG_2_TCG_HELPER_DECL) types = ", ".join(args.types()) diff --git a/scripts/tracetool/format/tcg_helper_wrapper_h.py b/scripts/tracetool/format/tcg_helper_wrapper_h.py index cac5a878f..020f4422a 100644 --- a/scripts/tracetool/format/tcg_helper_wrapper_h.py +++ b/scripts/tracetool/format/tcg_helper_wrapper_h.py @@ -6,7 +6,7 @@ Generate trace/generated-helpers-wrappers.h. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -15,6 +15,7 @@ __email__ = "stefanha@linux.vnet.ibm.com" from tracetool import out from tracetool.transform import * +import tracetool.vcpu def generate(events, backend): @@ -33,7 +34,7 @@ def generate(events, backend): continue # tracetool.generate always transforms types to host - e_args = e.original.args + e_args = tracetool.vcpu.transform_args("tcg_helper_c", e.original, "wrapper") # mixed-type to TCG helper bridge args_tcg_compat = e_args.transform(HOST_2_TCG_COMPAT) diff --git a/scripts/tracetool/format/ust_events_c.py b/scripts/tracetool/format/ust_events_c.py index bc970936b..9967c7a82 100644 --- a/scripts/tracetool/format/ust_events_c.py +++ b/scripts/tracetool/format/ust_events_c.py @@ -22,6 +22,8 @@ def generate(events, backend): out('/* This file is autogenerated by tracetool, do not edit. */', '', + '#include "qemu/osdep.h"', + '', '#define TRACEPOINT_DEFINE', '#define TRACEPOINT_CREATE_PROBES', '', diff --git a/scripts/tracetool/transform.py b/scripts/tracetool/transform.py index fc5e679ed..e18b05315 100644 --- a/scripts/tracetool/transform.py +++ b/scripts/tracetool/transform.py @@ -6,7 +6,7 @@ Type-transformation rules. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -98,6 +98,7 @@ HOST_2_TCG = { "uint32_t": "TCGv_i32", "uint64_t": "TCGv_i64", "void *" : "TCGv_ptr", + "CPUArchState *": "TCGv_env", None: _host_2_tcg, } @@ -130,6 +131,7 @@ TCG_2_TCG_HELPER_DECL = { "TCGv_ptr": "ptr", "TCGv_i32": "i32", "TCGv_i64": "i64", + "TCGv_env": "env", None: _tcg_2_tcg_helper_decl_error, } diff --git a/scripts/tracetool/vcpu.py b/scripts/tracetool/vcpu.py new file mode 100644 index 000000000..452c7f589 --- /dev/null +++ b/scripts/tracetool/vcpu.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Generic management for the 'vcpu' property. + +""" + +__author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2016, Lluís Vilanova <vilanova@ac.upc.edu>" +__license__ = "GPL version 2 or (at your option) any later version" + +__maintainer__ = "Stefan Hajnoczi" +__email__ = "stefanha@linux.vnet.ibm.com" + + +from tracetool import Arguments, try_import + + +def transform_event(event): + """Transform event to comply with the 'vcpu' property (if present).""" + if "vcpu" in event.properties: + # events with 'tcg-trans' and 'tcg-exec' are auto-generated from + # already-patched events + assert "tcg-trans" not in event.properties + assert "tcg-exec" not in event.properties + + event.args = Arguments([("CPUState *", "__cpu"), event.args]) + if "tcg" in event.properties: + fmt = "\"cpu=%p \"" + event.fmt = [fmt + event.fmt[0], + fmt + event.fmt[1]] + else: + fmt = "\"cpu=%p \"" + event.fmt = fmt + event.fmt + return event + + +def transform_args(format, event, *args, **kwargs): + """Transforms the arguments to suit the specified format. + + The format module must implement function 'vcpu_args', which receives the + implicit arguments added by the 'vcpu' property, and must return suitable + arguments for the given format. + + The function is only called for events with the 'vcpu' property. + + Parameters + ========== + format : str + Format module name. + event : Event + args, kwargs + Passed to 'vcpu_transform_args'. + + Returns + ======= + Arguments + The transformed arguments, including the non-implicit ones. + + """ + if "vcpu" in event.properties: + ok, func = try_import("tracetool.format." + format, + "vcpu_transform_args") + assert ok + assert func + return Arguments([func(event.args[:1], *args, **kwargs), + event.args[1:]]) + else: + return event.args diff --git a/scripts/update-acpi.sh b/scripts/update-acpi.sh deleted file mode 100644 index b5f05ff3c..000000000 --- a/scripts/update-acpi.sh +++ /dev/null @@ -1,4 +0,0 @@ -cd x86_64-softmmu -for file in hw/i386/*.hex; do - cp -f $file ../$file.generated -done diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh index 096d0900c..f7d62d974 100755 --- a/scripts/update-linux-headers.sh +++ b/scripts/update-linux-headers.sh @@ -36,6 +36,7 @@ cp_portable() { -e 'linux/types' \ -e 'stdint' \ -e 'linux/if_ether' \ + -e 'input-event-codes' \ -e 'sys/' \ > /dev/null then @@ -48,6 +49,7 @@ cp_portable() { -e 's/__s\([0-9][0-9]*\)/int\1_t/g' \ -e 's/__le\([0-9][0-9]*\)/uint\1_t/g' \ -e 's/__be\([0-9][0-9]*\)/uint\1_t/g' \ + -e 's/"\(input-event-codes\.h\)"/"standard-headers\/linux\/\1"/' \ -e 's/<linux\/\([^>]*\)>/"standard-headers\/linux\/\1"/' \ -e 's/__bitwise__//' \ -e 's/__attribute__((packed))/QEMU_PACKED/' \ @@ -101,7 +103,7 @@ done rm -rf "$output/linux-headers/linux" mkdir -p "$output/linux-headers/linux" for header in kvm.h kvm_para.h vfio.h vhost.h \ - psci.h; do + psci.h userfaultfd.h; do cp "$tmpdir/include/linux/$header" "$output/linux-headers/linux" done rm -rf "$output/linux-headers/asm-generic" @@ -128,13 +130,15 @@ EOF rm -rf "$output/include/standard-headers/linux" mkdir -p "$output/include/standard-headers/linux" for i in "$tmpdir"/include/linux/*virtio*.h "$tmpdir/include/linux/input.h" \ + "$tmpdir/include/linux/input-event-codes.h" \ "$tmpdir/include/linux/pci_regs.h"; do cp_portable "$i" "$output/include/standard-headers/linux" done cat <<EOF >$output/include/standard-headers/linux/types.h -#include <stdint.h> -#include "qemu/compiler.h" +/* For QEMU all types are already defined via osdep.h, so this + * header does not need to do anything. + */ EOF cat <<EOF >$output/include/standard-headers/linux/if_ether.h #define ETH_ALEN 6 diff --git a/scripts/vmstate-static-checker.py b/scripts/vmstate-static-checker.py index b6c0bbead..b5ecaf644 100755 --- a/scripts/vmstate-static-checker.py +++ b/scripts/vmstate-static-checker.py @@ -99,6 +99,7 @@ def get_changed_sec_name(sec): # Section names can change -- see commit 292b1634 for an example. changes = { "ICH9 LPC": "ICH9-LPC", + "e1000-82540em": "e1000", } for item in changes: |